From b0ecf1502906716158f4ea464e854ceda0142650 Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Fri, 16 Oct 2026 22:06:02 +0200 Subject: [PATCH] Build indexes and rebuild joint rollup history in the background --- README.md | 3 +- .../server/SampleBenchmark.cs | 2 + .../ui/package.json | 14 +- .../docs/operations/deployment/deployment.mdx | 10 +- .../operations/deployment/wait-for-indexes.cs | 1 + .../observability/observability.mdx | 17 +- .../docs/performance/rollups/query-plan.cs | 3 + .../docs/performance/rollups/rollups.mdx | 1 + .../secondary-indexes/query-plan.cs | 3 + .../secondary-indexes/secondary-indexes.mdx | 12 +- .../reference/configuration/configuration.mdx | 2 +- .../content/docs/reference/engine/engine.mdx | 13 +- .../docs/reference/http-api/http-api.mdx | 5 +- .../reference/http-api/query-response.json | 1 + docs-site/content/docs/reference/index.mdx | 3 +- .../reference/query-model/query-model.mdx | 4 +- docs-site/package.json | 6 +- .../storage-format.md | 53 ++- .../ReportingResponseMapper.cs | 2 +- .../ReportingResponses.cs | 2 +- .../AspNetCoreTests.cs | 2 +- .../BackfillLoggingTests.cs | 33 +- .../BackgroundIndexingTests.cs | 390 +++++++++++++++++ .../IndexCrashRecoveryTests.cs | 11 +- .../ReportingSerializationTests.cs | 4 +- .../RollupValueLayoutTests.cs | 41 +- .../StorageFormatRegressionTests.cs | 24 +- .../TestEngine.cs | 20 +- src/ZoneTree.Analytics/AnalyticsEngine.cs | 89 +++- src/ZoneTree.Analytics/AnalyticsExceptions.cs | 17 +- src/ZoneTree.Analytics/AnalyticsStatistics.cs | 7 + .../Query/AnalyticsResult.cs | 10 +- src/ZoneTree.Analytics/Query/FunnelEngine.cs | 11 +- src/ZoneTree.Analytics/Query/QueryContext.cs | 2 + src/ZoneTree.Analytics/Query/QueryEngine.cs | 35 +- .../Query/QueryPreparation.cs | 4 +- .../Query/QueryResultBuilder.cs | 2 +- .../Storage/DimensionIndexStore.cs | 313 ++++++++++---- .../Storage/IndexMaintenance.cs | 127 ++++++ .../Storage/IndexRecovery.cs | 68 +-- .../Storage/JointBackfillCheckpoint.cs | 49 ++- .../Storage/JointRollupReconstruction.cs | 395 ++++++++++++++++++ src/ZoneTree.Analytics/Storage/KeyCodec.cs | 7 + .../Storage/PartitionedStore.cs | 16 + .../Storage/RetentionClock.cs | 29 +- .../Storage/RollupActivations.cs | 23 +- .../Storage/RollupIndexBackfill.cs | 221 ---------- .../Storage/RollupIndexStore.cs | 295 +++++++++---- src/ZoneTree.Analytics/Storage/RollupStore.cs | 32 +- src/ZoneTree.Analytics/Storage/RollupValue.cs | 83 +++- .../Storage/RollupWriteBatch.cs | 56 ++- .../Storage/{StartupLog.cs => StageLog.cs} | 22 +- yarn.lock | 198 +++++---- 53 files changed, 2155 insertions(+), 638 deletions(-) create mode 100644 docs-site/content/docs/operations/deployment/wait-for-indexes.cs rename docs-site/content/docs/reference/storage-format/storage-format.mdx => internal-docs/storage-format.md (85%) create mode 100644 src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs create mode 100644 src/ZoneTree.Analytics/Storage/IndexMaintenance.cs create mode 100644 src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs delete mode 100644 src/ZoneTree.Analytics/Storage/RollupIndexBackfill.cs rename src/ZoneTree.Analytics/Storage/{StartupLog.cs => StageLog.cs} (66%) diff --git a/README.md b/README.md index d02d99b..2a54a2a 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ Use `analytics.GetStatistics()` and host logging to monitor the engine. See [dep | [Product analytics][product] | [Secondary indexes][indexes] | | [Service telemetry][telemetry] | [HTTP API reference][http-api] | | [Sites and tenants][tenants] | [Extensibility][extensibility] | -| [Deployment and lifecycle][deployment] | [Storage format][storage-format] | +| [Deployment and lifecycle][deployment] | | ## Build and contribute @@ -292,7 +292,6 @@ Released under the [MIT license](https://github.com/ZoneTree/ZoneTree.Analytics/ [accuracy]: https://analytics.zonetree.dev/docs/concepts/time-and-accuracy/ [architecture]: https://analytics.zonetree.dev/docs/concepts/architecture/ [configuration]: https://analytics.zonetree.dev/docs/reference/configuration/ -[storage-format]: https://analytics.zonetree.dev/docs/reference/storage-format/ [web-reports]: https://analytics.zonetree.dev/docs/guides/web-reports/ [rollups]: https://analytics.zonetree.dev/docs/performance/rollups/ [product]: https://analytics.zonetree.dev/docs/guides/product-analytics/ diff --git a/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs b/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs index d366190..278bebb 100644 --- a/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs +++ b/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs @@ -89,6 +89,8 @@ async Task MeasureAsync(string directory, AnalyticsEvent[ compacting = true; progress(new("compacting", events.Length, events.Length, statistics.EventsProcessed, stopwatch.Elapsed.TotalMilliseconds, statistics.EventsProcessed / Math.Max(0.000001, stopwatch.Elapsed.TotalSeconds))); + // Reports measure queries that use the indexes. + await engine.WaitForIndexesAsync(applicationStopping); await engine.CompactAsync(applicationStopping); if (engine.GetStatistics(includePartitions: false).Errors > 0) throw new InvalidOperationException("The benchmark dataset could not be finalized successfully."); diff --git a/apps/ZoneTree.Analytics.Dashboard/ui/package.json b/apps/ZoneTree.Analytics.Dashboard/ui/package.json index ffa409f..0bd0ade 100644 --- a/apps/ZoneTree.Analytics.Dashboard/ui/package.json +++ b/apps/ZoneTree.Analytics.Dashboard/ui/package.json @@ -10,13 +10,13 @@ "publish": "purestack publish --content ./content" }, "dependencies": { - "@purestack/ts-components": "^1.1.12", - "@purestack/ts-css": "^1.1.12", - "@purestack/ts-html": "^1.1.12", - "@purestack/ts-style": "^1.1.12", - "@purestack/ts-svg-icons": "^1.1.12", - "@purestack/ts-util": "^1.1.12", - "purestack": "^1.1.11", + "@purestack/ts-components": "^1.1.14", + "@purestack/ts-css": "^1.1.14", + "@purestack/ts-html": "^1.1.14", + "@purestack/ts-style": "^1.1.14", + "@purestack/ts-svg-icons": "^1.1.14", + "@purestack/ts-util": "^1.1.14", + "purestack": "^1.1.14", "regor": "^1.7.5" }, "devDependencies": { diff --git a/docs-site/content/docs/operations/deployment/deployment.mdx b/docs-site/content/docs/operations/deployment/deployment.mdx index 55b0054..88b46ad 100644 --- a/docs-site/content/docs/operations/deployment/deployment.mdx +++ b/docs-site/content/docs/operations/deployment/deployment.mdx @@ -31,9 +31,15 @@ Metadata, event history, aggregates, sessions, and logs belong to the dataset. C ## Account for startup -Startup checks configuration and index coverage. Clean reuse can be quick; adding indexes to a large retained dataset can require substantial backfill work. +Startup checks configuration and index coverage. Adding indexes to a large retained dataset can require substantial backfill work, which runs in the background: reports stay available meanwhile, and use the new indexes once it completes. -Use initialization logs to explain startup duration and set readiness budgets accordingly. The dashboard prepares retained sample datasets before serving their reports. +Use initialization logs to follow the backfill. To delay readiness until background index work finishes, await: + + + +The wait includes index builds, repairs, and reconstruction of indexed rollup history. It raises `AnalyticsStorageException` if an index cannot be built in this engine instance; queries continue using the available sources, and reopening the engine resumes the build. Canceling the token stops waiting without stopping background work. + +The dashboard prepares retained sample datasets before serving their reports. ## Close cleanly diff --git a/docs-site/content/docs/operations/deployment/wait-for-indexes.cs b/docs-site/content/docs/operations/deployment/wait-for-indexes.cs new file mode 100644 index 0000000..10da65e --- /dev/null +++ b/docs-site/content/docs/operations/deployment/wait-for-indexes.cs @@ -0,0 +1 @@ +await analytics.WaitForIndexesAsync(cancellationToken); diff --git a/docs-site/content/docs/operations/observability/observability.mdx b/docs-site/content/docs/operations/observability/observability.mdx index 98004d0..a5dfdf9 100644 --- a/docs-site/content/docs/operations/observability/observability.mdx +++ b/docs-site/content/docs/operations/observability/observability.mdx @@ -18,7 +18,7 @@ Use engine statistics, initialization logs, and query plans to understand collec | Live visitors decrease | Identities leaving the recent sliding window | [Time and freshness](../../concepts/time-and-accuracy/time-and-accuracy.mdx#report-freshness) | | A filtered report is much slower | Plan sources and required joint dimensions | [Explain a slow report](#explain-a-slow-report) | | Country rows are missing | Geolocation adapter or trusted country dimensions | [Website context](../../concepts/data-model/data-model.mdx#standard-website-context) | -| Startup takes longer after configuration changes | Index necessity checks and backfill progress | [Explain startup duration](#explain-startup-duration) | +| Reports are slower after an index change | Plan notices and backfill progress | [Explain index work](#explain-index-work) | | Reporting returns 403 | Host policy and AuthorizeSite for the normalized site | [Sites and tenants](../../guides/sites-and-tenants/sites-and-tenants.mdx) | | Reporting returns 503 | Error body, concurrency, timeout, and response-size limits | [HTTP limits](../../reference/http-api/http-api.mdx#limits-and-authorization) | @@ -38,10 +38,11 @@ Statistics cover the current engine instance. Lightweight polling with `includeP | EventsFiltered grows | Collection policy is excluding events | | Errors grows | Extension, processing, storage, or maintenance failures | | Flushes stops advancing during activity | Aggregate-saving progress or storage health | +| IndexMaintenance has a value | An index build, repair, or rollup history reconstruction is running | Accepted and processed counters can differ while background work is active. Bot counters distinguish excluded and separately reported automated traffic. -## Explain startup duration +## Explain index work Enable Information logging for `ZoneTree.Analytics` in an ASP.NET Core host to see initialization detail. The standalone core defaults to warning-level console output. @@ -55,9 +56,15 @@ For a standalone engine, set its logger before opening it: An engine opened outside dependency injection in an ASP.NET Core host can also use the integration's `UseLogger` extension to forward engine and storage messages to a host logger. -Backfill checks report whether stores are disabled, reusable, need repair of affected days, or require rebuilding. Work logs report durable checkpoints, resumed event or row counts, completion duration, and throughput. Each identifies the dataset directory. +Backfill checks report whether stores are disabled, reusable, need repair of affected days, or require rebuilding. Work logs report saved checkpoints, resumed event or row counts, completion duration, and throughput. Each identifies the dataset directory. -These logs distinguish several engines in one host and explain why a restart can take longer after changing index configuration. +These logs distinguish several engines in one host. `Plan.Notices` identifies temporary query fallbacks during background index work; the [index guide](../../performance/secondary-indexes/secondary-indexes.mdx#plan-backfill-work) explains how to interpret them. + +`IndexMaintenance` is null when no job is running, including after a failed build. Use `WaitForIndexesAsync` to confirm successful completion. A failed build raises `AnalyticsStorageException` through that wait and is logged; the remaining jobs still run. Queries keep using available sources until the engine reopens and resumes the failed build. + +## Explain shutdown duration + +Disposal logs each stage it waits for, such as draining queued events and closing each store, with its duration. Use them to choose the host's shutdown timeout. ## Explain a slow report @@ -69,7 +76,7 @@ These logs distinguish several engines in one host and explain why a restart can The number of rows displayed is not the number of records read. A filtered card can do far more work than the same unfiltered card. -The [rollup guide](../../performance/rollups/rollups.mdx#follow-a-reports-plan) includes code for inspecting sources, index dimensions, and warnings. Capture the exact site, range, filter values, metrics, and whether the read was cold or repeated with the timing. +The [rollup guide](../../performance/rollups/rollups.mdx#follow-a-reports-plan) includes code for inspecting sources, index dimensions, warnings, and notices. Capture the exact site, range, filter values, metrics, and whether the read was cold or repeated with the timing. If the plan reads raw events, first check whether the requested joint rollup is missing. If it already reads rollups but visits many irrelevant aggregate rows, consider an equality-leading [rollup index](../../performance/secondary-indexes/secondary-indexes.mdx). Increasing raw-scan parallelism does not address an aggregate-read bottleneck. diff --git a/docs-site/content/docs/performance/rollups/query-plan.cs b/docs-site/content/docs/performance/rollups/query-plan.cs index 9845e82..a9a6008 100644 --- a/docs-site/content/docs/performance/rollups/query-plan.cs +++ b/docs-site/content/docs/performance/rollups/query-plan.cs @@ -8,3 +8,6 @@ foreach (var warning in result.Plan.Warnings) Console.WriteLine($"Warning: {warning}"); + +foreach (var notice in result.Plan.Notices) + Console.WriteLine($"Notice: {notice}"); diff --git a/docs-site/content/docs/performance/rollups/rollups.mdx b/docs-site/content/docs/performance/rollups/rollups.mdx index af169b8..2f40a83 100644 --- a/docs-site/content/docs/performance/rollups/rollups.mdx +++ b/docs-site/content/docs/performance/rollups/rollups.mdx @@ -40,6 +40,7 @@ It can read daily summaries for aligned days, hourly or minute summaries for fin | UsedDimensionIndexes | An ordered copy narrowed raw reads | | UsedRollupIndexes | A covering equality index read aggregate rows | | Warnings | A condition affects coverage or interpretation | +| Notices | Temporary source fallback during background index work | Each step names an event, a time range, and a source. Indexed steps also identify the equality dimension. diff --git a/docs-site/content/docs/performance/secondary-indexes/query-plan.cs b/docs-site/content/docs/performance/secondary-indexes/query-plan.cs index 9845e82..a9a6008 100644 --- a/docs-site/content/docs/performance/secondary-indexes/query-plan.cs +++ b/docs-site/content/docs/performance/secondary-indexes/query-plan.cs @@ -8,3 +8,6 @@ foreach (var warning in result.Plan.Warnings) Console.WriteLine($"Warning: {warning}"); + +foreach (var notice in result.Plan.Notices) + Console.WriteLine($"Notice: {notice}"); diff --git a/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx b/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx index bba9524..de1243f 100644 --- a/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx +++ b/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx @@ -56,13 +56,19 @@ The default ten-million-row partition target counts combined copies. An event ca A new partition does not hide earlier data. Queries read every overlapping partition needed for their selected range. -## Plan startup work +## Plan backfill work Opening the engine checks whether indexes can be reused. Missing or incomplete dimension indexes rebuild from retained raw events. Index-managed joint combinations can rebuild eligible aggregate history, then populate their ordered rollup stores. -Backfills complete before normal ingestion starts. Completed coverage is persisted immediately, so killing an idle process does not invalidate its indexes. After interrupted writes, startup repairs the affected days from authoritative data, including missing copies and orphaned copies from interrupted deletions. +Backfills run in the background after the engine opens, while it ingests events and answers queries. Queries use raw events or primary rollups while an index or its rollup history is being built. `Plan.Notices` explains the temporary fallback; notices do not affect `Plan.IsComplete`. Continue to check `Plan.Warnings` for coverage or interpretation concerns. -Interrupted backfills resume from durable checkpoints. Copy backfills checkpoint every 100,000 input records and at partition boundaries. Historical joint-rollup reconstruction also preserves active sessions and replays absolute replacements, so resuming does not count events twice. +Await `WaitForIndexesAsync` before measuring query performance, or to delay readiness until the work finishes. See [deployment and lifecycle](../../operations/deployment/deployment.mdx#account-for-startup) for an example and failure behavior. + +Completed coverage is recorded for reuse after reopening. After interrupted writes, the engine repairs the affected days from authoritative data, including missing copies and orphaned copies from interrupted deletions. + +Interrupted backfills resume from saved checkpoints. Copy backfills checkpoint every 100,000 input records and at partition boundaries. Historical joint-rollup reconstruction also preserves active sessions and replays absolute contributions to avoid counting the same chunk twice. + +Recovery follows the configured write-ahead log (WAL) mode. The default `AsyncCompressed` mode can lose recent writes in a crash, including index copies or reconstructed aggregate contributions. Raw events can remain stored while their reconstructed aggregates miss those contributions. Use synchronous WAL for the stores involved when checkpoint recovery must retain every completed write. Information logs report reuse, repair, or rebuild decisions, checkpoint progress, resumed counts, duration, and throughput. Configuration changes, missing retained partitions, and stores configured without a write-ahead log can still require a full rebuild. diff --git a/docs-site/content/docs/reference/configuration/configuration.mdx b/docs-site/content/docs/reference/configuration/configuration.mdx index d3128fe..0dec5d8 100644 --- a/docs-site/content/docs/reference/configuration/configuration.mdx +++ b/docs-site/content/docs/reference/configuration/configuration.mdx @@ -13,7 +13,7 @@ Use this page to locate a setting, check its default, and understand its depende Set group members and add to collections in place. Groups are created by `AnalyticsOptions`; they are not replaced wholesale. Options are cloned when an engine opens, so edits afterward do not reconfigure it. -Reporting time zone, sketch precision, and the chosen separate write-ahead-log directory are persisted dataset choices. Reopening incompatible settings fails. Index changes can trigger startup backfills; ordinary rollup and aggregate changes can introduce activation boundaries. +Reporting time zone, sketch precision, and the chosen separate write-ahead-log directory are persisted dataset choices. Reopening incompatible settings fails. Index changes can trigger background backfills; ordinary rollup and aggregate changes can introduce activation boundaries. ## Find a setting diff --git a/docs-site/content/docs/reference/engine/engine.mdx b/docs-site/content/docs/reference/engine/engine.mdx index b5e4506..5d09b65 100644 --- a/docs-site/content/docs/reference/engine/engine.mdx +++ b/docs-site/content/docs/reference/engine/engine.mdx @@ -61,10 +61,15 @@ Storage-reading asynchronous forms run the work on a thread-pool thread. They do | Compact / CompactAsync | Flush and merge open stores toward disk | | ApplyRetention / ApplyRetentionAsync | Run physical retention maintenance | | EraseVisitor / EraseVisitorAsync | Remove an explicit visitor's retained raw events, index copies, and session | +| WaitForIndexes / WaitForIndexesAsync | Wait for index builds, repairs, and reconstruction of indexed rollup history | | Dispose / DisposeAsync | Stop ingestion, drain accepted work, flush, and release ownership | Erasure returns deleted raw-event count and does not subtract already aggregated rollup history. Compaction is useful after bulk imports but is separate from ordinary periodic flushing. +Index work runs in the background after the engine opens. Queries use the available sources meanwhile; `Plan.Notices` explains temporary fallbacks. See [secondary indexes](../../performance/secondary-indexes/secondary-indexes.mdx#plan-backfill-work) for query and recovery behavior. + +`WaitForIndexesAsync` completes immediately when there is no work. It raises `AnalyticsStorageException` if an index cannot be built in this engine instance, after the remaining jobs have run. Reopening resumes the failed build. Canceling the caller's token cancels only the wait; disposing the engine cancels unfinished background work. + ## Statistics `GetStatistics()` includes partition information. `GetStatistics(includePartitions:false)` gives lightweight counters without inspecting those stores. @@ -72,15 +77,17 @@ Erasure returns deleted raw-event count and does not subtract already aggregated | Group | Fields | | --- | --- | | Ingestion | EventsAccepted, EventsProcessed, EventsDropped, EventsRejected, EventsFiltered | -| Health | QueueLength, Errors, Flushes | +| Health | QueueLength, Errors, Flushes, IndexMaintenance | | Web activity | SessionsStarted, BotsExcluded, BotsTracked | | Storage | OpenPartitions, PartitionsDropped, Partitions | Partition record counts are approximate and may be null for partitions not opened since startup. Counters start again with a new engine instance. +`IndexMaintenance` names the current job: `"dimension indexes"`, `"joint rollup history"`, or `"rollup indexes"`. Null means no job is running; use `WaitForIndexesAsync` to distinguish successful completion from a failed build. + ## Errors and cancellation -`AnalyticsException` is the base for engine-specific errors. `AnalyticsConfigurationException` reports incompatible options or dataset configuration. `AnalyticsQueryException` reports unanswerable or bounded queries; `AnalyticsQueryTimeoutException` is its timeout subclass. `AnalyticsStorageException` reports a flush whose rollup rows could not be written; its inner exception is the storage failure, and the rows are retried by later flushes. +`AnalyticsException` is the base for engine-specific errors. `AnalyticsConfigurationException` reports incompatible options or dataset configuration. `AnalyticsQueryException` reports unanswerable or bounded queries; `AnalyticsQueryTimeoutException` is its timeout subclass. `AnalyticsStorageException` reports data that could not be written to storage: a flush whose rollup rows are retried by later flushes, or, from `WaitForIndexesAsync`, an index build that requires reopening to resume. Its inner exception carries the underlying storage failure when available. Invalid arguments use standard argument exceptions. Caller cancellation uses the cancellation token; configured query timeout additionally protects expensive report execution. @@ -132,8 +139,6 @@ than ZoneTree and is not exposed. For shared indexes, restrict iterators using the returned logical `KeyPrefix`. Each accessor exposes a logical namespace within the shared physical store. -See the [storage format reference](../storage-format/storage-format.mdx) for directory layouts, exact key and payload encodings, metadata generations, and recovery behavior. - ## Related pages - [First events and reports](../../getting-started/first-events/first-events.mdx) diff --git a/docs-site/content/docs/reference/http-api/http-api.mdx b/docs-site/content/docs/reference/http-api/http-api.mdx index 24444cd..454508d 100644 --- a/docs-site/content/docs/reference/http-api/http-api.mdx +++ b/docs-site/content/docs/reference/http-api/http-api.mdx @@ -139,10 +139,13 @@ A general query returns metric-keyed objects. Here is an example response for a | totalRows | Result-row count before global pagination | | previous | Comparison object, when one is available | | plan.complete / warnings | Historical coverage and interpretation concerns | +| plan.notices | Temporary source fallback during background index work | Numeric rates such as bounce rate and conversion rate are fractions: `0.25` means 25%. Built-in session durations are seconds. Custom numeric values retain their producer's unit, such as `duration_ms` or `revenue_usd`. -Plans include source steps, warnings, completeness, and rollup suggestions. Use them when historical coverage matters. +Plans include source steps, warnings, notices, completeness, and rollup suggestions. Use them when historical coverage matters. + +Notices do not affect `plan.complete`; warnings still describe coverage and interpretation concerns. HTTP plans expose `steps[].source` and optional `steps[].indexDimension`. The .NET-only convenience flags such as `UsedRollupIndexes` are not additional fields in the JSON contract. diff --git a/docs-site/content/docs/reference/http-api/query-response.json b/docs-site/content/docs/reference/http-api/query-response.json index a66a7d0..604b026 100644 --- a/docs-site/content/docs/reference/http-api/query-response.json +++ b/docs-site/content/docs/reference/http-api/query-response.json @@ -18,6 +18,7 @@ "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "purchase", diff --git a/docs-site/content/docs/reference/index.mdx b/docs-site/content/docs/reference/index.mdx index 16fcc3e..e3b4375 100644 --- a/docs-site/content/docs/reference/index.mdx +++ b/docs-site/content/docs/reference/index.mdx @@ -11,7 +11,7 @@ Look up exact API contracts and defaults. These pages complement the worked exam ## Find the contract -Use the [engine API](./engine/engine.mdx) for entry points, [query model](./query-model/query-model.mdx) for calculation semantics, [configuration](./configuration/configuration.mdx) for settings, [HTTP API](./http-api/http-api.mdx) for integration, and [storage format](./storage-format/storage-format.mdx) for persisted layouts and recovery contracts. +Use the [engine API](./engine/engine.mdx) for entry points, [query model](./query-model/query-model.mdx) for calculation semantics, [configuration](./configuration/configuration.mdx) for settings, and [HTTP API](./http-api/http-api.mdx) for integration. ## In this section @@ -20,7 +20,6 @@ Use the [engine API](./engine/engine.mdx) for entry points, [query model](./quer - ## Continue exploring diff --git a/docs-site/content/docs/reference/query-model/query-model.mdx b/docs-site/content/docs/reference/query-model/query-model.mdx index 2b746c2..32e4829 100644 --- a/docs-site/content/docs/reference/query-model/query-model.mdx +++ b/docs-site/content/docs/reference/query-model/query-model.mdx @@ -170,7 +170,7 @@ Use them in grouping, filters, and conditional metrics. They are computed rather | --- | --- | | AnalyticsResult | From, To, TimeZone, Rows, TotalRows, Totals, Comparison, Plan | | AnalyticsRow | Bucket, Dimensions, Values, Previous, metric and dimension indexers | -| QueryPlan | Steps, Warnings, RollupSuggestions, IsComplete, source-use flags | +| QueryPlan | Steps, Warnings, Notices, RollupSuggestions, IsComplete, source-use flags | | QueryPlanStep | Event, Source, From, To, IndexDimension | | FunnelResult | From, To, Steps, ConversionRate, Plan | @@ -178,6 +178,8 @@ Use them in grouping, filters, and conditional metrics. They are computed rather Plan sources are DayRollups, HourRollups, MinuteRollups, RawEvents, and DimensionIndex. A covering rollup index keeps its tier source and sets `IndexDimension`. Suggestions identify missing ordinary rollup definitions; they do not make historical coverage complete. +`Notices` describes temporary source fallbacks during background index work and does not affect `IsComplete`. `Warnings` describes coverage and interpretation concerns. See [secondary indexes](../../performance/secondary-indexes/secondary-indexes.mdx#plan-backfill-work) for background work and recovery behavior. + ## Standard web vocabulary | Dimension | Source | diff --git a/docs-site/package.json b/docs-site/package.json index 59c8872..ea67e7e 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -11,9 +11,9 @@ "deploy": "pgdeploy deploy --host https://analytics.zonetree.dev --package ./dist/web --allow-path 'pagefind/*,.ts-ssg/manifest.json' --skip-path 'docs/*.json,*.cs,*.http,*.ps1,*.sh'" }, "dependencies": { - "@purestack/ts-style": "^1.1.12", - "@purestack/ts-util": "^1.1.12", - "purestack": "^1.1.11", + "@purestack/ts-style": "^1.1.14", + "@purestack/ts-util": "^1.1.14", + "purestack": "^1.1.14", "regor": "^1.7.5" }, "devDependencies": { diff --git a/docs-site/content/docs/reference/storage-format/storage-format.mdx b/internal-docs/storage-format.md similarity index 85% rename from docs-site/content/docs/reference/storage-format/storage-format.mdx rename to internal-docs/storage-format.md index d36dd0c..21c48e1 100644 --- a/docs-site/content/docs/reference/storage-format/storage-format.mdx +++ b/internal-docs/storage-format.md @@ -1,15 +1,9 @@ ---- -title: "Storage format" -description: "Directory layout, binary keys and values, metadata, partition routing, and recovery contracts." -order: 5 -template: doc ---- # Storage format ZoneTree.Analytics stores a dataset as several ordered ZoneTree stores plus small, checksummed metadata files. This reference describes the format implemented by the current source: dataset `format-version` **2**, dimension-index format **2**, and rollup-index format **2**. -Use this page to inspect a dataset, understand storage costs, or review a codec change. For application code, prefer the public [storage accessors and codecs](../engine/engine.mdx#advanced-storage-access). Those APIs return typed records without requiring callers to reproduce byte offsets. +Use this page to inspect a dataset, understand storage costs, or review a codec change. For application code, prefer the public [storage accessors and codecs](../docs-site/content/docs/reference/engine/engine.mdx#advanced-storage-access). Those APIs return typed records without requiring callers to reproduce byte offsets. ## Format boundaries and compatibility @@ -196,6 +190,7 @@ Keys order by site, event, dimension set, bucket, then combined values. All valu | `02` | Visitor sketch section | | `04` | Value detail section | | `08` | Distinct-dimension sketch section | +| `10` | Rebuilt part | Live flags combine with bitwise OR. Readers reject other bits, a live header without the present bit, and trailing bytes. A live row containing no sketches, measured values, or optional details still has a header, event count, and zero measured-value count. Count zero does not make a row a tombstone: session and page metrics can contribute sum-only updates without adding another event. @@ -210,11 +205,14 @@ Live flags combine with bitwise OR. Readers reject other bits, a live header wit | 5 | Visitor section | `vu` byte length, then the HLL layout below; only with header bit `02` | | 6 | Detail section | `vu` byte length, then `vu` detail count equal to the measured value count, then per measured value a `vu` entry length and its detail record; length `0` means the value has no details; only with header bit `04` | | 7 | Distinct section | `vu` byte length, then `vu` dimension count, then per dimension its name `str`, a nonzero `vu` entry length, and its HLL sketch; only with header bit `08` | +| 8 | Rebuilt part | `vu` byte length, then a live value in this same layout, without a rebuilt part of its own; only with header bit `10` | Sections are nonempty and appear in this order, after every scalar field. The reader decodes the count and the scalar values with the row, checks each section's length and the detail count, and keeps the sections as stored bytes. When the value arrives as a slice of a larger buffer, such as a decompressed disk block, the sections are first copied into a buffer of their own, so a cached or merged row never keeps the block alive. Decoding happens one part at a time, the first time a query or a merge reads it: the visitor section, then a single value detail or a single distinct sketch. Reading a section's entry lengths and dimension names does not decode any sketch. A query that reads only counts and sums therefore never decodes a sketch, and a query for one distinct dimension or one value's percentiles decodes only that entry. A merge or index copy writes every section and entry it leaves unchanged back from its stored bytes, without re-encoding it. +A rebuilt part remains in a row while historical joint-rollup reconstruction runs or its output awaits folding, including across restarts. It holds the contributions the reconstruction has made to the row so far, apart from what ingestion merges into the row meanwhile, and readers ignore it. Ingestion keeps it unchanged, and the reconstruction replaces it with absolute values, so each can repeat its writes after a failure without undoing the other's. A row holding only a rebuilt part has no ordinary data and is skipped by queries. After reconstruction completes, the rollup index build merges the rebuilt parts into their rows and removes the rebuilt sections. A build also folds pending rows of retained definitions whose indexes were removed. + Measured-value names and distinct-dimension names are stored in ordinal order by the engine. Empty visitor sketches are omitted. A distinct-dimension sketch represents distinct values of that named dimension, independently of the row's visitor sketch. Each scalar measured value has this layout: @@ -383,6 +381,7 @@ The JSON body maps string keys to string values, with keys emitted in ordinal or | `rollup-index.dirty.` | Per-tier day intents, with tier and day ticks in the key suffix | | `rollup-index.backfill.` | Rollup copy-backfill configuration, phase, cursor, and count | | `rollup-index.backfill-pending` | Newline-separated pending historical joint-rollup definitions | +| `rollup-index.folding` | Completed reconstructions whose rebuilt parts the rollup index build folds in: per line, the coverage start ticks, a tab, and the definition's activation key | ### Generations and writes @@ -390,13 +389,13 @@ A metadata update creates the next generation by replacing the older file in pla When keys are removed, the store writes two successive generations so the removed entries disappear from both files. This applies to expired salts as well as completed recovery checkpoints. It is removal from the live metadata files, not secure erasure of historical backups or underlying storage media. -Activation metadata records historical coverage. Definitions and aggregates present at dataset creation start at tick zero. A newly enabled definition, aggregate, or tier on an existing dataset starts at its activation time; removing and later restoring it does not claim continuous coverage of the gap. Eligible historical joint-rollup reconstruction can publish earlier coverage after its durable backfill completes. Existing row bytes alone are insufficient to infer complete coverage. +Activation metadata records historical coverage. Definitions and aggregates present at dataset creation start at tick zero. A newly enabled definition, aggregate, or tier on an existing dataset starts at its activation time; removing and later restoring it does not claim continuous coverage of the gap. Eligible historical joint-rollup reconstruction can publish earlier coverage after its durable backfill completes and its rows are folded in. Existing row bytes alone are insufficient to infer complete coverage. ## Writes, durability, and recovery An accepted event first enters the ingestion queue. Raw records, session snapshots, rollups, and index copies are then maintained through different write paths. There is no dataset-wide transaction committing all of them together. -`StoreOptions.WriteAheadLogMode` defaults to `AsyncCompressed`. Persistent tree writes follow the configured ZoneTree WAL mode. A successful tracking call reports queue acceptance, and a crash can lose queued events and buffered aggregates. `Flush` drains prior accepted ingestion work and writes buffered aggregates and session state; it does not make every configured WAL mode synchronous. Clean disposal drains work, flushes, closes stores, and releases the directory lock. `Compact` additionally moves open trees toward merged disk storage. +`StoreOptions.WriteAheadLogMode` defaults to `AsyncCompressed`. Persistent tree writes follow the configured ZoneTree WAL mode; in that mode a process that dies can lose its most recent writes. A successful tracking call reports queue acceptance, and a crash can lose queued events and buffered aggregates. `Flush` drains prior accepted ingestion work and writes buffered aggregates and session state; it does not make every configured WAL mode synchronous. Clean disposal drains work, flushes, closes stores, and releases the directory lock. `Compact` additionally moves open trees toward merged disk storage. Ingestion workers do not write rollup rows themselves. On flush, a worker hands its buffered rows to dedicated rollup writers (`Ingestion.RollupWriterCount`), each of which owns a share of the rows by key. A writer merges the rows of all workers that flushed since its last write, writes each row once, and only then lets those workers persist their session states. Rollup writes serialize the updates of each row. A row's key selects one of 64 lock stripes of its partition, so writers update different rows of a partition in parallel and the same row one at a time; its covering index copies are written under the same lock, in order. The writer merges a delta with the existing value once, retains the absolute replacement for retries, and finishes a pending replacement before applying a newer delta in its stripe. This prevents retrying a failed upsert from applying the same contribution twice within that write path. It does not provide crash-atomic coordination between raw events and every aggregate tier. @@ -404,17 +403,22 @@ Ingestion workers do not write rollup rows themselves. On flush, a worker hands The primary raw store is authoritative for dimension copies; primary joint rollups are authoritative for rollup copies. Before changing a source/copy pair, the engine persists a dirty-day intent in metadata. The day is obtained by flooring that record's time coordinate to a whole day; minute/hour/raw coordinates are UTC, while day-rollup coordinates represent local calendar dates. -Startup completes index checks and required repairs before ingestion workers begin: +Opening the engine checks the indexes and decides the work; the work runs on one background thread once ingestion has started, in this order: dimension indexes, historical joint-rollup reconstruction, rollup indexes. Queries do not use an index until its work completes. + +An index copy failure leaves the affected indexes unavailable and their checkpoint incomplete. Maintenance continues with the remaining jobs, then fails `WaitForIndexesAsync` with `AnalyticsStorageException`. Reopening the engine resumes the failed build. `IndexMaintenance` is null whenever no job is running, including after such a failure. -| State | Startup behavior | +| State | Behavior | | --- | --- | | Matching configuration, format, retained partitions, and clean checkpoint | Reuse completed indexes | | Dirty-day intents with an otherwise reusable checkpoint | Remove affected copies and reconstruct those days from authoritative records, including interrupted deletions | -| Configuration change, incomplete checkpoint, or missing retained index partitions | Rebuild the affected shared index stores | +| Interrupted build of the same configuration | Resume from its checkpoint, then repair the days written since it began | +| Configuration change, new indexes, or missing retained index partitions | Drop the copies when the engine opens, then rebuild the shared index stores | | Source or destination WAL disabled | Rebuild instead of trusting a durable completed checkpoint | | Explicit incompatible index format | Fail the format check | -Copy backfills use metadata keys ending in `.config`, `.phase`, `.partition`, `.key`, and `.count`. Phase is `clear` or `copy`; the cursor key is base64, and the count is decimal text. Checkpoints advance after destination data is persisted, every 100,000 input records and at partition boundaries. Absolute upserts make replay of an unfinished chunk idempotent. +Ingestion keeps writing its own copies while a build runs. A dimension-index build reads the raw events up to the sequence stored before the engine opened; ingestion copied the later ones. Copying an event again writes the same key and value. Erasure and each chunk of a build take turns, and every chunk reads the raw events again, so a build never copies an erased event back. A rollup-index build writes each copy under the row lock of the flushes, from the row's current value. + +Copy backfills use metadata keys ending in `.config`, `.phase`, `.partition`, `.key`, and `.count`. Phase is `clear` or `copy`; the cursor key is base64, and the count is decimal text. When a new build has dropped the old copies, the index format, configuration, and empty partition manifest are recorded with the index marked unclean, so an interrupted build resumes. Checkpoints advance after their writes return, every 100,000 input records and at partition boundaries. Absolute upserts make replay of an unfinished chunk idempotent. With the default asynchronous write-ahead log, copy and reconstruction writes can still be queued when the checkpoint advances. A crash can lose them even when their source raw events were already durable, leaving historical aggregates without those contributions. Synchronous logs for the stores involved make checkpoint recovery exact. Historical reconstruction of joint rollups has a separate binary journal, `backfill/joint-rollups/journal-a.bin` and `journal-b.bin`, written alternately like the metadata files: @@ -426,9 +430,11 @@ Historical reconstruction of joint rollups has a separate binary journal, `backf | 4 | Body checksum | 32-byte SHA-256 of the body | | 5 | Body | The checkpoint | -The header is written last, so a torn write never validates; startup reads the valid file with the highest generation. The body holds the configuration signature, coverage, site/partition/key cursor, progress counters, active reconstructed sessions, and a redo list of absolute rollup replacements. Redo records include granularity, partition, primary key bytes, and serialized rollup value bytes, stored as bytes rather than text. Replaying replacements avoids adding the same contributions twice after a crash; completed coverage is published only after the durable reconstruction checkpoint, and the journal files are deleted when reconstruction completes. +The header is written last, so a torn write never validates; startup reads the valid file with the highest generation. The body holds the configuration signature, coverage, the fence, the raw partitions when the reconstruction began, the site/partition/key cursor, progress counters, active reconstructed sessions, and a redo list of absolute rebuilt parts. Redo records include granularity, partition, primary key bytes, and the serialized rebuilt part, stored as bytes rather than text. + +The reconstruction runs while ingestion writes to the same rows. It reads the raw events whose sequence is at most the fence, the raw event sequence when it began; ingestion writes every later event, late ones included, so each event counts once. Its contributions go to the rows' rebuilt parts under the row locks of the flushes. Replaying the absolute rebuilt parts avoids adding the same contributions twice after a crash and never undoes a flush. When the reconstruction begins, it deletes the rows the definitions kept from an earlier activation, before ingestion starts. Raw retention does not pass the coverage start until it completes. Then the journal files are deleted, the definitions are listed in `rollup-index.folding`, and the rollup index build folds the rebuilt parts in before the coverage is published. -Recovery of derived indexes cannot recover raw events lost before persistence or history already removed by retention. Ordinary startup index reconciliation also does not imply that all ordinary raw history is replayed into every primary rollup after a crash. +Recovery of derived indexes cannot recover raw events lost before persistence or history already removed by retention. Ordinary index reconciliation also does not imply that all ordinary raw history is replayed into every primary rollup after a crash. ## Deletion, expiration, and storage reclamation @@ -444,7 +450,7 @@ Maintenance drops fully expired partitions and merges partially expired trees. W Defaults retain raw history for 90 days, minute rollups for two days when enabled, hourly rollups for 400 days, and daily rollups indefinitely. The default enabled tiers are Hour and Day. Null retention means keep indefinitely. Retention is rounded in whole UTC days, or whole hours for durations under one day; daily aggregate comparisons use the corresponding reporting-zone date. -Visitor erasure removes retained raw events, their dimension-index copies, and the active session for an explicit visitor. It does not subtract contributions from existing aggregate rollups or their covering indexes. See [retention and erasure](../../operations/retention-and-erasure/retention-and-erasure.mdx) for the public operation. +Visitor erasure removes retained raw events, their dimension-index copies, and the active session for an explicit visitor. It does not subtract contributions from existing aggregate rollups or their covering indexes. See [retention and erasure](../docs-site/content/docs/operations/retention-and-erasure/retention-and-erasure.mdx) for the public operation. ## Inspecting and transferring a dataset @@ -456,7 +462,7 @@ Index accessors expose a logical namespace inside a shared tree. Restrict iterat Treat stored instances, arrays, and sketches as immutable. Direct tree writes bypass ingestion, aggregation, index coordination, session caches, and coverage metadata. Do not dispose or drop engine-owned trees or maintainers, or create an additional maintainer. Release leases before engine disposal. -For a coordinated file copy, close the engine cleanly and copy the entire data root plus the separate WAL root, if configured. Preserve metadata, sessions, all tiers, and any active recovery journal together. If the identity secret is supplied in configuration, retain that configuration securely as well. Reopening requires the persisted format settings, including the same configured absolute WAL root; relocation of that root is not supported by simply changing the option. Test a restored copy in isolation before depending on it. [Deployment and lifecycle](../../operations/deployment/deployment.mdx) covers operational ownership and backups. +For a coordinated file copy, close the engine cleanly and copy the entire data root plus the separate WAL root, if configured. Preserve metadata, sessions, all tiers, and any active recovery journal together. If the identity secret is supplied in configuration, retain that configuration securely as well. Reopening requires the persisted format settings, including the same configured absolute WAL root; relocation of that root is not supported by simply changing the option. Test a restored copy in isolation before depending on it. [Deployment and lifecycle](../docs-site/content/docs/operations/deployment/deployment.mdx) covers operational ownership and backups. ## Implementation map @@ -477,12 +483,13 @@ These source files define the contracts described here: | Embedded sketches | [HyperLogLog.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Sketches/HyperLogLog.cs), [QuantileSketch.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Sketches/QuantileSketch.cs) | | Index key layouts | [DimensionIndexKeyCodec.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/DimensionIndexKeyCodec.cs), [RollupIndexKeyCodec.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/RollupIndexKeyCodec.cs) | | Metadata generations and historical coverage | [MetaStore.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/MetaStore.cs), [RollupActivations.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/RollupActivations.cs) | -| Copy recovery and historical reconstruction journal | [IndexRecovery.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/IndexRecovery.cs), [JointBackfillCheckpoint.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs) | +| Background index work and its order | [IndexMaintenance.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/IndexMaintenance.cs) | +| Copy recovery, historical reconstruction, and its journal | [IndexRecovery.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/IndexRecovery.cs), [JointRollupReconstruction.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs), [JointBackfillCheckpoint.cs](https://github.com/ZoneTree/ZoneTree.Analytics/blob/main/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs) | ## Related pages -- [Engine API and advanced storage access](../engine/engine.mdx) -- [Secondary indexes and backfilling](../../performance/secondary-indexes/secondary-indexes.mdx) -- [Configuration reference](../configuration/configuration.mdx) -- [Time and accuracy](../../concepts/time-and-accuracy/time-and-accuracy.mdx) -- [Deployment and lifecycle](../../operations/deployment/deployment.mdx) +- [Engine API and advanced storage access](../docs-site/content/docs/reference/engine/engine.mdx) +- [Secondary indexes and backfilling](../docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx) +- [Configuration reference](../docs-site/content/docs/reference/configuration/configuration.mdx) +- [Time and accuracy](../docs-site/content/docs/concepts/time-and-accuracy/time-and-accuracy.mdx) +- [Deployment and lifecycle](../docs-site/content/docs/operations/deployment/deployment.mdx) diff --git a/src/ZoneTree.Analytics.AspNetCore/ReportingResponseMapper.cs b/src/ZoneTree.Analytics.AspNetCore/ReportingResponseMapper.cs index 691e7cf..fc97500 100644 --- a/src/ZoneTree.Analytics.AspNetCore/ReportingResponseMapper.cs +++ b/src/ZoneTree.Analytics.AspNetCore/ReportingResponseMapper.cs @@ -96,7 +96,7 @@ PlanResponse MapPlan(QueryPlan plan) IReadOnlyList warnings = plan.Warnings; if (nonFinite) warnings = [.. warnings, "Non-finite metric values (overflow or undefined arithmetic) are represented as null in this JSON response."]; - return new(plan.IsComplete && !nonFinite, warnings, + return new(plan.IsComplete && !nonFinite, warnings, plan.Notices, plan.Steps.Select(step => new PlanStep(step.Event, step.Source.ToString(), step.From, step.To) { IndexDimension = step.IndexDimension }).ToArray(), plan.RollupSuggestions.Select(suggestion => new RollupSuggestionResponse( diff --git a/src/ZoneTree.Analytics.AspNetCore/ReportingResponses.cs b/src/ZoneTree.Analytics.AspNetCore/ReportingResponses.cs index db6d4d9..2425f16 100644 --- a/src/ZoneTree.Analytics.AspNetCore/ReportingResponses.cs +++ b/src/ZoneTree.Analytics.AspNetCore/ReportingResponses.cs @@ -51,7 +51,7 @@ sealed record FunnelResponse( sealed record FunnelRow( string Name, long Visitors, double? ConversionRate, double? StepConversionRate, long DropOff); -sealed record PlanResponse(bool Complete, IReadOnlyList Warnings, PlanStep[] Steps, RollupSuggestionResponse[] RollupSuggestions); +sealed record PlanResponse(bool Complete, IReadOnlyList Warnings, IReadOnlyList Notices, PlanStep[] Steps, RollupSuggestionResponse[] RollupSuggestions); sealed record PlanStep(string Event, string Source, DateTimeOffset From, DateTimeOffset To) { diff --git a/src/ZoneTree.Analytics.UnitTests/AspNetCoreTests.cs b/src/ZoneTree.Analytics.UnitTests/AspNetCoreTests.cs index e91a974..794c481 100644 --- a/src/ZoneTree.Analytics.UnitTests/AspNetCoreTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/AspNetCoreTests.cs @@ -494,7 +494,7 @@ public async Task ReportingApiSerializationFailureReturnsACompleteErrorResponse( // any part of the success body reaches the client. var metrics = new SummaryMetrics(1, 1, 1, double.NaN, null, 1, null, null); var response = new SummaryResponse(DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, metrics, - new PlanResponse(true, [], [], [])); + new PlanResponse(true, [], [], [], [])); await ReportingApi.Run(context, (_, _, _) => Task.FromResult(response), ReportingJsonContext.Default.SummaryResponse); body.Position = 0; using var document = await JsonDocument.ParseAsync(body); diff --git a/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs b/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs index 9bc60b9..c1a9cb9 100644 --- a/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs @@ -72,8 +72,9 @@ public void MigrationLogsChecksBuildDurationsAndActualCopiesThenCleanRestartOnly Assert.That(Number(copies, "index rows written"), Is.EqualTo(4)); Assert.That(copies, Does.Contain("historical backfill=True")); AssertChecks(provider, "rebuild required"); - var total = Number(Completed(provider, "Rollup index initialization"), "duration"); - Assert.That(total, Is.GreaterThanOrEqualTo(Number(joint, "duration") + Number(copies, "duration") - 0.1)); + // Opening only decides the work: the builds run in the background. + Assert.That(Completed(provider, "Rollup index initialization"), Does.Contain("decision=rebuild in background")); + Assert.That(Completed(provider, "Dimension index initialization"), Does.Contain("decision=rebuild in background")); TestContext.Out.WriteLine(string.Join(Environment.NewLine, provider.Messages.Select(log => log.Message).Where(message => message.Contains("backfill", StringComparison.Ordinal)))); } provider.Messages.Clear(); @@ -128,7 +129,7 @@ public void HostRegistrationAndManuallyCreatedSampleEnginesUseHostLoggingFilters .AddZoneTreeAnalytics(options => { options.DataDirectory = directory; TestEngine.UseTestDefaults(options); Indexes(options); }) .BuildServiceProvider()) { - _ = services.GetRequiredService(); + services.GetRequiredService().WaitForIndexesAsync().GetAwaiter().GetResult(); Assert.That(provider.Messages.Any(log => log.Category == "ZoneTree.Analytics" && log.Message.StartsWith("Rollup index backfill started;", StringComparison.Ordinal)), Is.True); } provider.Messages.Clear(); @@ -142,19 +143,41 @@ public void HostRegistrationAndManuallyCreatedSampleEnginesUseHostLoggingFilters Assert.That(provider.Messages, Is.Empty); } + [Test] + public void ShutdownLogsWhatItWaitsForAndHowLong() + { + var logger = new IndexCrashRecoveryTests.RecoveryLogger(); + var engine = TestEngine.Create(directory, options => { Configure(options); options.Sessions.Enabled = true; options.Storage.Logger = logger; }); + for (var i = 0; i < 3; i++) + Assert.That(engine.Track(new AnalyticsEvent("pageview") { VisitorId = "v" + i, Timestamp = Utc(DateTime.UtcNow.AddMinutes(-i)) }), Is.True); + engine.Dispose(); + var messages = logger.Messages.ToArray(); + string Completed(string stage) => messages.Single(message => message.StartsWith(stage + " completed;", StringComparison.Ordinal)); + Assert.Multiple(() => + { + Assert.That(messages.Single(message => message.StartsWith("Shutdown started;", StringComparison.Ordinal)), Does.Contain("index maintenance=none")); + Assert.That(Completed("Ingestion shutdown"), Does.Contain("events processed=3;").And.Contain("flushes=1")); + Assert.That(messages.Where(message => message.StartsWith("Partition store shutdown completed;", StringComparison.Ordinal)), + Has.Some.Contains("closed partitions=1").And.Some.Contains(Path.Combine("rollups", "day"))); + Assert.That(Completed("Session store shutdown"), Does.Contain("duration=")); + Assert.That(Number(Completed("Shutdown"), "duration"), Is.GreaterThanOrEqualTo(Number(Completed("Ingestion shutdown"), "duration"))); + Assert.That(Array.FindLastIndex(messages, message => message.StartsWith("Shutdown completed;", StringComparison.Ordinal)), Is.EqualTo(messages.Length - 1)); + }); + } + [Test] public void FailedStageReportsElapsedTimeAndThrowingLogProviderDoesNotMaskFailure() { using var provider = new LogProvider(); using var factory = LoggerFactory.Create(logging => logging.AddProvider(provider)); var logger = new LoggerAdapter(factory.CreateLogger("Failure")); - using (var stage = new StartupLog(logger, "Dimension index backfill", directory)) { } + using (var stage = new StageLog(logger, "Dimension index backfill", directory)) { } Assert.That(provider.Messages.Last().Message, Does.StartWith("Dimension index backfill failed;").And.Contain("duration=")); Assert.That(provider.Messages.Any(log => log.Message.StartsWith("Dimension index backfill completed;", StringComparison.Ordinal)), Is.False); provider.Throw = true; Assert.That(() => { - using var stage = new StartupLog(logger, "Joint rollup backfill", directory); + using var stage = new StageLog(logger, "Joint rollup backfill", directory); throw new IOException("original storage failure"); }, Throws.TypeOf().With.Message.EqualTo("original storage failure")); } diff --git a/src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs b/src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs new file mode 100644 index 0000000..83bdade --- /dev/null +++ b/src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs @@ -0,0 +1,390 @@ +using ZoneTree.Analytics.Internal; +using ZoneTree.Analytics.Storage; +using ZoneTree.Analytics.Web; + +namespace ZoneTree.Analytics.UnitTests; + +/// +/// Index builds and history reconstructions run after the engine opens, while it ingests events +/// and answers queries with the same results and notices. +/// +public sealed class BackgroundIndexingTests : DirectoryTest +{ + static DateTime Start => DateTime.UtcNow.Date.AddDays(-5); + + static void View(AnalyticsEngine engine, string visitor, DateTime at, string page, string source) => + Assert.That(engine.Track(new AnalyticsEvent(StandardEvents.PageView) + { + VisitorId = visitor, Timestamp = Utc(at), Url = page, UserAgent = UserAgents.ChromeWindows, + Dimensions = { ["source"] = source, ["country"] = "DE" } + }), Is.True); + + /// Grouped by country, a query rollups answer; by page, one that scans raw events. + static AnalyticsQuery Query(QuerySource source = QuerySource.Auto, string group = "country") + { + var query = new AnalyticsQuery + { + Range = Range(Start.AddDays(-1), Start.AddDays(1)), Source = source, Filters = { Filter.Equal("source", "GitHub") }, + GroupBy = { group }, Metrics = { Metric.Events, Metric.Visitors } + }; + // Session metrics group by session-level dimensions only. + if (group != "page") + query.Metrics.AddRange([Metric.Sessions, Metric.BounceRate, Metric.AverageSessionDuration]); + return query; + } + + static void Same(AnalyticsResult result, AnalyticsResult raw) => + Assert.That(result.Rows.Select(row => string.Join('|', row.Dimensions) + "=" + string.Join('|', row.Values)), + Is.EqualTo(raw.Rows.Select(row => string.Join('|', row.Dimensions) + "=" + string.Join('|', row.Values)))); + + /// + /// Opens an engine whose background work waits at the start of a stage. Fails when opening + /// waits for that work. + /// + static AnalyticsEngine OpenWaitingAt(string directory, string stage, ManualResetEventSlim gate, Action configure) => + OpenWaitingAt(directory, stage, gate, configure, out _); + + static AnalyticsEngine OpenWaitingAt(string directory, string stage, ManualResetEventSlim gate, Action configure, + out IndexCrashRecoveryTests.RecoveryLogger log) + { + var logger = log = new IndexCrashRecoveryTests.RecoveryLogger(message => + { + if (message.StartsWith(stage + " started;", StringComparison.Ordinal)) + gate.Wait(TimeSpan.FromSeconds(30)); + }); + var options = TestEngine.Options(directory, options => { configure(options); options.Storage.Logger = logger; }); + var opening = Task.Run(() => new AnalyticsEngine(options)); + // Before: opening the engine ran the stage itself, so it waited for the gate. + Assert.That(opening.Wait(TimeSpan.FromSeconds(10)), Is.True, "Opening waited for the background work."); + return opening.Result; + } + + [Test] + public void IndexesAreBuiltInTheBackgroundWhileTheEngineIngestsAndAnswersQueries() + { + void Rollups(AnalyticsOptions options) => options.Rollups.Definitions.Add(new("source", "country")); + using (var seeded = TestEngine.Create(directory, Rollups)) + { + for (var i = 0; i < 6; ++i) + View(seeded, "v" + i, Start.AddMinutes(i * 40), "/docs", i % 2 == 0 ? "GitHub" : "Google"); + seeded.Flush(); + } + using var gate = new ManualResetEventSlim(); + var engine = OpenWaitingAt(directory, "Dimension index backfill", gate, options => + { + Rollups(options); + options.Storage.DimensionIndexes.Dimensions.Add("source"); + options.Storage.RollupIndexes.Definitions.Add(new("source", "country")); + }); + try + { + Assert.That(engine.GetStatistics().IndexMaintenance, Is.EqualTo("dimension indexes")); + // Ingestion writes its own copies while the indexes are built. + View(engine, "late", Start.AddMinutes(30), "/late", "GitHub"); + engine.Flush(); + var raw = engine.Query(Query(QuerySource.RawEvents)); + var scanned = engine.Query(Query(group: "page")); + var rollups = engine.Query(Query()); + Assert.Multiple(() => + { + Assert.That(raw.Totals[Metric.Events], Is.EqualTo(4)); + Assert.That(scanned.Plan.UsedDimensionIndexes, Is.False); + Assert.That(scanned.Plan.Notices, Is.EqualTo(new[] { DimensionIndexStore.UnbuiltNotice("source") })); + Assert.That(scanned.Plan.IsComplete, Is.True, "Notices do not make a result incomplete."); + Same(scanned, engine.Query(Query(QuerySource.RawEvents, "page"))); + Assert.That(rollups.Plan.UsedRollupIndexes, Is.False); + Assert.That(rollups.Plan.Notices.Single(), Does.Contain("rollup index on 'source' is being built")); + Same(rollups, raw); + }); + gate.Set(); + engine.WaitForIndexesAsync().WaitAsync(TimeSpan.FromSeconds(30)).GetAwaiter().GetResult(); + var indexed = engine.Query(Query(group: "page")); + var indexedRollups = engine.Query(Query()); + Assert.Multiple(() => + { + Assert.That(engine.GetStatistics().IndexMaintenance, Is.Null); + Assert.That(indexed.Plan.UsedDimensionIndexes, Is.True); + Assert.That(indexedRollups.Plan.UsedRollupIndexes, Is.True); + Assert.That(indexed.Plan.Notices, Is.Empty); + Assert.That(indexedRollups.Plan.Notices, Is.Empty); + Same(indexed, engine.Query(Query(QuerySource.RawEvents, "page"))); + Same(indexedRollups, raw); + }); + } + finally + { + gate.Set(); + engine.Dispose(); + } + } + + [Test] + public void JointHistoryIsRebuiltWhileIngestionWritesToTheSameRows() + { + using (var seeded = TestEngine.Create(directory)) + { + View(seeded, "a", Start.AddMinutes(1), "/docs", "GitHub"); + View(seeded, "a", Start.AddMinutes(2), "/pricing", "Google"); + View(seeded, "b", Start.AddMinutes(4), "/docs", "Google"); + seeded.Flush(); + } + void Indexes(AnalyticsOptions options) + { + foreach (var group in new[] { "country", "page" }) + options.Storage.RollupIndexes.Definitions.Add(new("source", group)); + } + using var gate = new ManualResetEventSlim(); + var engine = OpenWaitingAt(directory, "Joint rollup backfill", gate, Indexes); + try + { + Assert.That(engine.GetStatistics().IndexMaintenance, Is.EqualTo("joint rollup history")); + // A late event that continues session a, and a new visitor: ingestion writes them to the + // rows the reconstruction rebuilds, and the reconstruction leaves them out. + View(engine, "a", Start.AddMinutes(3), "/continue", "Google"); + View(engine, "c", Start.AddMinutes(10), "/docs", "GitHub"); + engine.Flush(); + var raw = engine.Query(Query(QuerySource.RawEvents)); + var during = engine.Query(Query()); + Assert.Multiple(() => + { + // The source of a session applies to all its events. + Assert.That(raw.Totals[Metric.Events], Is.EqualTo(4)); + Assert.That(during.Plan.UsedRollupIndexes, Is.False); + Assert.That(during.Plan.Notices, Has.Some.Contains("rollup history of [country, source] is being rebuilt")); + Same(during, raw); + }); + gate.Set(); + engine.WaitForIndexesAsync().WaitAsync(TimeSpan.FromSeconds(30)).GetAwaiter().GetResult(); + var rebuilt = engine.Query(Query()); + var pages = engine.Web.GetPages(new WebReportRequest { Range = Day(Start), Filters = { Filter.Equal("source", "GitHub") } }); + Assert.Multiple(() => + { + Assert.That(rebuilt.Plan.UsedRollupIndexes, Is.True); + Assert.That(rebuilt.Plan.UsedRawEvents, Is.False); + Assert.That(rebuilt.Plan.Notices, Is.Empty); + Same(rebuilt, raw); + Assert.That(engine.Web.GetSummary(new WebReportRequest { Range = Day(Start) }).Summary.PageViews, Is.EqualTo(5)); + Assert.That(pages.Plan.UsedRawEvents, Is.False); + Assert.That(pages.Rows.Single(row => row.Page == "/docs").Entrances, Is.EqualTo(2)); + }); + } + finally + { + gate.Set(); + engine.Dispose(); + } + using var reopened = TestEngine.Create(directory, Indexes); + Same(reopened.Query(Query()), reopened.Query(Query(QuerySource.RawEvents))); + } + + [Test] + public void ABuildStoppedByDisposalResumesWhenTheEngineOpensAgain() + { + using (var seeded = TestEngine.Create(directory)) + { + View(seeded, "a", Start, "/docs", "GitHub"); + seeded.Flush(); + } + void Indexes(AnalyticsOptions options) => options.Storage.DimensionIndexes.Dimensions.Add("source"); + using var gate = new ManualResetEventSlim(); + var engine = OpenWaitingAt(directory, "Dimension index backfill", gate, Indexes, out var stopped); + var waiting = engine.WaitForIndexesAsync(); + var disposal = Task.Run(engine.Dispose); + gate.Set(); + Assert.That(disposal.Wait(TimeSpan.FromSeconds(30)), Is.True); + Assert.That(stopped.Messages, Has.Some.StartsWith("Index maintenance shutdown completed;").And.Some.Contains("running=dimension indexes")); + Assert.That(() => waiting.GetAwaiter().GetResult(), Throws.InstanceOf()); + Assert.That(new MetaStore(Path.Combine(directory, "meta")).Get("dimension-index.clean"), Is.EqualTo("false"), + "An unfinished build does not claim coverage."); + using var reopened = TestEngine.Create(directory, Indexes); + var result = reopened.Query(Query(group: "page")); + Assert.That(result.Plan.UsedDimensionIndexes, Is.True); + Assert.That(result.Totals[Metric.Events], Is.EqualTo(1)); + } + + static void JointIndex(AnalyticsOptions options) => options.Storage.RollupIndexes.Definitions.Add(new("source", "country")); + + void SeedJointHistory() + { + using var seeded = TestEngine.Create(directory); + View(seeded, "a", Start.AddMinutes(1), "/docs", "GitHub"); + View(seeded, "a", Start.AddMinutes(2), "/pricing", "Google"); + View(seeded, "b", Start.AddMinutes(4), "/docs", "GitHub"); + seeded.Flush(); + } + + [Test] + public void AJournalLeftByACompletedReconstructionIsNotResumedWhenItsDefinitionsReturn() + { + SeedJointHistory(); + // The journal as a reconstruction leaves it when the process dies right after it completes. + var journal = Path.Combine(directory, "backfill", "joint-rollups"); + Dictionary leftover = null; + var logger = new IndexCrashRecoveryTests.RecoveryLogger(message => + { + if (leftover == null && message.StartsWith("Joint rollup backfill progress;", StringComparison.Ordinal) && message.Contains("checkpoint saved;", StringComparison.Ordinal)) + leftover = Directory.GetFiles(journal).ToDictionary(path => path, File.ReadAllBytes); + }); + using (TestEngine.Create(directory, options => { JointIndex(options); options.Storage.Logger = logger; })) { } + Assert.That(leftover, Is.Not.Empty); + // Removing the index forgets the definition; an event arrives while it is gone. + using (var removed = TestEngine.Create(directory)) + { + View(removed, "c", Start.AddMinutes(30), "/docs", "GitHub"); + removed.Flush(); + } + foreach (var (path, bytes) in leftover) + File.WriteAllBytes(path, bytes); + using var readded = TestEngine.Create(directory, JointIndex); + var result = readded.Query(Query()); + // Before: the journal resumed, kept the rows of the earlier activation and missed event c. + Assert.That(result.Plan.UsedRollupIndexes, Is.True); + Same(result, readded.Query(Query(QuerySource.RawEvents))); + Assert.That(result.Totals[Metric.Events], Is.EqualTo(4), "The source of session a applies to both its events."); + } + + [Test] + public void HistoryWaitingForItsFoldIsNotReportedAsRebuildingOnceTheIndexIsRemoved() + { + SeedJointHistory(); + using var gate = new ManualResetEventSlim(); + var engine = OpenWaitingAt(directory, "Rollup index backfill", gate, JointIndex); + // The reconstruction completes; the rollup index build that folds it in never runs. + Assert.That(SpinWait.SpinUntil(() => engine.GetStatistics(includePartitions: false).IndexMaintenance == "rollup indexes", + TimeSpan.FromSeconds(30)), Is.True); + var disposal = Task.Run(engine.Dispose); + gate.Set(); + Assert.That(disposal.Wait(TimeSpan.FromSeconds(30)), Is.True); + // The joint definition stays, without its index. + using var reopened = TestEngine.Create(directory, options => options.Rollups.Definitions.Add(new("source", "country"))); + var result = reopened.Query(Query()); + // Before: nothing folds the history in any more, but every query reported it as being rebuilt. + Assert.That(result.Plan.Notices, Is.Empty); + Same(result, reopened.Query(Query(QuerySource.RawEvents))); + } + + static void DimensionIndex(AnalyticsOptions options) => options.Storage.DimensionIndexes.Dimensions.Add("source"); + + [Test] + public void ARepairKeepsTheCopiesIngestionWritesMeanwhile() + { + using (var seeded = TestEngine.Create(directory, DimensionIndex)) + { + View(seeded, "a", Start.AddMinutes(1), "/docs", "GitHub"); + seeded.Flush(); + } + // The writes of that day were not reconciled when the previous run ended. + new MetaStore(Path.Combine(directory, "meta")).Set("dimension-index.dirty." + Start.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture), "true"); + using var gate = new ManualResetEventSlim(); + var engine = OpenWaitingAt(directory, "Dimension index backfill", gate, DimensionIndex); + try + { + // Ingestion copies a new event of the day while the repair waits. + View(engine, "b", Start.AddHours(1), "/docs", "GitHub"); + engine.Flush(); + gate.Set(); + engine.WaitForIndexesAsync().WaitAsync(TimeSpan.FromSeconds(30)).GetAwaiter().GetResult(); + var result = engine.Query(Query(group: "page")); + // Before: the repair deleted the new copy and copied the events stored before the engine opened only. + Assert.That(result.Plan.UsedDimensionIndexes, Is.True); + Assert.That(result.Totals[Metric.Events], Is.EqualTo(2)); + Same(result, engine.Query(Query(QuerySource.RawEvents, "page"))); + } + finally + { + gate.Set(); + engine.Dispose(); + } + } + + [Test] + public void HistoryWaitingForItsFoldIsFoldedInWhenItsIndexIsRemovedAndItsRollupsRemain() + { + SeedJointHistory(); + using var gate = new ManualResetEventSlim(); + var engine = OpenWaitingAt(directory, "Rollup index backfill", gate, options => + { + JointIndex(options); + options.Storage.RollupIndexes.Definitions.Add(new("source", "page")); + }); + // Both histories are rebuilt; the rollup index build that folds them in stops before it starts. + Assert.That(SpinWait.SpinUntil(() => engine.GetStatistics(includePartitions: false).IndexMaintenance == "rollup indexes", + TimeSpan.FromSeconds(30)), Is.True); + var disposal = Task.Run(engine.Dispose); + gate.Set(); + Assert.That(disposal.Wait(TimeSpan.FromSeconds(30)), Is.True); + // One index goes; its joint rollups stay. + using var reopened = TestEngine.Create(directory, options => + { + JointIndex(options); + options.Rollups.Definitions.Add(new("source", "page")); + }); + var result = reopened.Query(Query(group: "page")); + // Before: its history was published without being folded in, so the rollups read nothing. + Assert.That(result.Plan.UsedRollups, Is.True); + Assert.That(result.Plan.UsedRawEvents, Is.False); + Same(result, reopened.Query(Query(QuerySource.RawEvents, "page"))); + } + + [Test] + public void ABuildWhoseCopyFailedIsNotRecordedAsComplete() + { + using (var seeded = TestEngine.Create(directory)) + { + for (var i = 0; i < 3; ++i) + View(seeded, "v" + i, Start.AddMinutes(i), "/docs", "GitHub"); + seeded.Flush(); + } + var opens = 0; + // The dimension index build's first copy fails; a joint rollup index is built after it. + using (var engine = new AnalyticsEngine(TestEngine.Options(directory, options => + { + DimensionIndex(options); + JointIndex(options); + options.Storage.DimensionIndexes.Store.ConfigureMaintainer = _ => + { + if (++opens == 1) throw new IOException("Injected dimension index copy failure"); + }; + }))) + { + // Before: the retry found the indexes invalid, returned, and readiness reported success. + var failure = Assert.ThrowsAsync(() => engine.WaitForIndexesAsync().WaitAsync(TimeSpan.FromSeconds(30))); + var rollups = engine.Query(Query()); + Assert.Multiple(() => + { + Assert.That(failure.InnerException, Is.TypeOf()); + Assert.That(engine.GetStatistics().IndexMaintenance, Is.Null); + Assert.That(engine.Query(Query(group: "page")).Plan.UsedDimensionIndexes, Is.False); + // The jobs after the failed one still ran. + Assert.That(rollups.Plan.UsedRollupIndexes, Is.True); + Same(rollups, engine.Query(Query(QuerySource.RawEvents))); + }); + } + // Before: the retry skipped every copy and recorded the indexes as clean. + Assert.That(new MetaStore(Path.Combine(directory, "meta")).Get("dimension-index.clean"), Is.EqualTo("false")); + using var reopened = TestEngine.Create(directory, DimensionIndex); + var result = reopened.Query(Query(group: "page")); + Assert.That(result.Plan.UsedDimensionIndexes, Is.True); + Assert.That(result.Totals[Metric.Events], Is.EqualTo(3)); + } + + [Test] + public void QueriesSeeTheRawEventsRetentionHolds() + { + var options = TestEngine.Options(directory, options => options.Retention.RawEvents = TimeSpan.FromDays(2)); + var clock = new RetentionClock(options.Retention, new ReportingCalendar(options.TimeZone, options.FirstDayOfWeek)); + var now = DateTime.UtcNow.Ticks; + var held = DateTime.UtcNow.Date.AddDays(-10).Ticks; + var cutoff = clock.At(now).RawEvents; + clock.HoldRawEvents(held); + Assert.Multiple(() => + { + // Before: queries computed the cutoff from the options, so held events were reported as outside retention. + Assert.That(clock.At(now).RawEvents, Is.EqualTo(held)); + Assert.That(clock.Current.RawEvents, Is.EqualTo(held)); + Assert.That(clock.At(now).RawEventsWarning(held), Is.Null); + }); + clock.ReleaseRawEvents(); + Assert.That(clock.At(now).RawEvents, Is.EqualTo(cutoff)); + } +} diff --git a/src/ZoneTree.Analytics.UnitTests/IndexCrashRecoveryTests.cs b/src/ZoneTree.Analytics.UnitTests/IndexCrashRecoveryTests.cs index 302de01..45d84d1 100644 --- a/src/ZoneTree.Analytics.UnitTests/IndexCrashRecoveryTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/IndexCrashRecoveryTests.cs @@ -14,6 +14,12 @@ public sealed class IndexCrashRecoveryTests : DirectoryTest internal static void Configure(AnalyticsOptions options, string scenario, bool indexed) { + // A killed process loses the most recent writes of the default asynchronous write-ahead + // log, which the engine accepts. These tests check what the checkpoints resume, so every + // write reaches its log before it returns. + foreach (var store in new[] { options.Storage.RawEvents, options.Storage.Rollups, options.Storage.Sessions, + options.Storage.DimensionIndexes.Store, options.Storage.RollupIndexes.Store }) + store.WriteAheadLogMode = ZoneTree.Options.WriteAheadLogMode.Sync; options.Ingestion.WorkerCount = 1; options.Queries.MaxDegreeOfParallelism = 1; options.Sessions.Enabled = scenario.StartsWith("joint", StringComparison.Ordinal); @@ -309,7 +315,7 @@ void Block() { if (stage.Length > 0 && message.StartsWith(stage, StringComparison.Ordinal) && (!(scenario is "joint-final" or "joint-expire") || !message.Contains("sessions=0;", StringComparison.Ordinal)) && - message.Contains(scenario == "joint-redo" ? "checkpoint prepared;" : "checkpoint saved;", StringComparison.Ordinal)) Block(); + message.Contains(scenario switch { "joint-redo" => "checkpoint prepared;", "repair" => "partition repaired;", _ => "checkpoint saved;" }, StringComparison.Ordinal)) Block(); }); using var engine = TestEngine.Create(directory, options => { @@ -331,8 +337,9 @@ void Block() Assert.That(iterator.Next(), Is.True); var value = iterator.CurrentValue; lease.ZoneTree.ForceDelete(iterator.CurrentKey); + // The orphan of an event stored earlier, whose raw record an interrupted deletion removed. lease.ZoneTree.Upsert(DimensionIndexKeyCodec.Key(index.KeyPrefix.Span, "default", "GitHub", - IndexCrashRecoveryTests.Start.AddDays(10).AddMinutes(3).Ticks, long.MaxValue), value); + IndexCrashRecoveryTests.Start.AddDays(10).AddMinutes(3).Ticks, 1), value); lease.Maintainer.EvictToDisk(); lease.Maintainer.WaitForBackgroundThreads(); } diff --git a/src/ZoneTree.Analytics.UnitTests/ReportingSerializationTests.cs b/src/ZoneTree.Analytics.UnitTests/ReportingSerializationTests.cs index f38b187..159018d 100644 --- a/src/ZoneTree.Analytics.UnitTests/ReportingSerializationTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/ReportingSerializationTests.cs @@ -37,7 +37,7 @@ public void SummaryContractKeepsNullMetricsOmitsMissingComparisonAndUsesSeconds( "visitors": 2, "pageViews": 3, "sessions": null, "bounceRate": null, "averageSessionDuration": 1.25, "pageViewsPerSession": 1.5, "botHits": null }, - "plan": { "complete": true, "warnings": [], "steps": [], "rollupSuggestions": [] } + "plan": { "complete": true, "warnings": [], "notices": [], "steps": [], "rollupSuggestions": [] } } """); } @@ -119,7 +119,7 @@ public void EmptyListContractKeepsTotalRowsAndEmptyArrays() AssertJson(Serialize(new ReportingResponseMapper().Pages(report), ReportingJsonContext.Default.PagesResponse), """ { "from": "2025-01-01T00:00:00+00:00", "to": "2025-01-02T00:00:00+00:00", "totalRows": 42, "rows": [], - "plan": { "complete": true, "warnings": [], "steps": [], "rollupSuggestions": [] } } + "plan": { "complete": true, "warnings": [], "notices": [], "steps": [], "rollupSuggestions": [] } } """); } diff --git a/src/ZoneTree.Analytics.UnitTests/RollupValueLayoutTests.cs b/src/ZoneTree.Analytics.UnitTests/RollupValueLayoutTests.cs index 4766d33..eb45d36 100644 --- a/src/ZoneTree.Analytics.UnitTests/RollupValueLayoutTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/RollupValueLayoutTests.cs @@ -270,7 +270,7 @@ public void RollupReaderRejectsWhatItDoesNotUnderstand() { var bytes = Serializer.Serialize(HeavyRow(10)).ToArray(); var unknownBit = bytes.ToArray(); - unknownBit[0] |= 0x10; + unknownBit[0] |= 0x20; var notPresent = bytes.ToArray(); notPresent[0] &= 0xFE; Assert.Multiple(() => @@ -315,6 +315,45 @@ public void ASparseDeltaWithOverflowingBitsIsRejected() Assert.That(Serializer.Deserialize(largest).Visitors.Estimate(), Is.EqualTo(1)); } + [Test] + public void ARebuiltPartStaysApartFromFlushesUntilItIsFoldedIn() + { + var live = HeavyRow(10); + var rebuilt = HeavyRow(30); + var stored = Serializer.Deserialize(Serializer.Serialize(RollupValue.WithRebuilt(live, rebuilt))); + // A flush merges its delta into the row and keeps the rebuilt part as it is. + var flushed = Serializer.Deserialize(Serializer.Serialize(RollupValue.Merge(stored, HeavyRow(5)))); + var folded = flushed.Folded(); + var onlyRebuilt = Serializer.Deserialize(Serializer.Serialize(RollupValue.WithRebuilt(null, rebuilt))); + Assert.Multiple(() => + { + Assert.That((stored.Count, stored.Rebuilt.Count), Is.EqualTo((10L, 30L))); + Assert.That((flushed.Count, flushed.Rebuilt.Count), Is.EqualTo((15L, 30L))); + Assert.That(flushed.Rebuilt.Visitors.Estimate(), Is.EqualTo(rebuilt.Visitors.Estimate())); + Assert.That((folded.Count, folded.Rebuilt), Is.EqualTo((45L, (RollupValue)null))); + // Folding adds the rebuilt part as if a flush had merged it. + Assert.That(Serializer.Serialize(folded).ToArray(), Is.EqualTo(Serializer.Serialize(RollupValue.Merge(RollupValue.Merge(live, HeavyRow(5)), rebuilt)).ToArray())); + Assert.That(stored.IsRebuiltOnly, Is.False); + Assert.That(onlyRebuilt.IsRebuiltOnly, Is.True); + Assert.That(onlyRebuilt.Folded().Count, Is.EqualTo(30)); + Assert.That(folded.Folded(), Is.SameAs(folded), "a row without a rebuilt part folds to itself"); + }); + } + + [Test] + public void ARebuiltPartCannotHaveARebuiltPartOfItsOwn() + { + const byte live = 1, rebuiltBit = 16; + byte[] empty = [live, 0, 0]; + byte[] nested = [live | rebuiltBit, 0, 0, .. Section(empty)]; + Assert.Multiple(() => + { + Assert.That(Serializer.Deserialize(nested.ToArray()).Rebuilt.Count, Is.Zero); + Assert.Throws(() => Serializer.Deserialize((byte[])[live | rebuiltBit, 0, 0, .. Section(nested)]), "a nested rebuilt part"); + Assert.Throws(() => Serializer.Deserialize((byte[])[live | rebuiltBit, 0, 0, .. Section(0)]), "a deleted rebuilt part"); + }); + } + [Test] public void RawAndSessionReadersRejectWhatTheyDoNotUnderstand() { diff --git a/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs b/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs index 4ca15fd..bf92e2d 100644 --- a/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs @@ -213,12 +213,17 @@ public void SessionStoreAcceptsAStateWithoutDimensions() }); } + JointBackfillCheckpoint Journal(string configuration, long coverage) => + new(directory, configuration, () => new() { Coverage = coverage }); + [Test] public void JointBackfillJournalRoundTripsEveryField() { - var checkpoint = new JointBackfillCheckpoint(directory, "configuration", 5); + var checkpoint = Journal("configuration", 5); var state = checkpoint.State; state.Copying = true; + state.Fence = 42; + state.Partitions = ["2026-10-14", "2026-10-15"]; state.Site = "shop"; state.Partition = "2026-10-15"; state.Key = [1, 2, 3]; @@ -238,7 +243,7 @@ public void JointBackfillJournalRoundTripsEveryField() state.Redo.Add(new JointBackfillReplacement { Granularity = Granularity.Day, Partition = "2026-10-14", Key = [4], Value = [5, 6] }); checkpoint.Save(); - var reopened = new JointBackfillCheckpoint(directory, "configuration", 99); + var reopened = Journal("configuration", 99); var loaded = reopened.State; var session = loaded.Active[9]; Assert.Multiple(() => @@ -246,6 +251,7 @@ public void JointBackfillJournalRoundTripsEveryField() Assert.That(reopened.Resuming, Is.True); Assert.That((loaded.Copying, loaded.Coverage, loaded.Site, loaded.SiteCompleted, loaded.Partition), Is.EqualTo((true, 5L, "shop", false, "2026-10-15"))); Assert.That(loaded.Key, Is.EqualTo(new byte[] { 1, 2, 3 })); + Assert.That((loaded.Fence, loaded.Partitions), Is.EqualTo((42L, new[] { "2026-10-14", "2026-10-15" }))); Assert.That((loaded.Events, loaded.Sessions, loaded.Updates, loaded.Flushes, loaded.Sites), Is.EqualTo((10L, 11L, 12L, 13L, 2))); Assert.That((session.Start, session.Last, session.PageViews, session.Interactions, session.Visitor, session.Allowed), Is.EqualTo((100L, 200L, 3, 4, 77UL, true))); Assert.That(session.Dimensions, Is.EqualTo(state.Active[9].Dimensions)); @@ -256,10 +262,10 @@ public void JointBackfillJournalRoundTripsEveryField() Assert.That(loaded.Redo.Single().Partition, Is.EqualTo("2026-10-14")); Assert.That(loaded.Redo.Single().Key, Is.EqualTo(new byte[] { 4 })); Assert.That(loaded.Redo.Single().Value, Is.EqualTo(new byte[] { 5, 6 })); - Assert.That(new JointBackfillCheckpoint(directory, "other", 5).Resuming, Is.False); + Assert.That(Journal("other", 5).Resuming, Is.False); }); reopened.Complete(); - Assert.That(new JointBackfillCheckpoint(directory, "configuration", 5).Resuming, Is.False); + Assert.That(Journal("configuration", 5).Resuming, Is.False); } [Test] @@ -270,13 +276,13 @@ public void JointBackfillJournalHoldsAFlushOfDenseRows() const int rows = 7_500; var value = new byte[16_386]; new Random(1).NextBytes(value); - var checkpoint = new JointBackfillCheckpoint(directory, "configuration", 0); + var checkpoint = Journal("configuration", 0); for (var i = 0; i < rows; ++i) checkpoint.State.Redo.Add(new JointBackfillReplacement { Granularity = Granularity.Hour, Partition = "2026-10-15", Key = BitConverter.GetBytes(i), Value = value }); checkpoint.Save(); var journal = Directory.GetFiles(Path.Combine(directory, "backfill", "joint-rollups")).Sum(path => new FileInfo(path).Length); Assert.That(journal, Is.LessThan(rows * (value.Length + 64L)), "the journal stores bytes, not text"); - var redo = new JointBackfillCheckpoint(directory, "configuration", 0).State.Redo; + var redo = Journal("configuration", 0).State.Redo; Assert.That(redo, Has.Count.EqualTo(rows)); Assert.That(redo[^1].Key, Is.EqualTo(BitConverter.GetBytes(rows - 1))); Assert.That(redo[^1].Value, Is.EqualTo(value)); @@ -295,14 +301,14 @@ public void AJointBackfillJournalOfAnOlderReleaseIsRefused() var journal = Path.Combine(directory, "backfill", "joint-rollups"); Directory.CreateDirectory(journal); File.WriteAllBytes(Path.Combine(journal, "journal-b.bin"), [.. header, .. body]); - var error = Assert.Throws(() => _ = new JointBackfillCheckpoint(directory, "configuration", 0)); + var error = Assert.Throws(() => _ = Journal("configuration", 0)); Assert.That(error.Message, Does.Contain("older release")); } [Test] public void ATornJointBackfillJournalFallsBackToThePreviousCheckpoint() { - var checkpoint = new JointBackfillCheckpoint(directory, "configuration", 0); + var checkpoint = Journal("configuration", 0); checkpoint.State.Events = 1; checkpoint.Save(); checkpoint.State.Events = 2; @@ -313,6 +319,6 @@ public void ATornJointBackfillJournalFallsBackToThePreviousCheckpoint() .OrderByDescending(path => BitConverter.ToInt64(File.ReadAllBytes(path), 8)).First(); var bytes = File.ReadAllBytes(newest); File.WriteAllBytes(newest, bytes[..(bytes.Length - 1)]); - Assert.That(new JointBackfillCheckpoint(directory, "configuration", 0).State.Events, Is.EqualTo(1)); + Assert.That(Journal("configuration", 0).State.Events, Is.EqualTo(1)); } } diff --git a/src/ZoneTree.Analytics.UnitTests/TestEngine.cs b/src/ZoneTree.Analytics.UnitTests/TestEngine.cs index 7d4f367..0770811 100644 --- a/src/ZoneTree.Analytics.UnitTests/TestEngine.cs +++ b/src/ZoneTree.Analytics.UnitTests/TestEngine.cs @@ -74,8 +74,24 @@ public static long ApplyRetentionCountingDrops(this AnalyticsEngine engine) return engine.GetStatistics().PartitionsDropped - before; } - public static AnalyticsEngine Create(string directory, Action configure = null) => - new(Options(directory, configure)); + /// + /// Opens an engine and waits until its background index work is done, so tests see the same + /// plans as a host that waits for readiness. + /// + public static AnalyticsEngine Create(string directory, Action configure = null) + { + var engine = new AnalyticsEngine(Options(directory, configure)); + try + { + engine.WaitForIndexesAsync().WaitAsync(TimeSpan.FromMinutes(2)).GetAwaiter().GetResult(); + } + catch + { + engine.Dispose(); + throw; + } + return engine; + } public static void Delete(string directory) { diff --git a/src/ZoneTree.Analytics/AnalyticsEngine.cs b/src/ZoneTree.Analytics/AnalyticsEngine.cs index 8d03e56..b4f93ce 100644 --- a/src/ZoneTree.Analytics/AnalyticsEngine.cs +++ b/src/ZoneTree.Analytics/AnalyticsEngine.cs @@ -60,6 +60,8 @@ public sealed class AnalyticsEngine : IAnalyticsTracker, IDisposable, IAsyncDisp readonly RollupIndexStore rollupIndexes; + readonly IndexMaintenance maintenance; + bool started; readonly VisitorHasher visitorHasher; @@ -108,8 +110,15 @@ public sealed class AnalyticsEngine : IAnalyticsTracker, IDisposable, IAsyncDisp /// Opens or creates the analytics data directory and starts the ingestion workers. /// /// + /// /// The engine works on a copy of : changes made to them afterwards /// have no effect. Extensions, hooks and the logger are shared, not copied. + /// + /// + /// Index builds and repairs, and the history of newly indexed rollups, run in the background + /// once the engine is open; see . Opening does the quick part: + /// it decides that work and removes data it replaces. + /// /// /// The options are invalid. /// The options conflict with the storage format of the data directory, or its metadata is damaged. @@ -149,8 +158,9 @@ public AnalyticsEngine(AnalyticsOptions options) if (options.Sessions.Enabled) { var sessionResources = new DisposalGroup(); - ownedStores.Add(sessionResources); - sessions = factory.OpenSessions(Path.Combine(root, "sessions"), enricher.Schema); + var sessionsDirectory = Path.Combine(root, "sessions"); + ownedStores.Add(new StageDisposal(sessionResources, storageLogger, "Session store shutdown", sessionsDirectory)); + sessions = factory.OpenSessions(sessionsDirectory, enricher.Schema); sessionResources.Add(sessions); sessionsMaintainer = factory.CreateSessionsMaintainer(sessions); sessionResources.Add(sessionsMaintainer); @@ -166,15 +176,24 @@ public AnalyticsEngine(AnalyticsOptions options) ownedStores.Add(rawEvents); } + // Opening decides the index work; it runs in the background once ingestion has started. dimensionIndexes = new DimensionIndexStore(Path.Combine(root, "indexes"), options.Storage.DimensionIndexes, rawEvents, factory, meta, storageLogger); ownedStores.Add(dimensionIndexes); - dimensionIndexes.Initialize(); + // Every event stored so far has a sequence up to the current one: background work reads + // those, and ingestion writes the later ones to the indexes and rollups itself. + var fence = Interlocked.Read(ref sequence); + dimensionIndexes.Initialize(fence); if (rawEvents != null) rawEvents.Indexes = dimensionIndexes; + var reconstruction = JointRollupReconstruction.Prepare(options, rollups, rawEvents, activations, calendar, + enricher.Schema, meta, storageLogger, fence); + if (reconstruction != null) + retentionClock.HoldRawEvents(reconstruction.Coverage); + rollupIndexes = new RollupIndexStore(Path.Combine(root, "rollup-indexes"), options, rollups, factory, meta, storageLogger); ownedStores.Add(rollupIndexes); - rollupIndexes.Initialize(rawEvents, activations, calendar, enricher.Schema); + rollupIndexes.Initialize(activations); rollups.Indexes = rollupIndexes; // Registered after the stores it writes to, so it stops before they close. var rollupWriters = new RollupWriterPool(rollups, options.Ingestion.RollupWriterCount, logger); @@ -201,8 +220,8 @@ public AnalyticsEngine(AnalyticsOptions options) workers[i] = new IngestionWorker(services); queryEngine = new QueryEngine( - options, calendar, rollups, rawEvents, definitions, activations, enricher); - funnelEngine = new FunnelEngine(options, calendar, rawEvents, enricher); + options, calendar, rollups, rawEvents, definitions, activations, enricher, retentionClock); + funnelEngine = new FunnelEngine(options, calendar, rawEvents, enricher, retentionClock); Web = new WebAnalytics(this); Storage = new AnalyticsStorage(rawEvents?.Store, rollups, sessions, sessionsMaintainer, () => ObjectDisposedException.ThrowIf(IsDisposed, this), dimensionIndexes, rollupIndexes); @@ -212,6 +231,10 @@ public AnalyticsEngine(AnalyticsOptions options) timer = new Timer(OnTimer, null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); foreach (var worker in workers) worker.Start(); + // Registered after the stores it writes to, so it stops before they close. + maintenance = new IndexMaintenance([.. MaintenanceJobs(reconstruction)], storageLogger, root); + ownedStores.Add(maintenance); + maintenance.Start(); lastHousekeepingTicks = DateTime.UtcNow.Ticks; var period = options.Ingestion.FlushInterval; timer.Change(period, period); @@ -248,6 +271,42 @@ public static AnalyticsEngine Open(string dataDirectory, Action Run)> MaintenanceJobs(JointRollupReconstruction reconstruction) + { + if (dimensionIndexes.IsBuilding) + yield return ("dimension indexes", dimensionIndexes.Build); + if (reconstruction != null) + yield return ("joint rollup history", token => + { + reconstruction.Run(token); + retentionClock.ReleaseRawEvents(); + }); + if (rollupIndexes.IsBuilding) + yield return ("rollup indexes", rollupIndexes.Build); + } + + /// + /// Waits until the indexes are built or repaired and the rebuilt history of new indexed + /// rollups is available. + /// + /// + /// The engine does this work in the background after it opens, while it ingests events and + /// answers queries: until then queries read raw events or primary rollups instead, with the + /// same results, and say so in . Call this before measuring + /// query performance, or to report readiness. It completes at once when there is no such work. + /// + /// An index could not be built; queries keep answering without it until the engine opens again, which resumes the build. + /// The engine was disposed first, or was canceled. + public Task WaitForIndexesAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(IsDisposed, this); + return maintenance.Completion.WaitAsync(cancellationToken); + } + + /// + public void WaitForIndexes(CancellationToken cancellationToken = default) => + WaitForIndexesAsync(cancellationToken).GetAwaiter().GetResult(); + internal AnalyticsOptions Options => options; internal string NormalizeSite(string site) => enricher.NormalizeSite(site); @@ -715,8 +774,12 @@ public async Task EraseVisitorAsync(string visitorId, string site = null, long EraseFlushedVisitor(ulong hash, string site, CancellationToken cancellationToken) { using var suspension = dimensionIndexes.Suspend(); - try { return EraseFlushedVisitorCore(hash, site, cancellationToken); } - catch { dimensionIndexes.Invalidate(); throw; } + // A background index build waits, and reads the raw events again after the erasure. + lock (dimensionIndexes.Gate) + { + try { return EraseFlushedVisitorCore(hash, site, cancellationToken); } + catch { dimensionIndexes.Invalidate(); throw; } + } } long EraseFlushedVisitorCore(ulong hash, string site, CancellationToken cancellationToken) @@ -896,6 +959,7 @@ public AnalyticsStatistics GetStatistics(bool includePartitions) PartitionsDropped = Interlocked.Read(ref counters.PartitionsDropped), QueueLength = workers.Sum(x => (long)x.QueueLength), OpenPartitions = openPartitions, + IndexMaintenance = maintenance.Current, Partitions = includePartitions ? DescribePartitions() : [] }; } @@ -1013,10 +1077,18 @@ async Task DisposeCoreAsync(bool stopWorkers) DisposeStorage(); return; } + // Logs what shutdown waits for, and how long: queued events and the final flush, the + // running index job, and the background merges of every store. + using var shutdown = new StageLog(logger, "Shutdown", dataRoot, + $"queued events={workers.Sum(x => (long)x.QueueLength)}; index maintenance={maintenance.Current ?? "none"}"); timer?.Dispose(); try { + var processed = Interlocked.Read(ref counters.EventsProcessed); + var flushes = Interlocked.Read(ref counters.Flushes); + using var drain = new StageLog(logger, "Ingestion shutdown", dataRoot); await Task.WhenAll(workers.Select(x => x.DisposeAsync().AsTask())).ConfigureAwait(false); + drain.Complete($"events processed={Interlocked.Read(ref counters.EventsProcessed) - processed}; flushes={Interlocked.Read(ref counters.Flushes) - flushes}"); } finally { @@ -1032,6 +1104,7 @@ async Task DisposeCoreAsync(bool stopWorkers) } } } + shutdown.Complete(null); } void DisposeStorage() diff --git a/src/ZoneTree.Analytics/AnalyticsExceptions.cs b/src/ZoneTree.Analytics/AnalyticsExceptions.cs index 77fd962..1f34d6d 100644 --- a/src/ZoneTree.Analytics/AnalyticsExceptions.cs +++ b/src/ZoneTree.Analytics/AnalyticsExceptions.cs @@ -45,13 +45,20 @@ public AnalyticsConfigurationException(string message, Exception innerException) } /// -/// Raised by a flush when its rollup rows could not be written to storage. The rows are retained -/// and retried by later flushes. is the storage failure. +/// Raised when data could not be written to storage. is +/// the storage failure, when one caused it. /// /// -/// Rollup writer threads write the rows of every worker that flushed together, so one storage -/// failure can fail several flushes at once. Each of them raises an exception of its own, with -/// that failure as its inner exception. +/// +/// A flush raises it when its rollup rows could not be written; the rows are retained and retried +/// by later flushes. Rollup writer threads write the rows of every worker that flushed together, +/// so one storage failure can fail several flushes at once. Each of them raises an exception of +/// its own, with that failure as its inner exception. +/// +/// +/// raises it when an index could not be built. +/// Queries keep answering without that index until the engine opens again, which resumes the build. +/// /// public sealed class AnalyticsStorageException : AnalyticsException { diff --git a/src/ZoneTree.Analytics/AnalyticsStatistics.cs b/src/ZoneTree.Analytics/AnalyticsStatistics.cs index 29094d1..fd7b208 100644 --- a/src/ZoneTree.Analytics/AnalyticsStatistics.cs +++ b/src/ZoneTree.Analytics/AnalyticsStatistics.cs @@ -44,6 +44,13 @@ public sealed class AnalyticsStatistics /// Open ZoneTree partitions. public int OpenPartitions { get; init; } + /// + /// The index work running in the background since the engine opened: "dimension indexes", + /// "joint rollup history" or "rollup indexes". Null when no job is running, including after + /// a failed build; reports whether the work succeeded. + /// + public string IndexMaintenance { get; init; } + /// /// Completed partition deletions reported by retention. Failed or still-leased drops are /// counted by a later retention check after cleanup succeeds. diff --git a/src/ZoneTree.Analytics/Query/AnalyticsResult.cs b/src/ZoneTree.Analytics/Query/AnalyticsResult.cs index 8c8f584..ef877a4 100644 --- a/src/ZoneTree.Analytics/Query/AnalyticsResult.cs +++ b/src/ZoneTree.Analytics/Query/AnalyticsResult.cs @@ -239,11 +239,12 @@ public sealed record RollupSuggestion(string Event, RollupDefinition Definition) public sealed class QueryPlan { internal QueryPlan(IReadOnlyList steps, IReadOnlyList warnings, - IReadOnlyList rollupSuggestions = null) + IReadOnlyList rollupSuggestions = null, IReadOnlyList notices = null) { Steps = steps; Warnings = warnings; RollupSuggestions = rollupSuggestions ?? Array.Empty(); + Notices = notices ?? Array.Empty(); } /// The plan steps. @@ -258,6 +259,13 @@ internal QueryPlan(IReadOnlyList steps, IReadOnlyList war /// public IReadOnlyList Warnings { get; } + /// + /// Background work that made the query read slower sources without changing its result, for + /// example an index that is still being built after the engine opened. Notices do not affect + /// ; waits for that work. + /// + public IReadOnlyList Notices { get; } + /// True when the result covers the whole range exactly. public bool IsComplete => Warnings.Count == 0; diff --git a/src/ZoneTree.Analytics/Query/FunnelEngine.cs b/src/ZoneTree.Analytics/Query/FunnelEngine.cs index 0a8b997..14e3896 100644 --- a/src/ZoneTree.Analytics/Query/FunnelEngine.cs +++ b/src/ZoneTree.Analytics/Query/FunnelEngine.cs @@ -17,13 +17,17 @@ sealed class FunnelEngine readonly EnrichmentPipeline enricher; + readonly RetentionClock retention; + public FunnelEngine( - AnalyticsOptions options, ReportingCalendar storageCalendar, RawEventStore rawEvents, EnrichmentPipeline enricher) + AnalyticsOptions options, ReportingCalendar storageCalendar, RawEventStore rawEvents, EnrichmentPipeline enricher, + RetentionClock retention) { this.options = options; this.storageCalendar = storageCalendar; this.rawEvents = rawEvents; this.enricher = enricher; + this.retention = retention; } /// @@ -66,7 +70,7 @@ public FunnelResult Execute(FunnelQuery funnel, CancellationToken cancellationTo var site = enricher.NormalizeSite(funnel.Site); var filters = funnel.Filters.Where(x => x != null).ToArray(); var warnings = new List(); - if (RetentionCutoffs.Compute(options.Retention, storageCalendar, now).RawEventsWarning(from) is { } warning) + if (retention.At(now).RawEventsWarning(from) is { } warning) warnings.Add(warning); var segments = funnel.Segments.Where(x => x != null).ToArray(); @@ -77,12 +81,13 @@ public FunnelResult Execute(FunnelQuery funnel, CancellationToken cancellationTo var progress = Scan(site, from, to, steps, filters, segmentSets, calendar, funnel.Window, cancellationToken, out var index); var results = Summarize(steps, progress); + var unbuilt = index == null ? rawEvents.Indexes?.Unbuilt(filters, _ => true) : null; var plan = new QueryPlan( steps.Select(x => x.Event).Distinct().Select(x => new QueryPlanStep( x, index == null ? PlanSource.RawEvents : PlanSource.DimensionIndex, new DateTimeOffset(from, TimeSpan.Zero), new DateTimeOffset(to, TimeSpan.Zero)) { IndexDimension = index?.Index.Dimension }).ToList(), - warnings); + warnings, notices: unbuilt == null ? null : [DimensionIndexStore.UnbuiltNotice(unbuilt)]); return new FunnelResult( results, TimeZoneInfo.ConvertTime(new DateTimeOffset(from, TimeSpan.Zero), calendar.TimeZone), diff --git a/src/ZoneTree.Analytics/Query/QueryContext.cs b/src/ZoneTree.Analytics/Query/QueryContext.cs index 6a6652d..a9cef05 100644 --- a/src/ZoneTree.Analytics/Query/QueryContext.cs +++ b/src/ZoneTree.Analytics/Query/QueryContext.cs @@ -42,6 +42,8 @@ sealed class QueryContext public readonly HashSet Warnings = new(StringComparer.Ordinal); + public readonly HashSet Notices = new(StringComparer.Ordinal); + public readonly List RollupSuggestions = []; public SegmentSets SegmentSets; diff --git a/src/ZoneTree.Analytics/Query/QueryEngine.cs b/src/ZoneTree.Analytics/Query/QueryEngine.cs index 3d17301..d5ae73a 100644 --- a/src/ZoneTree.Analytics/Query/QueryEngine.cs +++ b/src/ZoneTree.Analytics/Query/QueryEngine.cs @@ -1,3 +1,4 @@ +using System.Globalization; using ZoneTree.Analytics.Ingestion; using ZoneTree.Analytics.Internal; using ZoneTree.Analytics.Sketches; @@ -47,6 +48,8 @@ sealed class QueryEngine readonly EnrichmentPipeline enricher; + readonly RetentionClock retention; + readonly RawSessionReader sessionReader; readonly QueryPreparation preparation; @@ -58,7 +61,8 @@ public QueryEngine( RawEventStore rawEvents, RollupDefinition[] definitions, RollupActivations activations, - EnrichmentPipeline enricher) + EnrichmentPipeline enricher, + RetentionClock retention) { this.options = options; this.storageCalendar = storageCalendar; @@ -66,8 +70,9 @@ public QueryEngine( this.rawEvents = rawEvents; this.activations = activations; this.enricher = enricher; + this.retention = retention; sessionReader = new(rawEvents, enricher.Schema, options); - preparation = new(options, storageCalendar, definitions, activations, enricher); + preparation = new(options, storageCalendar, definitions, activations, enricher, retention); } #region Entry points @@ -151,6 +156,14 @@ void PlanReads(QueryContext context, SourceContext source, QuerySource selection context.CancellationToken, context.Interval); source.Plan = plan.Segments; SuggestRollup(context, source, selection); + if (source.Definition != null && selection != QuerySource.RawEvents && activations.IsRebuilding(source.Definition.Id) && + plan.Segments.Any(x => x.Source == PlanSource.RawEvents)) + { + var since = new DateTime(activations.DefinitionSince(source.Definition.Id), DateTimeKind.Utc); + context.Notices.Add( + $"The rollup history of [{string.Join(", ", source.Definition.Dimensions)}] is being rebuilt in the background; " + + $"ranges of '{source.Event}' before {since.ToString("yyyy-MM-dd HH:mm", CultureInfo.InvariantCulture)} UTC were read from raw events."); + } foreach (var segment in plan.Segments.Where(x => x.Source == PlanSource.RawEvents)) AddWarning(context, context.Cutoffs.RawEventsWarning(segment.From)); foreach (var granularity in CoarseToFine) @@ -291,9 +304,13 @@ void Visit(long bucket, string combined, RollupValue value) var indexDimension = rollups.Indexes?.Scan(granularity, context.Site, source.Event, definition.Id, fromBucket, toBucket, source.Filters, Visit, context.CancellationToken); if (indexDimension != null) + { context.Steps[^1] = context.Steps[^1] with { IndexDimension = indexDimension }; - else - rollups.Scan(granularity, context.Site, source.Event, definition.Id, fromBucket, toBucket, Visit); + return; + } + if (rollups.Indexes?.Unbuilt(granularity, definition.Id, source.Filters) is { } unbuilt) + context.Notices.Add($"The rollup index on '{unbuilt}' is being built in the background; primary rollups were read instead."); + rollups.Scan(granularity, context.Site, source.Event, definition.Id, fromBucket, toBucket, Visit); } /// @@ -360,10 +377,12 @@ void ReadRawEvents(QueryContext context, ReadSegment segment, List allowed = dimension => + !trackSessions || enricher.Schema.IsSessionLevel(dimension) || dimension == StandardDimensions.EntryPage; var index = context.UseDimensionIndexes ? rawEvents.Indexes?.Select( - context.Site, segment.From, segment.To, context.Filters, - dimension => !trackSessions || enricher.Schema.IsSessionLevel(dimension) || dimension == StandardDimensions.EntryPage, - context.CancellationToken) : null; + context.Site, segment.From, segment.To, context.Filters, allowed, context.CancellationToken) : null; + if (index == null && context.UseDimensionIndexes && rawEvents.Indexes?.Unbuilt(context.Filters, allowed) is { } unbuilt) + context.Notices.Add(DimensionIndexStore.UnbuiltNotice(unbuilt)); var slices = rawEvents.ScanIndexed( index, context.Site, segment.From, segment.To, maxDegree, () => new RawSlice(eventSources.Length, shardCount, context.GroupBy.Length), @@ -586,7 +605,7 @@ public IReadOnlyList GetEventNames( var now = DateTime.UtcNow.Ticks; var (from, to) = range.Resolve(preparation.CalendarFor(timeZone), now); site = enricher.NormalizeSite(site); - var cutoffs = RetentionCutoffs.Compute(options.Retention, storageCalendar, now); + var cutoffs = retention.At(now); var tiers = AvailableRollups(storageCalendar, TimeInterval.None, cutoffs); var plan = QueryRangePlanner.Build(storageCalendar, from, to, tiers, rawEvents != null, cancellationToken); var names = new SortedSet(StringComparer.Ordinal); diff --git a/src/ZoneTree.Analytics/Query/QueryPreparation.cs b/src/ZoneTree.Analytics/Query/QueryPreparation.cs index 484497f..58e4826 100644 --- a/src/ZoneTree.Analytics/Query/QueryPreparation.cs +++ b/src/ZoneTree.Analytics/Query/QueryPreparation.cs @@ -8,7 +8,7 @@ namespace ZoneTree.Analytics.Query; /// Validates a query and resolves the sources, dimensions and aggregates it needs. sealed class QueryPreparation( AnalyticsOptions options, ReportingCalendar storageCalendar, RollupDefinition[] definitions, - RollupActivations activations, EnrichmentPipeline enricher) + RollupActivations activations, EnrichmentPipeline enricher, RetentionClock retention) { public ReportingCalendar CalendarFor(TimeZoneInfo timeZone) { @@ -105,7 +105,7 @@ public QueryContext Prepare(AnalyticsQuery query, long now, CancellationToken ca Precision = options.Rollups.HyperLogLogPrecision, CancellationToken = cancellationToken, NowTicks = now, - Cutoffs = RetentionCutoffs.Compute(options.Retention, storageCalendar, now), + Cutoffs = retention.At(now), UseDimensionIndexes = query.Source == QuerySource.Auto }; diff --git a/src/ZoneTree.Analytics/Query/QueryResultBuilder.cs b/src/ZoneTree.Analytics/Query/QueryResultBuilder.cs index ddb888b..793f066 100644 --- a/src/ZoneTree.Analytics/Query/QueryResultBuilder.cs +++ b/src/ZoneTree.Analytics/Query/QueryResultBuilder.cs @@ -126,7 +126,7 @@ public static (AnalyticsResult, Dictionary) Build( page.ToList(), totalRows, totalsRow, - new QueryPlan(context.Steps, [.. context.Warnings], context.RollupSuggestions.AsReadOnly())); + new QueryPlan(context.Steps, [.. context.Warnings], context.RollupSuggestions.AsReadOnly(), [.. context.Notices])); return (result, bucketIndexes); } diff --git a/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs b/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs index 8bbbbc4..f455ae4 100644 --- a/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs +++ b/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs @@ -15,14 +15,24 @@ sealed class DimensionIndexUnavailableException : Exception; /// /// Owns optional covering indexes. The primary raw store is authoritative. A durable dirty /// intent is saved before writes. Completed builds survive abrupt shutdown; only affected -/// days are reconciled at the next open, before workers start. +/// days are reconciled after the next open. /// +/// +/// Builds and repairs run on the background maintenance thread while ingestion writes its own +/// copies: copying an event again writes the same key and value. Queries scan raw events until +/// the indexes are ready. A build resumes from its checkpoint after a restart; the days that +/// ingestion wrote meanwhile are repaired. Erasure and the build's chunks hold +/// one at a time, and every chunk reads the raw events again, so a build never copies an +/// erased event back. +/// sealed class DimensionIndexStore : IDisposable { const string DimensionsKey = "dimension-index.dimensions"; const string CleanKey = "dimension-index.clean"; const string PartitionsKey = "dimension-index.partitions.shared"; + const string UncleanReason = "missing or unclean checkpoint"; const int ProbeLimit = 1024; + const int ChunkSize = 4096; readonly MetaStore meta; readonly string manifest; readonly string directory; @@ -36,14 +46,25 @@ sealed class DimensionIndexStore : IDisposable int invalid; int suspended; long revision; + volatile bool ready; + bool rebuild; + long fence; + long[] dirtyDays = []; + long[] repairDays = []; public IReadOnlyList Indexes => indexes; public PartitionedStore, Memory> Store { get; private set; } public IEnumerable, Memory>> AllStores => Store == null ? [] : [Store]; - public bool IsAvailable => Volatile.Read(ref invalid) == 0 && Volatile.Read(ref suspended) == 0; + public bool IsAvailable => ready && Volatile.Read(ref invalid) == 0 && Volatile.Read(ref suspended) == 0; public bool IsHealthy => Volatile.Read(ref invalid) == 0; + /// Whether the indexes are being built or repaired; queries scan raw events meanwhile. + public bool IsBuilding => indexes.Count > 0 && !ready && IsHealthy; + + /// Held by erasure and by each chunk of a build, so that a build never copies an erased event. + public object Gate { get; } = new(); + public DimensionIndexStore(string directory, DimensionIndexOptions options, RawEventStore rawEvents, StoreFactory factory, MetaStore meta, ILogger logger) { @@ -57,10 +78,17 @@ public DimensionIndexStore(string directory, DimensionIndexOptions options, RawE manifest = string.Join('\n', options.Dimensions.Order(StringComparer.Ordinal)); } - public void Initialize() + /// + /// Opens the indexes and decides whether they need a build or a repair, which + /// then runs in the background. A new build drops the old copies first, + /// before ingestion writes new ones. Builds copy the raw events up to , + /// the last sequence stored before ingestion starts: ingestion copies the later ones. + /// + public void Initialize(long fence) { - using var startup = new StartupLog(logger, "Dimension index initialization", directory); - using var check = new StartupLog(logger, "Dimension index backfill check", directory); + this.fence = fence; + using var startup = new StageLog(logger, "Dimension index initialization", directory); + using var check = new StageLog(logger, "Dimension index backfill check", directory); var format = meta.Get(StorageFormats.DimensionIndexKey); if (format != null) StorageFormats.Require(format, StorageFormats.DimensionIndex, "The dimension indexes of the analytics data directory"); @@ -69,11 +97,11 @@ public void Initialize() if (format == null) reasons.Add("new indexes"); else { - if (meta.Get(CleanKey) != "true") reasons.Add("missing or unclean checkpoint"); + if (meta.Get(CleanKey) != "true") reasons.Add(UncleanReason); if (meta.Get(DimensionsKey) != manifest) reasons.Add("configuration changed"); } if (!factory.RawEventsDurable || !factory.DimensionIndexesDurable) reasons.Add("write-ahead logging disabled"); - var dirtyDays = recovery.Days; + dirtyDays = recovery.Days; var partitions = 0; if (options.Dimensions.Count > 0) { @@ -85,10 +113,15 @@ public void Initialize() if (IndexRecoveryScan.MissingRetainedPartitions(expected, names, factory.RawRetentionCutoff)) reasons.Add("partition manifest changed"); } - var rebuild = reasons.Count > 0; + rebuild = reasons.Count > 0 && indexes.Count > 0; + var checkpoint = Checkpoint(); + // Only an interrupted build of the same configuration resumes; any other reason starts over. + var resuming = rebuild && checkpoint.Copying && reasons.All(reason => reason == UncleanReason); + // A new build copies every day; one that resumes repairs the days written since it began. + repairDays = rebuild && !resuming ? [] : dirtyDays; var decision = indexes.Count == 0 ? "disabled" : rebuild ? "rebuild required" : dirtyDays.Length > 0 ? "repair affected days" : "reuse existing indexes"; var reason = indexes.Count == 0 ? "no configured dimensions" : rebuild ? string.Join(", ", reasons) : dirtyDays.Length > 0 ? "durable write intents require reconciliation" : "clean checkpoint and matching manifests"; - check.Complete($"decision={decision}; indexes={indexes.Count}; physical stores={(Store == null ? 0 : 1)}; partitions={partitions}; dirty days={dirtyDays.Length}; reasons={reason}"); + check.Complete($"decision={decision}; indexes={indexes.Count}; physical stores={(Store == null ? 0 : 1)}; partitions={partitions}; dirty days={dirtyDays.Length}; resuming={resuming}; reasons={reason}"); // Configuration changes rebuild the shared tree, removing discarded namespaces. // Disabling every dimension drops its partitions; interrupted cleanup is retried. if (Store == null && Directory.Exists(directory)) @@ -96,86 +129,192 @@ public void Initialize() var obsolete = Open(); foreach (var partition in obsolete.ListPartitions()) obsolete.Drop(partition); } - if ((rebuild || dirtyDays.Length > 0) && indexes.Count > 0) - Backfill(rawEvents, rebuild ? null : dirtyDays); - MarkClean(); - startup.Complete($"decision={(rebuild && indexes.Count > 0 ? "rebuild completed" : dirtyDays.Length > 0 && indexes.Count > 0 ? "repair completed" : decision)}; indexes={indexes.Count}; physical stores={(Store == null ? 0 : 1)}"); + if (rebuild && !resuming) + { + meta.Set(CleanKey, "false"); + checkpoint.Begin(); + foreach (var partition in Store.ListPartitions()) Store.Drop(partition); + Started(checkpoint); + // A new directory has nothing to copy: the build completes at once. + if (rawEvents.Store.ListPartitions().Count == 0) + Build(CancellationToken.None); + } + if (!rebuild && repairDays.Length == 0) + { + ready = true; + MarkClean(); + } + startup.Complete($"decision={(ready ? rebuild ? "rebuild completed" : decision : rebuild ? "rebuild in background" : "repair in background")}; indexes={indexes.Count}; physical stores={(Store == null ? 0 : 1)}"); } - PartitionedStore, Memory> Open() + IndexBackfillCheckpoint Checkpoint() => new(meta, "dimension-index.backfill", manifest); + + /// + /// Stops a build once a copy has failed: the copies after it were skipped, so no checkpoint + /// or coverage may claim them. + /// + void RequireHealthy() { - var store = new PartitionedStore, Memory>(directory, - options.RecordsPerPartition, factory.OpenDimensionIndex, logger, - factory.CreateDimensionIndexMaintainer, factory.WriteAheadLogDirectoryOf); - resources.Add(store); - return store; + if (!IsHealthy) + throw Failed(null); } - void Backfill(RawEventStore rawEvents, long[] days) + /// The failure of a build that cannot succeed until the engine opens again. + static AnalyticsStorageException Failed(Exception cause) => new( + "The dimension indexes could not be built" + (cause == null ? ": a copy failed." : ": " + cause.Message) + + " Queries scan raw events without them until the engine opens again, which resumes the build.", cause); + + /// Records that the indexes cover the raw events, and makes them available to queries. + void Complete(IndexBackfillCheckpoint checkpoint) { - using var build = new StartupLog(logger, "Dimension index backfill", directory); - var source = rawEvents.Store.ListPartitions(); - var checkpoint = new IndexBackfillCheckpoint(meta, "dimension-index.backfill", - manifest + "\n" + string.Join(',', source) + "\n" + (days == null ? "all" : string.Join(',', days))); - if (days == null) meta.Set(CleanKey, "false"); - if (!checkpoint.Resuming) checkpoint.Begin(); - if (!checkpoint.Copying) + RequireHealthy(); + meta.Update(entries => { - if (days == null) - foreach (var partition in Store.ListPartitions()) Store.Drop(partition); - else - foreach (var partition in Store.ListPartitions()) - { - using var lease = Store.Acquire(partition); - if (lease == null) continue; - IndexRecoveryScan.Visit(lease.Tree, days, DimensionIndexKeyCodec.HeadLength, (key, _) => lease.Tree.ForceDelete(key)); - } - IndexRecoveryScan.Persist(AllStores); - checkpoint.Cleared(); - } + SaveCoverage(entries); + checkpoint.Clear(entries); + }); + recovery.ClearRepaired(dirtyDays); + ready = true; + Interlocked.Increment(ref revision); + } + + /// + /// Records the configuration and format of a build that has dropped the old copies: an + /// interrupted build of them resumes at the next open, while the indexes stay unclean. + /// + void Started(IndexBackfillCheckpoint checkpoint) + { + meta.Update(entries => + { + SaveCoverage(entries); + entries[CleanKey] = "false"; + }); + checkpoint.Cleared(); + } + + /// + /// Runs the build or repair that decided, on the background + /// maintenance thread, then makes the indexes available to queries. Every attempt resumes + /// from the checkpoint, so a failed attempt can simply be repeated. + /// + public void Build(CancellationToken cancellationToken) + { + if (ready) return; + // Indexes that failed wait for the next open, which resumes them from their last checkpoint. + RequireHealthy(); + using var build = new StageLog(logger, "Dimension index backfill", directory); + var checkpoint = Checkpoint(); var record = new RawEvent(); var selector = new DimensionSelector(options.Dimensions); - long records = checkpoint.Count; + long records = rebuild ? checkpoint.Count : 0; var resumed = records; long rows = 0; long bytes = 0; long skipped = 0; var partitions = 0; using var writer = CreateWriter(); - var resumePartition = checkpoint.Partition; - var resumeKey = checkpoint.Key; - foreach (var partition in source) + Memory lastKey = default; + void Copy(Memory key, Memory value) { - if (string.CompareOrdinal(partition, resumePartition) < 0) continue; - using var lease = rawEvents.Store.Acquire(partition); - if (lease == null) continue; - partitions++; - Memory lastKey = default; - IndexRecoveryScan.Visit(lease.Tree, days, KeyCodec.RawEventHead, (key, value) => + lastKey = key; + ++records; + // Ingestion has copied the events it stored since the engine opened. + if (KeyCodec.ReadRawSequence(key.Span) > fence) return; + var rawKey = StorageKeyCodec.DecodeRawEvent(key.Span); + if (!RawEventCodec.TryDecode(value.Span, rawKey.Ticks, null, default, record, selector)) { skipped++; return; } + int copies; + try { - lastKey = key; - var rawKey = StorageKeyCodec.DecodeRawEvent(key.Span); - if (!RawEventCodec.TryDecode(value.Span, rawKey.Ticks, null, default, record, selector)) { skipped++; return; } - var copies = writer.Write(rawKey.Site, rawKey.Ticks, rawKey.Sequence, record.Dimensions, value); - rows += copies; - bytes += (long)copies * value.Length; - if (++records % IndexBackfillCheckpoint.Interval == 0) + copies = writer.Write(rawKey.Site, rawKey.Ticks, rawKey.Sequence, record.Dimensions, value); + } + catch (Exception exception) + { + // The writer has invalidated the indexes: no retry in this run can complete them. + throw Failed(exception); + } + rows += copies; + bytes += (long)copies * value.Length; + } + // Copies the events of a partition from a key on, a chunk at a time, and saves a checkpoint + // every interval of events. + void CopyPartition(PartitionLease, Memory> lease, long[] days, Memory after, Action> save) + { + var next = (records / IndexBackfillCheckpoint.Interval + 1) * IndexBackfillCheckpoint.Interval; + do + { + cancellationToken.ThrowIfCancellationRequested(); + lock (Gate) + after = IndexRecoveryScan.Visit(lease.Tree, days, KeyCodec.RawEventHead, Copy, after, (int)Math.Min(ChunkSize, next - records)); + if (records == next && !after.IsEmpty) { - writer.Dispose(); - IndexRecoveryScan.Persist(AllStores); - checkpoint.Save(partition, lastKey, records); - build.Progress($"checkpoint saved; events={records}; resumed events={resumed}; index rows written={rows}; payload bytes copied={bytes}", records - resumed, "events"); + save?.Invoke(after); + next += IndexBackfillCheckpoint.Interval; } - }, partition == resumePartition ? resumeKey : default); - writer.Dispose(); - IndexRecoveryScan.Persist(AllStores); - checkpoint.Save(partition, lastKey.IsEmpty && partition == resumePartition ? resumeKey : lastKey, records); - build.Progress($"checkpoint saved; partition={partition}; events={records}; resumed events={resumed}", records - resumed, "events"); + } while (!after.IsEmpty); + } + void Save(string partition, Memory key) + { + RequireHealthy(); + checkpoint.Save(partition, key, records); + } + if (rebuild) + { + var resumePartition = checkpoint.Partition; + var resumeKey = checkpoint.Key; + // Events stored later have copies from ingestion: the partitions of now hold every event to copy. + foreach (var partition in rawEvents.Store.ListPartitions()) + { + if (string.CompareOrdinal(partition, resumePartition) < 0) continue; + using var lease = rawEvents.Store.Acquire(partition); + if (lease == null) continue; + partitions++; + lastKey = partition == resumePartition ? resumeKey : default; + CopyPartition(lease, null, lastKey, key => + { + Save(partition, key); + build.Progress($"checkpoint saved; events={records}; resumed events={resumed}; index rows written={rows}; payload bytes copied={bytes}", records - resumed, "events"); + }); + Save(partition, lastKey); + build.Progress($"checkpoint saved; partition={partition}; events={records}; resumed events={resumed}", records - resumed, "events"); + } + } + if (repairDays.Length > 0) + { + // Removes the copies of the days that were stored before the engine opened, orphans of + // interrupted deletions included, then copies the days again up to the same fence. + foreach (var partition in Store.ListPartitions()) + { + using var lease = Store.Acquire(partition); + if (lease == null) continue; + IndexRecoveryScan.Visit(lease.Tree, repairDays, DimensionIndexKeyCodec.HeadLength, (key, _) => + { + if (DimensionIndexKeyCodec.Sequence(key.Span) <= fence) + lease.Tree.ForceDelete(key); + }); + } + foreach (var partition in rawEvents.Store.ListPartitions()) + { + using var lease = rawEvents.Store.Acquire(partition); + if (lease == null) continue; + partitions++; + CopyPartition(lease, repairDays, default, null); + build.Progress($"partition repaired; partition={partition}; events={records}; dirty days={repairDays.Length}", records, "events"); + } } writer.Dispose(); + // Merges the copies into disk segments, as ingestion goes on. Store.Compact(); - Store.CloseIdle(TimeSpan.Zero, keepLatestOpen: false); - build.Complete($"indexes={indexes.Count}; physical stores=1; raw partitions={partitions}; events={records}; resumed events={resumed}; dirty days={(days == null ? 0 : days.Length)}; skipped events={skipped}; index rows written={rows}; payload bytes copied={bytes}", records - resumed, "events"); + Complete(checkpoint); + build.Complete($"indexes={indexes.Count}; physical stores=1; raw partitions={partitions}; events={records}; resumed events={resumed}; dirty days={repairDays.Length}; skipped events={skipped}; index rows written={rows}; payload bytes copied={bytes}", records - resumed, "events"); + } + + PartitionedStore, Memory> Open() + { + var store = new PartitionedStore, Memory>(directory, + options.RecordsPerPartition, factory.OpenDimensionIndex, logger, + factory.CreateDimensionIndexMaintainer, factory.WriteAheadLogDirectoryOf); + resources.Add(store); + return store; } public void BeforeWrite(long ticks) @@ -219,6 +358,25 @@ public void Check(DimensionIndexSelection selection) public bool CanUse(DimensionIndexSelection selection) => IsAvailable && selection.Revision == Interlocked.Read(ref revision); + /// The query plan notice of a scan that could not read the index of a dimension yet. + public static string UnbuiltNotice(string dimension) => + $"The dimension index on '{dimension}' is being built in the background; raw events were scanned without it."; + + /// + /// The indexed dimension a query would read while the indexes are being built, or null: the + /// dimension of its first mandatory equality filter on an indexed dimension. + /// + public string Unbuilt(IEnumerable filters, Func allowed) + { + if (!IsBuilding) return null; + foreach (var filter in Mandatory(filters)) + { + if (TryEquality(filter, out var dimension, out _) && allowed(dimension) && indexes.Any(index => index.Dimension == dimension)) + return dimension; + } + return null; + } + /// /// Selects a mandatory, case-sensitive equality predicate. AND children are safe; an OR /// can only be used when every branch is equality on the same indexed dimension. @@ -277,19 +435,28 @@ internal static bool TryEquality(Filter filter, out string dimension, out string return true; } + /// + /// Records that the indexes cover the raw events, when they do: after an orderly shutdown no + /// day needs a repair. An unfinished build keeps its checkpoint and resumes at the next open. + /// public void MarkClean() { - if (Volatile.Read(ref invalid) != 0) return; + if (!ready || Volatile.Read(ref invalid) != 0) return; meta.Update(entries => { - entries[DimensionsKey] = manifest; - entries[StorageFormats.DimensionIndexKey] = StorageFormats.Text(StorageFormats.DimensionIndex); - entries[PartitionsKey] = Store == null ? "" : string.Join(',', Store.ListPartitions()); - entries[CleanKey] = factory.RawEventsDurable && factory.DimensionIndexesDurable ? "true" : "false"; + SaveCoverage(entries); recovery.Clear(entries); - new IndexBackfillCheckpoint(meta, "dimension-index.backfill", manifest).Clear(entries); + Checkpoint().Clear(entries); }); } + void SaveCoverage(Dictionary entries) + { + entries[DimensionsKey] = manifest; + entries[StorageFormats.DimensionIndexKey] = StorageFormats.Text(StorageFormats.DimensionIndex); + entries[PartitionsKey] = Store == null ? "" : string.Join(',', Store.ListPartitions()); + entries[CleanKey] = factory.RawEventsDurable && factory.DimensionIndexesDurable ? "true" : "false"; + } + public void Dispose() => Disposal.DisposeAll(resources); } diff --git a/src/ZoneTree.Analytics/Storage/IndexMaintenance.cs b/src/ZoneTree.Analytics/Storage/IndexMaintenance.cs new file mode 100644 index 0000000..0183a31 --- /dev/null +++ b/src/ZoneTree.Analytics/Storage/IndexMaintenance.cs @@ -0,0 +1,127 @@ +using ZoneTree.Logger; + +namespace ZoneTree.Analytics.Storage; + +/// +/// Builds and repairs the indexes and rebuilds joint rollup history on one background thread, +/// while the engine ingests events and answers queries. +/// +/// +/// +/// The jobs run one after another, in the order given: a rollup index build folds in the history +/// a reconstruction rebuilt before it. Each job resumes from its own checkpoint, so a job that +/// fails is retried after a delay from where it stopped, and one interrupted by disposal or a +/// crash continues when the engine opens again. A job that throws +/// cannot succeed until then: it is not retried, the next +/// jobs still run, and fails with it. +/// +/// +/// Until a job completes, queries use what does not depend on it: raw events instead of +/// dimension indexes or rebuilt history, primary rollups instead of rollup indexes. +/// +/// +sealed class IndexMaintenance : IDisposable +{ + readonly (string Name, Action Run)[] jobs; + + readonly ILogger logger; + + readonly string directory; + + readonly CancellationTokenSource stopping = new(); + + readonly TaskCompletionSource completion = new(TaskCreationOptions.RunContinuationsAsynchronously); + + Task task; + + /// The jobs, in the order they run. + /// Receives, and counts, the failures of jobs; logs a stop that waits for one. + /// The data directory, which its logs name. + public IndexMaintenance((string Name, Action Run)[] jobs, ILogger logger, string directory) + { + this.jobs = jobs; + this.logger = logger; + this.directory = directory; + } + + /// The wait before a failed job runs again. + internal static TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Completes when every job has; fails when a job could not succeed in this run; canceled when + /// the engine closes first. + /// + public Task Completion => completion.Task; + + /// The job that is running, or null. + public string Current { get; private set; } + + public void Start() + { + if (jobs.Length == 0) + { + completion.TrySetResult(); + return; + } + task = Task.Factory.StartNew(Run, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + } + + void Run() + { + var token = stopping.Token; + var failures = new List(); + foreach (var (name, run) in jobs) + { + Current = name; + while (true) + { + try + { + run(token); + break; + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + completion.TrySetCanceled(token); + return; + } + catch (AnalyticsStorageException failure) + { + // The job cannot succeed until the engine opens again: the next jobs still run. + try { logger.LogError(failure); } catch { } + failures.Add(failure); + break; + } + catch (Exception exception) + { + try { logger.LogError(exception); } catch { } + if (token.WaitHandle.WaitOne(RetryDelay)) + { + completion.TrySetCanceled(token); + return; + } + } + } + } + Current = null; + if (failures.Count > 0) + completion.TrySetException(failures); + else + completion.TrySetResult(); + } + + /// Stops the running job at its next checkpoint and waits for it. + public void Dispose() + { + stopping.Cancel(); + if (task is { IsCompleted: false }) + { + // The job resumes from its checkpoint when the engine opens again. + using var log = new StageLog(logger, "Index maintenance shutdown", directory, $"running={Current}"); + task.GetAwaiter().GetResult(); + log.Complete(null); + } + completion.TrySetCanceled(stopping.Token); + stopping.Dispose(); + } +} diff --git a/src/ZoneTree.Analytics/Storage/IndexRecovery.cs b/src/ZoneTree.Analytics/Storage/IndexRecovery.cs index 371abd4..afab9d0 100644 --- a/src/ZoneTree.Analytics/Storage/IndexRecovery.cs +++ b/src/ZoneTree.Analytics/Storage/IndexRecovery.cs @@ -5,8 +5,9 @@ namespace ZoneTree.Analytics.Storage; /// /// Durable write intents, recorded once per affected UTC day before either tree is changed. -/// The intent remains until both trees have drained their WALs. Recovery replaces absolute -/// copies only in those days, including deletions and out-of-order events. +/// The intent remains until an orderly shutdown, or until a repair of the day completes and no +/// write of the running engine has recorded it. Recovery replaces absolute copies only in +/// those days, including deletions and out-of-order events. /// sealed class IndexRecovery(MetaStore meta, string prefix) { @@ -33,6 +34,20 @@ public void Clear(Dictionary entries) foreach (var key in entries.Keys.Where(key => key.StartsWith(prefix, StringComparison.Ordinal)).ToArray()) entries.Remove(key); } + + /// + /// Removes the intents of repaired days, except the days this run has written to since it + /// started: their writes may still need a repair after a crash. + /// + public void ClearRepaired(IEnumerable days) + { + lock (sync) + meta.Update(entries => + { + foreach (var day in days.Where(day => !tracked.ContainsKey(day))) + entries.Remove(prefix + day.ToString(CultureInfo.InvariantCulture)); + }); + } } /// Seeks past untouched time ranges rather than scanning old event history. @@ -51,43 +66,48 @@ public static bool MissingRetainedPartitions(string[] expected, IReadOnlyList(IZoneTree, TValue> tree, long[] days, - KeyHeadLength headLength, Action, TValue> visit, Memory after = default) + /// + /// Visits the records after in the given days, or all records when + /// is null. Stops after records and returns + /// the last one visited, to continue from; returns an empty key when the tree has no more. + /// + public static Memory Visit(IZoneTree, TValue> tree, long[] days, + KeyHeadLength headLength, Action, TValue> visit, Memory after = default, int limit = int.MaxValue) { using var iterator = tree.CreateIterator(new IteratorOptions { IteratorType = IteratorType.NoRefresh, DiskSegmentPrefetchSize = 32, ContributeToTheBlockCache = true }); if (after.IsEmpty) iterator.SeekFirst(); else iterator.Seek(after); + var visited = 0; while (iterator.Next()) { var key = iterator.CurrentKey; if (!after.IsEmpty && key.Span.SequenceCompareTo(after.Span) <= 0) continue; - if (days == null) { visit(key, iterator.CurrentValue); continue; } - var head = key.Span[..headLength(key.Span)]; - var ticks = KeyCodec.ReadBucket(key.Span, head.Length); - var day = ticks / TimeSpan.TicksPerDay * TimeSpan.TicksPerDay; - var index = Array.BinarySearch(days, day); - if (index >= 0) { visit(key, iterator.CurrentValue); continue; } - index = ~index; - iterator.Seek(index < days.Length - ? KeyCodec.SeekKey(head, days[index]) : KeyCodec.AfterPrefix(head)); - } - } - - public static void Persist(IEnumerable, TValue>> stores) - { - foreach (var store in stores) - { - store.Compact(); - store.CloseIdle(TimeSpan.Zero, keepLatestOpen: false); + if (days != null) + { + var head = key.Span[..headLength(key.Span)]; + var ticks = KeyCodec.ReadBucket(key.Span, head.Length); + var day = ticks / TimeSpan.TicksPerDay * TimeSpan.TicksPerDay; + var index = Array.BinarySearch(days, day); + if (index < 0) + { + index = ~index; + iterator.Seek(index < days.Length + ? KeyCodec.SeekKey(head, days[index]) : KeyCodec.AfterPrefix(head)); + continue; + } + } + visit(key, iterator.CurrentValue); + if (++visited == limit) return key.ToArray(); } + return default; } } /// -/// Copy operations are idempotent. A cursor advances only after destination copies reach -/// disk; replaying the unfinished chunk cannot add events or aggregate deltas twice. +/// Copy operations are idempotent, and a write is durable once it returns: a cursor advances +/// after its copies are written, and resuming repeats at most the copies after it. /// sealed class IndexBackfillCheckpoint(MetaStore meta, string prefix, string configuration) { diff --git a/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs b/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs index fa1c842..af047c6 100644 --- a/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs +++ b/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs @@ -10,6 +10,10 @@ sealed class JointBackfillState public string Configuration { get; set; } public bool Copying { get; set; } public long Coverage { get; set; } + /// The last raw event sequence the reconstruction reads; ingestion writes later events itself. + public long Fence { get; set; } + /// The raw partitions when the reconstruction began, which hold every event it reads. + public string[] Partitions { get; set; } = []; public string Site { get; set; } public bool SiteCompleted { get; set; } public string Partition { get; set; } @@ -32,8 +36,9 @@ sealed class JointBackfillReplacement } /// -/// The cursor, active sessions and absolute replacements are committed together. A crash -/// during the writes replays replacements instead of applying additive contributions twice. +/// The cursor, active sessions and absolute rebuilt parts are committed together. A crash +/// during the writes replays the rebuilt parts instead of applying additive contributions twice; +/// a rebuilt part is apart from what ingestion writes, so replaying it never undoes a flush. /// /// /// The journal is binary: a flush of replacements holds serialized rollup rows, whose dense @@ -57,15 +62,26 @@ sealed class JointBackfillCheckpoint public JointBackfillState State { get; } public bool Resuming { get; } - public JointBackfillCheckpoint(string root, string configuration, long coverage) + /// Opens the journal: resumes its checkpoint of the configuration, or begins a new one. + /// The data directory. + /// The signature of the reconstruction. + /// The state of a new reconstruction. + /// False to begin a new reconstruction even when a checkpoint of the configuration remains. + public JointBackfillCheckpoint(string root, string configuration, Func begin, bool resume = true) { var directory = Path.Combine(root, "backfill", "joint-rollups"); Directory.CreateDirectory(directory); paths = [Path.Combine(directory, "journal-a.bin"), Path.Combine(directory, "journal-b.bin")]; var loaded = Load(); - Resuming = loaded?.Configuration == configuration; - State = Resuming ? loaded : new() { Configuration = configuration, Coverage = coverage }; - if (!Resuming) Save(); + Resuming = resume && loaded?.Configuration == configuration; + if (Resuming) + State = loaded; + else + { + State = begin(); + State.Configuration = configuration; + Save(); + } } public void Save() @@ -91,6 +107,9 @@ public void Save() ++generation; } + /// + /// Writes the rebuilt parts of the checkpoint, then commits that they are written. + /// public void Replay(RollupStore rollups) { if (State.Redo.Count == 0) return; @@ -99,10 +118,10 @@ public void Replay(RollupStore rollups) { using var lease = rollups.GetStore(group.Key.Granularity).Acquire(group.Key.Partition); if (lease == null) continue; + var writer = rollups.WriterOf(group.Key.Granularity, lease.Tree); foreach (var row in group) - lease.Tree.Upsert(row.Key, serializer.Deserialize(row.Value)); + writer.Write(new RollupWrite(row.Key, serializer.Deserialize(row.Value), RollupWriteKind.Rebuild)); } - IndexRecoveryScan.Persist(rollups.PrimaryStores); State.Redo.Clear(); Save(); } @@ -159,6 +178,10 @@ static void Write(BinaryWriter writer, JointBackfillState state) writer.Write(state.Configuration); writer.Write(state.Copying); writer.Write(state.Coverage); + writer.Write(state.Fence); + writer.Write(state.Partitions.Length); + foreach (var partition in state.Partitions) + writer.Write(partition); WriteString(writer, state.Site); writer.Write(state.SiteCompleted); WriteString(writer, state.Partition); @@ -213,6 +236,8 @@ static JointBackfillState Read(BinaryReader reader) Configuration = reader.ReadString(), Copying = reader.ReadBoolean(), Coverage = reader.ReadInt64(), + Fence = reader.ReadInt64(), + Partitions = ReadStrings(reader), Site = ReadString(reader), SiteCompleted = reader.ReadBoolean(), Partition = ReadString(reader), @@ -276,6 +301,14 @@ static void WriteString(BinaryWriter writer, string value) static string ReadString(BinaryReader reader) => reader.ReadBoolean() ? reader.ReadString() : null; + static string[] ReadStrings(BinaryReader reader) + { + var values = new string[reader.ReadInt32()]; + for (var i = 0; i < values.Length; ++i) + values[i] = reader.ReadString(); + return values; + } + static void WriteBytes(BinaryWriter writer, byte[] value) { writer.Write(value?.Length ?? -1); diff --git a/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs b/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs new file mode 100644 index 0000000..bf94130 --- /dev/null +++ b/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs @@ -0,0 +1,395 @@ +using System.Globalization; +using System.Text; +using ZoneTree.Analytics.Ingestion; +using ZoneTree.Analytics.Internal; +using ZoneTree.Analytics.Query; +using ZoneTree.Logger; + +namespace ZoneTree.Analytics.Storage; + +/// +/// Rebuilds the history of indexed joint definitions from retained raw events and their stored +/// session IDs, while ingestion runs. +/// +/// +/// +/// A definition added to an existing directory covers events from then on. Its reconstruction +/// reads the raw events stored before: those up to a fence, the raw event sequence when it began. +/// Ingestion writes every later event to the definition itself, late events included, so each +/// event counts once. A session that spans the fence adds up the same way: the reconstruction +/// contributes it as it was at the fence, ingestion the changes after it. +/// +/// +/// The contributions go to the rebuilt part of each row, which flushes keep as it is, so the +/// journal can write them again after a crash without undoing a flush. When the reconstruction +/// completes, the rollup index build folds them into the rows and then publishes the history: +/// until then, queries read raw events for it. +/// +/// +sealed class JointRollupReconstruction +{ + const string PendingKey = "rollup-index.backfill-pending"; + + /// + /// Completed reconstructions whose rebuilt parts the rollup index build folds in: lines of the + /// coverage start and the definition's activation key. + /// + const string FoldingKey = "rollup-index.folding"; + + const int BufferedRows = 4096; + + readonly AnalyticsOptions options; + readonly RollupStore rollups; + readonly RawEventStore rawEvents; + readonly ReportingCalendar calendar; + readonly SessionSchema schema; + readonly MetaStore meta; + readonly ILogger logger; + readonly RollupDefinition[] definitions; + readonly string signature; + + JointRollupReconstruction(AnalyticsOptions options, RollupStore rollups, RawEventStore rawEvents, + ReportingCalendar calendar, SessionSchema schema, MetaStore meta, ILogger logger, + RollupDefinition[] definitions, string signature, long coverage) + { + this.options = options; + this.rollups = rollups; + this.rawEvents = rawEvents; + this.calendar = calendar; + this.schema = schema; + this.meta = meta; + this.logger = logger; + this.definitions = definitions; + this.signature = signature; + Coverage = coverage; + } + + /// The start of the rebuilt history: raw retention must not pass it until the reconstruction completes. + public long Coverage { get; } + + /// + /// Decides which definitions need their history rebuilt, and prepares a new reconstruction + /// before ingestion starts: it deletes the rows those definitions kept from an earlier + /// activation. Returns null when there is nothing to rebuild. A new reconstruction reads the + /// raw events up to , the last sequence stored before ingestion starts. + /// + public static JointRollupReconstruction Prepare(AnalyticsOptions options, RollupStore rollups, RawEventStore rawEvents, + RollupActivations activations, ReportingCalendar calendar, SessionSchema schema, MetaStore meta, ILogger logger, long fence) + { + using var check = new StageLog(logger, "Joint rollup backfill check", options.DataDirectory); + if (rawEvents == null || options.Storage.RollupIndexes.Definitions.Count == 0) + { + check.Complete($"decision=disabled; reason={(rawEvents == null ? "raw event storage disabled" : "no configured combinations")}"); + return null; + } + // Only a rollup index build folds rebuilt history in and publishes it. + var folding = Folding(meta).Select(entry => entry.Key).ToHashSet(StringComparer.Ordinal); + activations.Rebuilding(options.Rollups.Definitions + .Where(definition => folding.Contains(RollupActivations.DefinitionKey(definition))).Select(definition => definition.Id)); + var partitions = rawEvents.Store.PartitionsInRange(0, DateTime.MaxValue.Ticks).ToArray(); + var cutoffs = RetentionCutoffs.Compute(options.Retention, calendar, DateTime.UtcNow.Ticks); + var coverage = Math.Max(Math.Max(0, cutoffs.RawEvents), partitions.Length == 0 ? DateTime.UtcNow.Ticks : partitions[0].From); + var pending = (meta.Get(PendingKey) ?? "").Split('\n', StringSplitOptions.RemoveEmptyEntries).ToHashSet(StringComparer.Ordinal); + var configured = options.Storage.RollupIndexes.Definitions.Select(index => index.Definition.Id).ToHashSet(StringComparer.Ordinal); + var definitions = options.Rollups.Definitions.Where(definition => configured.Contains(definition.Id) && + !folding.Contains(RollupActivations.DefinitionKey(definition)) && + (pending.Contains(definition.Id) || activations.DefinitionSince(definition.Id) > coverage)).ToArray(); + var decision = definitions.Length == 0 ? "reuse existing rollups" : "rebuild required"; + var reason = definitions.Length == 0 ? "all definitions cover retained history" : "pending rebuild or definition activated after retained history"; + var pendingCount = definitions.Count(definition => pending.Contains(definition.Id)); + check.Complete($"decision={decision}; configured joint definitions={configured.Count}; definitions to rebuild={definitions.Length}; pending definitions={pendingCount}; raw partitions={partitions.Length}; coverage starts={new DateTime(coverage, DateTimeKind.Utc):O}; reason={reason}"); + if (definitions.Length == 0) return null; + // Invalidate copies before changing their authoritative rows; the rollup index build folds + // the rebuilt parts in and copies the rows again. + var selected = definitions.Select(definition => definition.Id).ToHashSet(StringComparer.Ordinal); + meta.Update(entries => + { + entries["rollup-index.clean"] = "false"; + entries[PendingKey] = string.Join('\n', selected); + }); + activations.Rebuilding(selected); + // The raw partitions are part of the state, not the signature: ingestion adds partitions + // while the reconstruction runs. + var signature = "joint-backfill-v2\n" + string.Join('\n', definitions.OrderBy(definition => definition.Id, StringComparer.Ordinal) + .Select(definition => definition.Id + ":" + (definition.Events == null ? "*" : string.Join(',', definition.Events.Order(StringComparer.Ordinal))))) + "\n" + + options.Rollups.Granularities + "\n" + + string.Join(',', options.Rollups.ValueAggregates.OrderBy(pair => pair.Key, StringComparer.Ordinal).Select(pair => pair.Key + ":" + pair.Value)) + "\n" + + string.Join(',', options.Rollups.DistinctDimensions.Order(StringComparer.Ordinal)) + "\n" + + string.Join(',', schema.Names) + "\n" + + FormattableString.Invariant($"{options.Rollups.HyperLogLogPrecision};{options.Sessions.Enabled};{options.Sessions.Timeout.Ticks};{options.Retention.RawEvents};{options.Retention.MinuteRollups};{options.Retention.HourRollups};{options.Retention.DayRollups}"); + // A checkpoint resumes only while its definitions are pending: one left behind by a completed + // reconstruction belongs to an earlier activation of them. + var checkpoint = new JointBackfillCheckpoint(options.DataDirectory, signature, () => new() + { + Coverage = coverage, Fence = fence, Partitions = [.. partitions.Select(partition => partition.Name)] + }, resume: pending.SetEquals(selected)); + if (!checkpoint.State.Copying) + { + // The copies of deleted rows go too: the rollup index build starts over and drops them. + meta.Update(entries => + { + foreach (var key in entries.Keys.Where(key => key.StartsWith(RollupIndexStore.CheckpointPrefix + ".", StringComparison.Ordinal)).ToArray()) + entries.Remove(key); + }); + // Ingestion writes to these rows from its start: rows of an earlier activation go first. + DeleteDefinitions(rollups, selected); + checkpoint.State.Copying = true; + checkpoint.Save(); + } + return new(options, rollups, rawEvents, calendar, schema, meta, logger, definitions, signature, checkpoint.State.Coverage); + } + + /// + /// Rebuilds the history from the journal's checkpoint, on the background maintenance thread. + /// Every attempt resumes from the journal, so a failed attempt can simply be repeated. + /// + public void Run(CancellationToken cancellationToken) + { + using var build = new StageLog(logger, "Joint rollup backfill", options.DataDirectory); + var checkpoint = new JointBackfillCheckpoint(options.DataDirectory, signature, + () => throw new InvalidOperationException("The joint rollup backfill journal is missing.")); + var state = checkpoint.State; + var coverage = state.Coverage; + var fence = state.Fence; + checkpoint.Replay(rollups); + var selected = definitions.Select(definition => definition.Id).ToHashSet(StringComparer.Ordinal); + var cutoffs = RetentionCutoffs.Compute(options.Retention, calendar, DateTime.UtcNow.Ticks); + var buffer = new RollupBuffer(options.Rollups.Definitions.ToArray(), options.Rollups.Granularities, calendar, + options.Rollups.HyperLogLogPrecision, schema, options.Rollups.ValueAggregates, options.Rollups.DistinctDimensions, selected) + { Cutoffs = cutoffs }; + var names = state.Partitions.ToHashSet(StringComparer.Ordinal); + var partitions = rawEvents.Store.PartitionsInRange(0, DateTime.MaxValue.Ticks).Where(partition => names.Contains(partition.Name)).ToArray(); + var record = new RawEvent(); + var dimensions = new Dictionary(StringComparer.Ordinal); + var changes = new List(); + var pageDimensions = new PageDimensions(); + var collector = new RawSessionReader(rawEvents, schema, options); + var timeout = options.Sessions.Timeout.Ticks; + var resumed = state.Events; + var resumeSite = state.Site; + var resumeSiteCompleted = state.SiteCompleted; + var resumePartition = state.Partition; + var resumeKey = state.Key; + void Flush(bool report = false) + { + if (buffer.Count > 0) + { + state.Redo = rollups.RebuiltParts(buffer.Swap()); + state.Updates += state.Redo.Count; + state.Flushes++; + } + // Cursor/session state and rebuilt parts become durable before any of them is written. + checkpoint.Save(); + if (report && state.Redo.Count > 0) + build.Progress($"checkpoint prepared; events={state.Events}; resumed events={resumed}; replacement rows={state.Redo.Count}", state.Events - resumed, "events"); + checkpoint.Replay(rollups); + if (report) build.Progress($"checkpoint saved; events={state.Events}; resumed events={resumed}; rollup row updates={state.Updates}; flushes={state.Flushes}; sessions={state.Sessions}", state.Events - resumed, "events"); + cancellationToken.ThrowIfCancellationRequested(); + } + void Finish(string site, RawSession session) + { + state.Sessions++; + if (session.Start >= coverage) + { + RawSessionContributions.SessionValues(session, changes); + buffer.Contribute(site, StandardEvents.Session, session.Start, session.Dimensions, 1, session.Visitor, changes, DimensionScope.Session); + } + if (RawSessionContributions.Entrance(session, out var ticks, out var page, out var hostname)) + ContributePageMark(site, session, ticks, page, hostname, StandardValues.Entrances); + if (RawSessionContributions.Exit(session, out ticks, out page, out hostname)) + ContributePageMark(site, session, ticks, page, hostname, StandardValues.Exits); + } + void ContributePageMark(string site, RawSession session, long ticks, string page, string hostname, string value) + { + if (ticks < coverage) + return; + changes.Clear(); + changes.Add(new(value, 1, true)); + buffer.Contribute(site, StandardEvents.PageView, ticks, + pageDimensions.Of(session.Dimensions, page, hostname), 0, 0, changes, DimensionScope.PageView); + } + // One chronological stream per site keeps only sessions within an inactivity window. + // Original session IDs/starts preserve attribution; events are never re-ingested or re-enriched. + foreach (var site in Sites(rawEvents, partitions)) + { + var comparison = string.CompareOrdinal(site, resumeSite); + if (comparison < 0 || (comparison == 0 && resumeSiteCompleted)) continue; + var continuing = comparison == 0; + var active = continuing ? state.Active : new Dictionary(); + if (!continuing) state.Sites++; + state.Site = site; + state.SiteCompleted = false; + state.Active = active; + var queue = new PriorityQueue<(ulong Id, long Last), long>(); + foreach (var (id, session) in active) queue.Enqueue((id, session.Last), session.Last); + var prefix = KeyCodec.SitePrefix(site); + foreach (var partition in partitions) + { + if (continuing && string.CompareOrdinal(partition.Name, resumePartition) < 0) continue; + using var lease = rawEvents.Store.Acquire(partition.Name); + if (lease == null) continue; + using var iterator = lease.Tree.CreateIterator(new IteratorOptions { DiskSegmentPrefetchSize = 64, ContributeToTheBlockCache = true }); + var after = continuing && partition.Name == resumePartition ? resumeKey : null; + iterator.Seek(after is { Length: > 0 } ? after : KeyCodec.SeekKey(prefix, Math.Max(coverage, partition.From))); + while (iterator.Next()) + { + var key = iterator.CurrentKey.Span; + if (!key.StartsWith(prefix)) break; + if (after is { Length: > 0 } && key.SequenceCompareTo(after) <= 0) continue; + // Ingestion has written the events after the fence to every definition. + if (KeyCodec.ReadRawSequence(key) > fence) continue; + var ticks = KeyCodec.ReadRawTicks(key); + if (!RawEventCodec.TryDecode(iterator.CurrentValue.Span, ticks, null, default, record)) continue; + while (queue.TryPeek(out var old, out var last) && last < ticks - Math.Min(ticks, timeout)) + { + queue.Dequeue(); + if (!active.TryGetValue(old.Id, out var session) || session.Last != old.Last) continue; + Finish(site, session); active.Remove(old.Id); + // The source cursor still points to the preceding event. Save finished + // sessions before processing this event, preserving the remaining active set. + if (buffer.Count >= BufferedRows) Flush(true); + } + dimensions.Clear(); + foreach (var pair in record.Dimensions) dimensions[pair.Key] = pair.Value; + changes.Clear(); + foreach (var pair in record.Values) changes.Add(new(pair.Key, pair.Value, false)); + buffer.Contribute(site, record.Name, ticks, dimensions, 1, record.Visitor, changes, + record.Name == StandardEvents.Bot ? DimensionScope.Bot : DimensionScope.All); + if (options.Sessions.Enabled && record.SessionId != 0) + { + collector.Collect(active, record, true); + var session = active[record.SessionId]; + queue.Enqueue((record.SessionId, session.Last), session.Last); + } + state.Partition = partition.Name; + state.Key = iterator.CurrentKey.ToArray(); + ++state.Events; + if (buffer.Count >= BufferedRows || state.Events % IndexBackfillCheckpoint.Interval == 0) Flush(true); + } + Flush(true); + } + foreach (var id in active.Keys.ToArray()) + { + Finish(site, active[id]); + active.Remove(id); + if (buffer.Count >= BufferedRows) Flush(true); + } + state.SiteCompleted = true; + state.Partition = null; + state.Key = null; + Flush(true); + } + Flush(); + // The journal goes first: if the process dies before the definitions wait for their fold, + // they are still pending and their reconstruction starts over. + checkpoint.Complete(); + meta.Update(entries => + { + var folding = entries.GetValueOrDefault(FoldingKey) ?? ""; + entries[FoldingKey] = folding + string.Concat(definitions.Select(definition => + coverage.ToString(CultureInfo.InvariantCulture) + "\t" + RollupActivations.DefinitionKey(definition) + "\n")); + entries[PendingKey] = ""; + }); + build.Complete($"definitions={definitions.Length}; raw partitions={partitions.Length}; sites={state.Sites}; events={state.Events}; resumed events={resumed}; sessions={state.Sessions}; rollup row updates={state.Updates}; flushes={state.Flushes}", state.Events - resumed, "events"); + } + + /// Whether a reconstruction has not completed: the rollup index build folds its rows in after it. + public static bool IsPending(MetaStore meta) => !string.IsNullOrEmpty(meta.Get(PendingKey)); + + /// Whether a completed reconstruction waits for the rollup index build to fold its rows in. + public static bool HasFolding(MetaStore meta) => Folding(meta).Any(); + + /// + /// The configured definitions whose rows wait to be folded in, indexed or not: a definition + /// can keep its rollups after its index is removed. + /// + public static HashSet FoldingDefinitions(MetaStore meta, IEnumerable configured) + { + var keys = Folding(meta).Select(entry => entry.Key).ToHashSet(StringComparer.Ordinal); + return configured.Where(definition => keys.Contains(RollupActivations.DefinitionKey(definition))) + .Select(definition => definition.Id).ToHashSet(StringComparer.Ordinal); + } + + /// + /// Publishes the history of the reconstructions whose rows the rollup index build folded in. + /// A definition whose configuration changed since is left out: its next reconstruction starts over. + /// + public static void PublishFolded(MetaStore meta, RollupActivations activations, IEnumerable configured) + { + var folded = Folding(meta).ToArray(); + if (folded.Length == 0) return; + var byKey = configured.ToDictionary(RollupActivations.DefinitionKey, StringComparer.Ordinal); + foreach (var group in folded.GroupBy(entry => entry.Since)) + activations.Backfilled(meta, group.Where(entry => byKey.ContainsKey(entry.Key)).Select(entry => byKey[entry.Key]), group.Key); + meta.Set(FoldingKey, ""); + } + + static IEnumerable<(long Since, string Key)> Folding(MetaStore meta) + { + foreach (var line in (meta.Get(FoldingKey) ?? "").Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + var separator = line.IndexOf('\t', StringComparison.Ordinal); + yield return (long.Parse(line.AsSpan(0, separator), CultureInfo.InvariantCulture), line[(separator + 1)..]); + } + } + + static SortedSet Sites(RawEventStore rawEvents, IEnumerable partitions) + { + var sites = new SortedSet(StringComparer.Ordinal); + foreach (var partition in partitions) + { + using var lease = rawEvents.Store.Acquire(partition.Name); + if (lease == null) continue; + using var iterator = lease.Tree.CreateIterator(); + iterator.SeekFirst(); + while (iterator.Next()) + { + var key = iterator.CurrentKey.Span; + var site = KeyCodec.ReadSegment(key, 0, out var head); + sites.Add(site); + iterator.Seek(KeyCodec.AfterSegment(key[..(head - 1)])); + } + } + return sites; + } + + /// + /// Deletes every row of the definitions. Rows are ordered by site, event and definition, so + /// it seeks from one site and event to the next instead of reading the other definitions. + /// + static void DeleteDefinitions(RollupStore rollups, HashSet definitions) + { + var ids = definitions.Select(Encoding.UTF8.GetBytes).Order(Comparer.Create((a, b) => a.AsSpan().SequenceCompareTo(b))).ToArray(); + foreach (var store in rollups.PrimaryStores) + foreach (var partition in store.ListPartitions()) + { + using var lease = store.Acquire(partition); + if (lease == null) continue; + var deleted = false; + using var iterator = lease.Tree.CreateIterator(new IteratorOptions { IteratorType = IteratorType.NoRefresh }); + iterator.SeekFirst(); + while (iterator.Next()) + { + var key = iterator.CurrentKey.Span; + var head = KeyCodec.HeadLength(key, 2); + var id = key[head..(head + key[head..].IndexOf(KeyCodec.Terminator))]; + var next = 0; + while (next < ids.Length && ids[next].AsSpan().SequenceCompareTo(id) < 0) + ++next; + if (next < ids.Length && ids[next].AsSpan().SequenceEqual(id)) + { + lease.Tree.ForceDelete(iterator.CurrentKey); + deleted = true; + continue; + } + // The next definition to delete of this site and event, or the next site and event. + Memory seek = next < ids.Length ? (byte[])[.. key[..head], .. ids[next], KeyCodec.Terminator] + : KeyCodec.AfterSegment(key[..(head - 1)]); + iterator.Seek(seek); + } + // The journal records that the rows are gone once this returns. + if (deleted) + lease.Persist(); + } + } +} diff --git a/src/ZoneTree.Analytics/Storage/KeyCodec.cs b/src/ZoneTree.Analytics/Storage/KeyCodec.cs index d038d65..d2f84ca 100644 --- a/src/ZoneTree.Analytics/Storage/KeyCodec.cs +++ b/src/ZoneTree.Analytics/Storage/KeyCodec.cs @@ -149,6 +149,13 @@ public static long ReadRawTicks(ReadOnlySpan key, int sitePrefixLength) public static long ReadRawTicks(ReadOnlySpan key) => (long)BinaryPrimitives.ReadUInt64BigEndian(key.Slice(key.Length - 16, 8)); + /// + /// The sequence of a raw key: its last 8 bytes. Sequences increase in the order the engine + /// stores events, across restarts. + /// + public static long ReadRawSequence(ReadOnlySpan key) => + (long)BinaryPrimitives.ReadUInt64BigEndian(key[^8..]); + /// /// The bucket of a rollup key: the 8 bytes after its third terminator. /// diff --git a/src/ZoneTree.Analytics/Storage/PartitionedStore.cs b/src/ZoneTree.Analytics/Storage/PartitionedStore.cs index b107edb..605bff2 100644 --- a/src/ZoneTree.Analytics/Storage/PartitionedStore.cs +++ b/src/ZoneTree.Analytics/Storage/PartitionedStore.cs @@ -727,6 +727,8 @@ public void Dispose() partition.State = PartitionState.Closing; Monitor.PulseAll(sync); } + // Closing a tree waits for its background merges: shutdown logs how long each store takes. + using var log = all.Count == 0 ? null : new StageLog(logger, "Partition store shutdown", RootDirectory, $"open partitions={all.Count}"); var failures = new List(); foreach (var partition in all) { @@ -741,6 +743,8 @@ public void Dispose() { failures.Add(exception); } + if (failures.Count == 0) + log?.Complete($"closed partitions={all.Count}"); Disposal.ThrowFailures(failures); } } @@ -773,6 +777,18 @@ public PartitionLease(PartitionedStore store, string name, IZoneTr public IMaintainer Maintainer { get; } + /// + /// Moves the partition's writes to disk segments and waits; writes may continue meanwhile. + /// The default write-ahead log writes in the background and can lose the most recent writes + /// when the process dies, which an analytics engine accepts. This is for the rare writes whose + /// loss would not be small, such as deleting the rows of an earlier activation. + /// + public void Persist() + { + Maintainer.EvictToDisk(); + Maintainer.WaitForBackgroundThreads(); + } + public void Dispose() { if (Interlocked.Exchange(ref released, 1) == 0) diff --git a/src/ZoneTree.Analytics/Storage/RetentionClock.cs b/src/ZoneTree.Analytics/Storage/RetentionClock.cs index cfc38bc..cd517ee 100644 --- a/src/ZoneTree.Analytics/Storage/RetentionClock.cs +++ b/src/ZoneTree.Analytics/Storage/RetentionClock.cs @@ -19,6 +19,8 @@ sealed class RetentionClock volatile Snapshot current = new(RetentionCutoffs.None); + long rawEventsHold = long.MaxValue; + sealed record Snapshot(RetentionCutoffs Cutoffs); public RetentionClock(RetentionOptions retention, ReportingCalendar calendar) @@ -35,11 +37,36 @@ public RetentionClock(RetentionOptions retention, ReportingCalendar calendar) /// public void Refresh(long nowUtcTicks) { - var cutoffs = RetentionCutoffs.Compute(retention, calendar, nowUtcTicks); + var cutoffs = At(nowUtcTicks); if (cutoffs != current.Cutoffs) current = new Snapshot(cutoffs); } + /// The cutoffs at a point in time, including a raw event hold. + public RetentionCutoffs At(long nowUtcTicks) + { + var cutoffs = RetentionCutoffs.Compute(retention, calendar, nowUtcTicks); + var hold = Interlocked.Read(ref rawEventsHold); + return cutoffs.RawEvents > hold ? cutoffs with { RawEvents = hold } : cutoffs; + } + + /// + /// Keeps the raw events from a point in time on, beyond their retention, until + /// : a history reconstruction reads them. + /// + public void HoldRawEvents(long fromUtcTicks) + { + Interlocked.Exchange(ref rawEventsHold, fromUtcTicks); + Refresh(DateTime.UtcNow.Ticks); + } + + /// Lets raw event retention move on again. + public void ReleaseRawEvents() + { + Interlocked.Exchange(ref rawEventsHold, long.MaxValue); + Refresh(DateTime.UtcNow.Ticks); + } + public bool IsRawEventExpired(ReadOnlySpan key) => KeyCodec.ReadRawTicks(key) < current.Cutoffs.RawEvents; diff --git a/src/ZoneTree.Analytics/Storage/RollupActivations.cs b/src/ZoneTree.Analytics/Storage/RollupActivations.cs index abd9dc7..638a793 100644 --- a/src/ZoneTree.Analytics/Storage/RollupActivations.cs +++ b/src/ZoneTree.Analytics/Storage/RollupActivations.cs @@ -21,7 +21,11 @@ sealed class RollupActivations const string GranularityPrefix = "granularity:"; - readonly Dictionary definitions = new(StringComparer.Ordinal); + // Replaced, never modified, once queries read it: a reconstruction publishes coverage while + // queries run. + volatile Dictionary definitions = new(StringComparer.Ordinal); + + volatile HashSet rebuilding = new(StringComparer.Ordinal); readonly Dictionary aggregates = new(StringComparer.Ordinal); @@ -81,7 +85,7 @@ static bool IsActivationKey(string key) => /// definition holds: its dimensions and its event restriction. Changing either starts a new /// activation. /// - static string DefinitionKey(RollupDefinition definition) + public static string DefinitionKey(RollupDefinition definition) { var key = DefinitionPrefix + definition.Id; if (definition.Events == null) @@ -109,6 +113,16 @@ static IEnumerable Flags(ValueAggregates aggregates) public long DefinitionSince(string id) => definitions.TryGetValue(id, out var since) ? since : long.MaxValue; + /// + /// Marks definitions whose history is being rebuilt in the background, until + /// publishes it. + /// + public void Rebuilding(IEnumerable ids) => + rebuilding = new HashSet(rebuilding.Concat(ids), StringComparer.Ordinal); + + /// Whether the history of a definition is being rebuilt; queries read raw events for it meanwhile. + public bool IsRebuilding(string id) => rebuilding.Contains(id); + /// Publishes historical coverage only after an indexed joint rollup finishes backfilling. public void Backfilled(MetaStore meta, IEnumerable completed, long since) { @@ -118,7 +132,10 @@ public void Backfilled(MetaStore meta, IEnumerable completed, foreach (var definition in values) entries[DefinitionKey(definition)] = since.ToString(CultureInfo.InvariantCulture); }); - foreach (var definition in values) definitions[definition.Id] = since; + var updated = new Dictionary(definitions, StringComparer.Ordinal); + foreach (var definition in values) updated[definition.Id] = since; + definitions = updated; + rebuilding = new HashSet(rebuilding.Except(values.Select(definition => definition.Id)), StringComparer.Ordinal); } /// Since when a tier has been continuously enabled. diff --git a/src/ZoneTree.Analytics/Storage/RollupIndexBackfill.cs b/src/ZoneTree.Analytics/Storage/RollupIndexBackfill.cs deleted file mode 100644 index 72d73a1..0000000 --- a/src/ZoneTree.Analytics/Storage/RollupIndexBackfill.cs +++ /dev/null @@ -1,221 +0,0 @@ -using ZoneTree.Analytics.Ingestion; -using ZoneTree.Analytics.Internal; -using ZoneTree.Analytics.Query; -using ZoneTree.Logger; - -namespace ZoneTree.Analytics.Storage; - -/// Rebuilds only indexed joint definitions, using enriched retained events and their stored session IDs. -static class RollupIndexBackfill -{ - const string PendingKey = "rollup-index.backfill-pending"; - - public static bool Build(AnalyticsOptions options, RollupStore rollups, RawEventStore rawEvents, - RollupActivations activations, ReportingCalendar calendar, SessionSchema schema, MetaStore meta, ILogger logger) - { - using var check = new StartupLog(logger, "Joint rollup backfill check", options.DataDirectory); - if (rawEvents == null || options.Storage.RollupIndexes.Definitions.Count == 0) - { - check.Complete($"decision=disabled; reason={(rawEvents == null ? "raw event storage disabled" : "no configured combinations")}"); - return false; - } - var partitions = rawEvents.Store.PartitionsInRange(0, DateTime.MaxValue.Ticks).ToArray(); - var cutoffs = RetentionCutoffs.Compute(options.Retention, calendar, DateTime.UtcNow.Ticks); - var coverage = Math.Max(Math.Max(0, cutoffs.RawEvents), partitions.Length == 0 ? DateTime.UtcNow.Ticks : partitions[0].From); - var pending = (meta.Get(PendingKey) ?? "").Split('\n', StringSplitOptions.RemoveEmptyEntries).ToHashSet(StringComparer.Ordinal); - var configured = options.Storage.RollupIndexes.Definitions.Select(index => index.Definition.Id).ToHashSet(StringComparer.Ordinal); - var definitions = options.Rollups.Definitions.Where(definition => configured.Contains(definition.Id) && - (pending.Contains(definition.Id) || activations.DefinitionSince(definition.Id) > coverage)).ToArray(); - var decision = definitions.Length == 0 ? "reuse existing rollups" : "rebuild required"; - var reason = definitions.Length == 0 ? "all definitions cover retained history" : "pending rebuild or definition activated after retained history"; - var pendingCount = definitions.Count(definition => pending.Contains(definition.Id)); - check.Complete($"decision={decision}; configured joint definitions={configured.Count}; definitions to rebuild={definitions.Length}; pending definitions={pendingCount}; raw partitions={partitions.Length}; coverage starts={new DateTime(coverage, DateTimeKind.Utc):O}; reason={reason}"); - if (definitions.Length == 0) return false; - using var build = new StartupLog(logger, "Joint rollup backfill", options.DataDirectory); - // Invalidate copies before changing their authoritative rows, including the window - // between finishing reconstruction and starting the ordered-copy backfill. - meta.Set("rollup-index.clean", "false"); - var selected = definitions.Select(definition => definition.Id).ToHashSet(StringComparer.Ordinal); - meta.Set(PendingKey, string.Join('\n', selected)); - var signature = "joint-backfill-v1\n" + string.Join('\n', definitions.OrderBy(definition => definition.Id, StringComparer.Ordinal) - .Select(definition => definition.Id + ":" + (definition.Events == null ? "*" : string.Join(',', definition.Events.Order(StringComparer.Ordinal))))) + "\n" + - string.Join(',', partitions.Select(partition => partition.Name)) + "\n" + options.Rollups.Granularities + "\n" + - string.Join(',', options.Rollups.ValueAggregates.OrderBy(pair => pair.Key, StringComparer.Ordinal).Select(pair => pair.Key + ":" + pair.Value)) + "\n" + - string.Join(',', options.Rollups.DistinctDimensions.Order(StringComparer.Ordinal)) + "\n" + - string.Join(',', schema.Names) + "\n" + - FormattableString.Invariant($"{options.Rollups.HyperLogLogPrecision};{options.Sessions.Enabled};{options.Sessions.Timeout.Ticks};{options.Retention.RawEvents};{options.Retention.MinuteRollups};{options.Retention.HourRollups};{options.Retention.DayRollups}"); - var checkpoint = new JointBackfillCheckpoint(options.DataDirectory, signature, coverage); - var state = checkpoint.State; - coverage = state.Coverage; - if (!state.Copying) - { - DeleteDefinitions(rollups, selected); - IndexRecoveryScan.Persist(rollups.PrimaryStores); - state.Copying = true; - checkpoint.Save(); - } - checkpoint.Replay(rollups); - var buffer = new RollupBuffer(options.Rollups.Definitions.ToArray(), options.Rollups.Granularities, calendar, - options.Rollups.HyperLogLogPrecision, schema, options.Rollups.ValueAggregates, options.Rollups.DistinctDimensions, selected) - { Cutoffs = cutoffs }; - var record = new RawEvent(); - var dimensions = new Dictionary(StringComparer.Ordinal); - var changes = new List(); - var pageDimensions = new PageDimensions(); - var collector = new RawSessionReader(rawEvents, schema, options); - var timeout = options.Sessions.Timeout.Ticks; - var resumed = state.Events; - var resumeSite = state.Site; - var resumeSiteCompleted = state.SiteCompleted; - var resumePartition = state.Partition; - var resumeKey = state.Key; - void Flush(bool report = false) - { - if (buffer.Count > 0) - { - state.Redo = rollups.BackfillReplacements(buffer.Swap()); - state.Updates += state.Redo.Count; - state.Flushes++; - } - // Cursor/session state and replacements become durable before any additive change. - checkpoint.Save(); - if (report && state.Redo.Count > 0) - build.Progress($"checkpoint prepared; events={state.Events}; resumed events={resumed}; replacement rows={state.Redo.Count}", state.Events - resumed, "events"); - checkpoint.Replay(rollups); - if (report) build.Progress($"checkpoint saved; events={state.Events}; resumed events={resumed}; rollup row updates={state.Updates}; flushes={state.Flushes}; sessions={state.Sessions}", state.Events - resumed, "events"); - } - void Finish(string site, RawSession session) - { - state.Sessions++; - if (session.Start >= coverage) - { - RawSessionContributions.SessionValues(session, changes); - buffer.Contribute(site, StandardEvents.Session, session.Start, session.Dimensions, 1, session.Visitor, changes, DimensionScope.Session); - } - if (RawSessionContributions.Entrance(session, out var ticks, out var page, out var hostname)) - ContributePageMark(site, session, ticks, page, hostname, StandardValues.Entrances); - if (RawSessionContributions.Exit(session, out ticks, out page, out hostname)) - ContributePageMark(site, session, ticks, page, hostname, StandardValues.Exits); - } - void ContributePageMark(string site, RawSession session, long ticks, string page, string hostname, string value) - { - if (ticks < coverage) - return; - changes.Clear(); - changes.Add(new(value, 1, true)); - buffer.Contribute(site, StandardEvents.PageView, ticks, - pageDimensions.Of(session.Dimensions, page, hostname), 0, 0, changes, DimensionScope.PageView); - } - // One chronological stream per site keeps only sessions within an inactivity window. - // Original session IDs/starts preserve attribution; events are never re-ingested or re-enriched. - foreach (var site in Sites(rawEvents)) - { - var comparison = string.CompareOrdinal(site, resumeSite); - if (comparison < 0 || (comparison == 0 && resumeSiteCompleted)) continue; - var continuing = comparison == 0; - var active = continuing ? state.Active : new Dictionary(); - if (!continuing) state.Sites++; - state.Site = site; - state.SiteCompleted = false; - state.Active = active; - var queue = new PriorityQueue<(ulong Id, long Last), long>(); - foreach (var (id, session) in active) queue.Enqueue((id, session.Last), session.Last); - var prefix = KeyCodec.SitePrefix(site); - foreach (var partition in partitions) - { - if (continuing && string.CompareOrdinal(partition.Name, resumePartition) < 0) continue; - using var lease = rawEvents.Store.Acquire(partition.Name); - if (lease == null) continue; - using var iterator = lease.Tree.CreateIterator(new IteratorOptions { DiskSegmentPrefetchSize = 64, ContributeToTheBlockCache = true }); - var after = continuing && partition.Name == resumePartition ? resumeKey : null; - iterator.Seek(after is { Length: > 0 } ? after : KeyCodec.SeekKey(prefix, Math.Max(coverage, partition.From))); - while (iterator.Next()) - { - var key = iterator.CurrentKey.Span; - if (!key.StartsWith(prefix)) break; - if (after is { Length: > 0 } && key.SequenceCompareTo(after) <= 0) continue; - var ticks = KeyCodec.ReadRawTicks(key); - if (!RawEventCodec.TryDecode(iterator.CurrentValue.Span, ticks, null, default, record)) continue; - while (queue.TryPeek(out var old, out var last) && last < ticks - Math.Min(ticks, timeout)) - { - queue.Dequeue(); - if (!active.TryGetValue(old.Id, out var session) || session.Last != old.Last) continue; - Finish(site, session); active.Remove(old.Id); - // The source cursor still points to the preceding event. Save finished - // sessions before processing this event, preserving the remaining active set. - if (buffer.Count >= 4096) Flush(true); - } - dimensions.Clear(); - foreach (var pair in record.Dimensions) dimensions[pair.Key] = pair.Value; - changes.Clear(); - foreach (var pair in record.Values) changes.Add(new(pair.Key, pair.Value, false)); - buffer.Contribute(site, record.Name, ticks, dimensions, 1, record.Visitor, changes, - record.Name == StandardEvents.Bot ? DimensionScope.Bot : DimensionScope.All); - if (options.Sessions.Enabled && record.SessionId != 0) - { - collector.Collect(active, record, true); - var session = active[record.SessionId]; - queue.Enqueue((record.SessionId, session.Last), session.Last); - } - state.Partition = partition.Name; - state.Key = iterator.CurrentKey.ToArray(); - ++state.Events; - if (buffer.Count >= 4096 || state.Events % IndexBackfillCheckpoint.Interval == 0) Flush(true); - } - Flush(true); - } - foreach (var id in active.Keys.ToArray()) - { - Finish(site, active[id]); - active.Remove(id); - if (buffer.Count >= 4096) Flush(true); - } - state.SiteCompleted = true; - state.Partition = null; - state.Key = null; - Flush(true); - } - Flush(); - IndexRecoveryScan.Persist(rollups.PrimaryStores); - activations.Backfilled(meta, definitions, coverage); - meta.Set(PendingKey, ""); - checkpoint.Complete(); - build.Complete($"definitions={definitions.Length}; raw partitions={partitions.Length}; sites={state.Sites}; events={state.Events}; resumed events={resumed}; sessions={state.Sessions}; rollup row updates={state.Updates}; flushes={state.Flushes}", state.Events - resumed, "events"); - return true; - } - - static SortedSet Sites(RawEventStore rawEvents) - { - var sites = new SortedSet(StringComparer.Ordinal); - foreach (var partition in rawEvents.Store.ListPartitions()) - { - using var lease = rawEvents.Store.Acquire(partition); - if (lease == null) continue; - using var iterator = lease.Tree.CreateIterator(); - iterator.SeekFirst(); - while (iterator.Next()) - { - var key = iterator.CurrentKey.Span; - var site = KeyCodec.ReadSegment(key, 0, out var head); - sites.Add(site); - iterator.Seek(KeyCodec.AfterSegment(key[..(head - 1)])); - } - } - return sites; - } - - static void DeleteDefinitions(RollupStore rollups, HashSet definitions) - { - foreach (var store in rollups.PrimaryStores) - foreach (var partition in store.ListPartitions()) - { - using var lease = store.Acquire(partition); - if (lease == null) continue; - using var iterator = lease.Tree.CreateIterator(new IteratorOptions { IteratorType = IteratorType.NoRefresh }); - iterator.SeekFirst(); - while (iterator.Next()) - if (definitions.Contains(KeyCodec.ReadDimensionSetId(iterator.CurrentKey.Span))) - lease.Tree.ForceDelete(iterator.CurrentKey); - } - } -} diff --git a/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs b/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs index 1061235..918e2de 100644 --- a/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs +++ b/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs @@ -10,9 +10,21 @@ sealed record RollupIndex(string Dimension, string DefinitionId, Granularity Gra } /// Recoverable, absolute copies of authoritative joint rollup rows. +/// +/// Builds and repairs run on the background maintenance thread while flushes write their own +/// copies. Queries read the primary rollups until the indexes are ready. A build resumes from +/// its checkpoint after a restart; the days that flushes wrote meanwhile are repaired. +/// sealed class RollupIndexStore : IDisposable { const string CleanKey = "rollup-index.clean"; + + /// The prefix of the build checkpoint, which a new history reconstruction clears. + public const string CheckpointPrefix = "rollup-index.backfill"; + + const string UncleanReason = "missing or unclean checkpoint"; + const string FoldingReason = "rebuilt history to fold in"; + const int BackgroundBatch = 4096; readonly string root; readonly AnalyticsOptions options; readonly RollupStore rollups; @@ -28,6 +40,11 @@ sealed class RollupIndexStore : IDisposable int invalid; long revision; string manifest; + RollupActivations activations; + volatile bool ready; + bool rebuild; + Dictionary dirtyDays = []; + Dictionary repairDays = []; public RollupIndexStore(string root, AnalyticsOptions options, RollupStore rollups, StoreFactory factory, MetaStore meta, ILogger logger) { @@ -41,15 +58,24 @@ public RollupIndexStore(string root, AnalyticsOptions options, RollupStore rollu public IReadOnlyList Indexes => indexes; public bool IsHealthy => Volatile.Read(ref invalid) == 0; + + /// Whether the indexes are being built or repaired; queries read the primary rollups meanwhile. + public bool IsBuilding => indexes.Count > 0 && !ready && IsHealthy; public IEnumerable, RollupValue>> AllStores => stores.Values; static string Registration(string dimension, string definitionId, Granularity granularity) => string.Join(KeyCodec.ValueSeparator, dimension, definitionId, granularity.ToString()); - public void Initialize(RawEventStore rawEvents, RollupActivations activations, ReportingCalendar calendar, Ingestion.SessionSchema schema) + /// + /// Opens the indexes and decides whether they need a build or a repair, which + /// then runs in the background. A new build drops the old copies first, + /// before ingestion writes new ones. + /// + public void Initialize(RollupActivations activations) { - using var startup = new StartupLog(logger, "Rollup index initialization", root); - using var check = new StartupLog(logger, "Rollup index backfill check", root); + this.activations = activations; + using var startup = new StageLog(logger, "Rollup index initialization", root); + using var check = new StageLog(logger, "Rollup index backfill check", root); var format = meta.Get(StorageFormats.RollupIndexKey); if (format != null) StorageFormats.Require(format, StorageFormats.RollupIndex, "The rollup indexes of the analytics data directory"); @@ -63,10 +89,12 @@ public void Initialize(RawEventStore rawEvents, RollupActivations activations, R if (format == null) reasons.Add("new indexes"); else { - if (meta.Get(CleanKey) != "true") reasons.Add("missing or unclean checkpoint"); + if (meta.Get(CleanKey) != "true") reasons.Add(UncleanReason); if (meta.Get("rollup-index.config") != manifest) reasons.Add("configuration changed"); } if (!factory.RollupsDurable || !factory.RollupIndexesDurable) reasons.Add("write-ahead logging disabled"); + // Rows of a completed history reconstruction wait for this build to fold them in. + if (JointRollupReconstruction.HasFolding(meta)) reasons.Add(FoldingReason); var partitions = 0; foreach (var granularity in RollupStore.Granularities.Where(granularity => rollups.IsEnabled(granularity) && options.Storage.RollupIndexes.Definitions.Count > 0)) @@ -83,12 +111,17 @@ public void Initialize(RawEventStore rawEvents, RollupActivations activations, R reasons.Add($"partition manifest changed: {granularity}"); } copies = indexes.GroupBy(index => (index.Granularity, index.DefinitionId)).ToDictionary(group => group.Key, group => group.ToArray()); - var rebuild = reasons.Count > 0; - var dirtyDays = recovery.ToDictionary(pair => pair.Key, pair => pair.Value.Days); + rebuild = reasons.Count > 0 && indexes.Count > 0; + var checkpoint = Checkpoint(); + // Only an interrupted build of the same configuration resumes; any other reason starts over. + var resuming = rebuild && checkpoint.Copying && reasons.All(reason => reason is UncleanReason or FoldingReason); + dirtyDays = recovery.ToDictionary(pair => pair.Key, pair => pair.Value.Days); + // A new build copies every day; one that resumes repairs the days written since it began. + repairDays = rebuild && !resuming ? [] : dirtyDays.Where(pair => pair.Value.Length > 0).ToDictionary(); var dirtyCount = dirtyDays.Values.Sum(days => days.Length); var decision = indexes.Count == 0 ? "disabled" : rebuild ? "rebuild required" : dirtyCount > 0 ? "repair affected days" : "reuse existing indexes unless historical rollups change"; var reason = indexes.Count == 0 ? "no configured combinations or enabled tiers" : rebuild ? string.Join(", ", reasons) : dirtyCount > 0 ? "durable write intents require reconciliation" : "clean checkpoint and matching manifests"; - check.Complete($"decision={decision}; indexes={indexes.Count}; physical stores={stores.Count}; partitions={partitions}; dirty days={dirtyCount}; reasons={reason}"); + check.Complete($"decision={decision}; indexes={indexes.Count}; physical stores={stores.Count}; partitions={partitions}; dirty days={dirtyCount}; resuming={resuming}; reasons={reason}"); // Disabled tiers must not retain copies. Only the three known tier directories are owned here. foreach (var granularity in RollupStore.Granularities.Where(granularity => !stores.ContainsKey(granularity))) { @@ -96,84 +129,164 @@ public void Initialize(RawEventStore rawEvents, RollupActivations activations, R var obsolete = Open(granularity); foreach (var partition in obsolete.ListPartitions()) obsolete.Drop(partition); } - // Historical reconstruction publishes coverage only after its durable checkpoint. - var historicalBackfill = RollupIndexBackfill.Build(options, rollups, rawEvents, activations, calendar, schema, meta, logger); - rebuild |= historicalBackfill; - if ((!rebuild && dirtyCount == 0) || indexes.Count == 0) + if (rebuild && !resuming) + { + meta.Set(CleanKey, "false"); + checkpoint.Begin(); + foreach (var store in stores.Values) + foreach (var partition in store.ListPartitions()) store.Drop(partition); + Started(checkpoint); + // A new directory has nothing to copy: the build completes at once. + if (rollups.PrimaryStores.All(store => store.ListPartitions().Count == 0) && !JointRollupReconstruction.IsPending(meta)) + Build(CancellationToken.None); + } + if (!rebuild && repairDays.Count == 0) { + ready = true; MarkClean(); - startup.Complete($"decision={(indexes.Count == 0 ? "disabled" : "reuse existing indexes")}; indexes={indexes.Count}; physical stores={stores.Count}"); - return; } - Backfill(rebuild ? null : dirtyDays, historicalBackfill); - MarkClean(); - startup.Complete($"decision={(rebuild ? "rebuild completed" : "repair completed")}; indexes={indexes.Count}; physical stores={stores.Count}"); + startup.Complete($"decision={(ready ? indexes.Count == 0 ? "disabled" : rebuild ? "rebuild completed" : "reuse existing indexes" : rebuild ? "rebuild in background" : "repair in background")}; indexes={indexes.Count}; physical stores={stores.Count}"); + } + + IndexBackfillCheckpoint Checkpoint() => new(meta, CheckpointPrefix, manifest); + + /// + /// Stops a build once a copy has failed: the copies after it were skipped, so no checkpoint, + /// coverage or published history may claim them. + /// + void RequireHealthy() + { + if (!IsHealthy) + throw new AnalyticsStorageException( + "The rollup indexes could not be built: a copy failed. Queries read the primary rollups until the engine opens again, which resumes the build."); + } + + /// + /// Publishes the history the build folded in, records that the indexes copy the rollups, and + /// makes them available to queries. + /// + void Complete(IndexBackfillCheckpoint checkpoint) + { + RequireHealthy(); + JointRollupReconstruction.PublishFolded(meta, activations, options.Rollups.Definitions); + meta.Update(entries => + { + SaveCoverage(entries); + checkpoint.Clear(entries); + }); + foreach (var (granularity, journal) in recovery) journal.ClearRepaired(dirtyDays[granularity]); + ready = true; + Interlocked.Increment(ref revision); } - void Backfill(Dictionary dirtyDays, bool historicalBackfill) + /// + /// Records the configuration and format of a build that has dropped the old copies: an + /// interrupted build of them resumes at the next open, while the indexes stay unclean. + /// + void Started(IndexBackfillCheckpoint checkpoint) { - using var build = new StartupLog(logger, "Rollup index backfill", root); + meta.Update(entries => + { + SaveCoverage(entries); + entries[CleanKey] = "false"; + }); + checkpoint.Cleared(); + } + + /// + /// Runs the build or repair that decided, on the background + /// maintenance thread after any history reconstruction, then makes the indexes available to + /// queries and publishes the rebuilt history. Every attempt resumes from the checkpoint, so a + /// failed attempt can simply be repeated. + /// + /// + /// Each row goes through the writer of its partition as a fold, under the row lock of the + /// flushes: the fold merges the row's rebuilt part in, and the row's current value is copied. + /// Flushes copy the rows they write themselves, so a copy is never older than its row. + /// + public void Build(CancellationToken cancellationToken) + { + if (ready) return; + // Indexes that failed wait for the next open, which resumes them from their last checkpoint. + RequireHealthy(); + using var build = new StageLog(logger, "Rollup index backfill", root); + var checkpoint = Checkpoint(); + var historical = JointRollupReconstruction.HasFolding(meta); + // Rebuilt rows wait for this fold even when their definition is no longer indexed. + var folding = JointRollupReconstruction.FoldingDefinitions(meta, options.Rollups.Definitions); + long count = rebuild ? checkpoint.Count : 0; + var resumed = count; + long rows = 0; + long matched = 0; + var sourcePartitions = 0; var sources = RollupStore.Granularities.Where(rollups.IsEnabled).SelectMany(granularity => rollups.GetStore(granularity).ListPartitions().Select(partition => (Granularity: granularity, Partition: partition, Id: ((int)granularity).ToString("D2", System.Globalization.CultureInfo.InvariantCulture) + ":" + partition))).OrderBy(source => source.Id, StringComparer.Ordinal).ToArray(); - var signature = manifest + "\n" + string.Join(',', sources.Select(source => source.Id)) + "\n" + - (dirtyDays == null ? "all" : string.Join(';', dirtyDays.OrderBy(pair => pair.Key).Select(pair => pair.Key + ":" + string.Join(',', pair.Value)))); - var checkpoint = new IndexBackfillCheckpoint(meta, "rollup-index.backfill", signature); - if (dirtyDays == null) meta.Set(CleanKey, "false"); - if (!checkpoint.Resuming) checkpoint.Begin(); - if (!checkpoint.Copying) + // Folds and copies the rows of a partition from a key on. + void FoldPartition((Granularity Granularity, string Partition, string Id) source, long[] days, Memory after, bool save) { - foreach (var (granularity, store) in stores) - foreach (var partition in store.ListPartitions()) + cancellationToken.ThrowIfCancellationRequested(); + using var lease = rollups.GetStore(source.Granularity).Acquire(source.Partition); + if (lease == null) return; + sourcePartitions++; + var writer = rollups.WriterOf(source.Granularity, lease.Tree); + var lastKey = after; + IndexRecoveryScan.Visit(lease.Tree, days, KeyCodec.RollupHead, (key, _) => + { + lastKey = key; + if (++count % BackgroundBatch == 0) cancellationToken.ThrowIfCancellationRequested(); + var definition = KeyCodec.ReadDimensionSetId(key.Span); + if (copies.TryGetValue((source.Granularity, definition), out var destinations)) { - if (dirtyDays == null) { store.Drop(partition); continue; } - using var lease = store.Acquire(partition); - if (lease == null) continue; - IndexRecoveryScan.Visit(lease.Tree, dirtyDays[granularity], RollupIndexKeyCodec.HeadLength, - (key, _) => lease.Tree.ForceDelete(key)); + writer.Write(new RollupWrite(key.ToArray(), null, RollupWriteKind.Fold)); + rows += destinations.Length; + matched++; } - IndexRecoveryScan.Persist(AllStores); - checkpoint.Cleared(); + else if (folding.Contains(definition)) + writer.Write(new RollupWrite(key.ToArray(), null, RollupWriteKind.Fold)); + if (save && count % IndexBackfillCheckpoint.Interval == 0) + { + RequireHealthy(); + checkpoint.Save(source.Id, key, count); + build.Progress($"checkpoint saved; rollup rows scanned={count}; resumed rows={resumed}; indexed source rows={matched}; index rows written={rows}", count - resumed, "rows"); + } + }, after); + if (!save) return; + RequireHealthy(); + checkpoint.Save(source.Id, lastKey, count); + build.Progress($"checkpoint saved; partition={source.Id}; rollup rows scanned={count}; resumed rows={resumed}", count - resumed, "rows"); } - long count = checkpoint.Count; - var resumed = count; - long rows = 0; - long matched = 0; - var sourcePartitions = 0; - var resumePartition = checkpoint.Partition; - var resumeKey = checkpoint.Key; - foreach (var source in sources) + if (rebuild) + { + var resumePartition = checkpoint.Partition; + var resumeKey = checkpoint.Key; + foreach (var source in sources) { if (string.CompareOrdinal(source.Id, resumePartition) < 0) continue; - if (dirtyDays != null && (!dirtyDays.TryGetValue(source.Granularity, out var selectedDays) || selectedDays.Length == 0)) continue; - using var lease = rollups.GetStore(source.Granularity).Acquire(source.Partition); - if (lease == null) continue; - sourcePartitions++; - Memory lastKey = default; - IndexRecoveryScan.Visit(lease.Tree, dirtyDays?.GetValueOrDefault(source.Granularity), KeyCodec.RollupHead, (key, value) => - { - lastKey = key; - var written = CopyCore(source.Granularity, key, value); - rows += written; - if (written > 0) matched++; - if (++count % IndexBackfillCheckpoint.Interval == 0) - { - IndexRecoveryScan.Persist(AllStores); - checkpoint.Save(source.Id, lastKey, count); - build.Progress($"checkpoint saved; rollup rows scanned={count}; resumed rows={resumed}; indexed source rows={matched}; index rows written={rows}", count - resumed, "rows"); - } - }, source.Id == resumePartition ? resumeKey : default); - IndexRecoveryScan.Persist(AllStores); - checkpoint.Save(source.Id, lastKey.IsEmpty && source.Id == resumePartition ? resumeKey : lastKey, count); - build.Progress($"checkpoint saved; partition={source.Id}; rollup rows scanned={count}; resumed rows={resumed}", count - resumed, "rows"); + FoldPartition(source, null, source.Id == resumePartition ? resumeKey : default, save: true); } - // Release replayed snapshots and drain index WALs before reporting the build as ready. - foreach (var store in AllStores) + } + if (repairDays.Count > 0) { - store.Compact(); - store.CloseIdle(TimeSpan.Zero, keepLatestOpen: false); + // Removes orphaned copies of interrupted writes, then copies the days again. A flush + // writes the row before its copy, so a copy deleted here is copied again below. + foreach (var (granularity, store) in stores) + { + if (!repairDays.TryGetValue(granularity, out var days)) continue; + foreach (var partition in store.ListPartitions()) + { + using var lease = store.Acquire(partition); + if (lease == null) continue; + IndexRecoveryScan.Visit(lease.Tree, days, RollupIndexKeyCodec.HeadLength, (key, _) => lease.Tree.ForceDelete(key)); + } + } + foreach (var source in sources.Where(source => repairDays.ContainsKey(source.Granularity))) + FoldPartition(source, repairDays[source.Granularity], default, save: false); } - build.Complete($"indexes={indexes.Count}; physical stores={stores.Count}; source partitions={sourcePartitions}; rollup rows scanned={count}; resumed rows={resumed}; indexed source rows={matched}; index rows written={rows}; historical backfill={historicalBackfill}", count - resumed, "rows"); + // Merges the copies into disk segments, as ingestion goes on. + foreach (var store in AllStores) store.Compact(); + Complete(checkpoint); + build.Complete($"indexes={indexes.Count}; physical stores={stores.Count}; source partitions={sourcePartitions}; rollup rows scanned={count}; resumed rows={resumed}; indexed source rows={matched}; index rows written={rows}; historical backfill={historical}", count - resumed, "rows"); } static string PartitionsKey(Granularity granularity) => "rollup-index.partitions." + granularity; @@ -208,30 +321,43 @@ public void BeforeWrite(Granularity granularity, Memory key) if (copies.ContainsKey((granularity, definition))) journal.Track(KeyCodec.ReadRollupBucket(key.Span)); } - int CopyCore(Granularity granularity, Memory originalKey, RollupValue value) + void CopyCore(Granularity granularity, Memory originalKey, RollupValue value) { // Ordinary rollup rows do not need dimension arrays/dictionaries merely to discover // that their combination is not indexed. var definitionId = KeyCodec.ReadDimensionSetId(originalKey.Span); - if (!copies.TryGetValue((granularity, definitionId), out var destinations)) return 0; + if (!copies.TryGetValue((granularity, definitionId), out var destinations)) return; var key = StorageKeyCodec.DecodeRollup(originalKey.Span); var store = destinations[0].Store; using var lease = store.Acquire(store.RouteWrite(key.BucketTicks)); - if (lease == null) return 0; - var written = 0; + if (lease == null) return; foreach (var index in destinations) { var match = KeyCodec.Stored(key.Dimensions[index.Dimension]); lease.Tree.Upsert(RollupIndexKeyCodec.Key(index.KeyPrefix, key.Site, key.Event, match, key.BucketTicks, originalKey.Span), value); - written++; } - return written; + } + + /// + /// The indexed dimension a rollup read would use while the indexes are being built, or null: + /// the dimension of its first mandatory equality filter with an index of the definition. + /// + public string Unbuilt(Granularity granularity, string definitionId, IEnumerable filters) + { + if (!IsBuilding) return null; + foreach (var filter in DimensionIndexStore.Mandatory(filters)) + { + if (DimensionIndexStore.TryEquality(filter, out var dimension, out _) && indexes.Any(index => + index.Dimension == dimension && index.DefinitionId == definitionId && index.Granularity == granularity)) + return dimension; + } + return null; } public string Scan(Granularity granularity, string site, string eventName, string definitionId, long from, long to, IEnumerable filters, Action visit, CancellationToken cancellationToken) { - if (!IsHealthy || from >= to) return null; + if (!IsHealthy || !ready || from >= to) return null; foreach (var filter in DimensionIndexStore.Mandatory(filters)) { if (!DimensionIndexStore.TryEquality(filter, out var dimension, out var values)) continue; @@ -295,19 +421,28 @@ public int ApplyRetention(RetentionCutoffs cutoffs, RetentionClock clock) return dropped; } + /// + /// Records that the indexes copy the rollups, when they do: after an orderly shutdown no day + /// needs a repair. An unfinished build keeps its checkpoint and resumes at the next open. + /// public void MarkClean() { - if (!IsHealthy) return; + if (!IsHealthy || !ready) return; meta.Update(entries => { - entries[CleanKey] = factory.RollupsDurable && factory.RollupIndexesDurable ? "true" : "false"; - entries[StorageFormats.RollupIndexKey] = StorageFormats.Text(StorageFormats.RollupIndex); - entries["rollup-index.config"] = manifest; - foreach (var (granularity, store) in stores) entries[PartitionsKey(granularity)] = string.Join(',', store.ListPartitions()); + SaveCoverage(entries); foreach (var journal in recovery.Values) journal.Clear(entries); - new IndexBackfillCheckpoint(meta, "rollup-index.backfill", manifest).Clear(entries); + Checkpoint().Clear(entries); }); } + void SaveCoverage(Dictionary entries) + { + entries[CleanKey] = factory.RollupsDurable && factory.RollupIndexesDurable ? "true" : "false"; + entries[StorageFormats.RollupIndexKey] = StorageFormats.Text(StorageFormats.RollupIndex); + entries["rollup-index.config"] = manifest; + foreach (var (granularity, store) in stores) entries[PartitionsKey(granularity)] = string.Join(',', store.ListPartitions()); + } + public void Dispose() => Disposal.DisposeAll(resources); } diff --git a/src/ZoneTree.Analytics/Storage/RollupStore.cs b/src/ZoneTree.Analytics/Storage/RollupStore.cs index e034c69..057defc 100644 --- a/src/ZoneTree.Analytics/Storage/RollupStore.cs +++ b/src/ZoneTree.Analytics/Storage/RollupStore.cs @@ -14,7 +14,8 @@ namespace ZoneTree.Analytics.Storage; /// A per partition serializes the merges of each row by lock stripe /// and resumes failed writes, which keeps concurrent writes consistent even when an upsert throws /// after updating memory. During ingestion, the threads of the -/// call . +/// call ; background index +/// maintenance writes rebuilt history and folds rows through the same writers (). /// sealed class RollupStore : IDisposable { @@ -86,9 +87,7 @@ public void Write(RollupWriteBatch batch) partition.Lease ??= GetStore(partition.Granularity).Acquire(partition.Name); if (partition.Lease == null) continue; // The partition was dropped by retention. - var granularity = partition.Granularity; - var writer = writers.GetValue(partition.Lease.Tree, tree => new RollupWriter(tree, - (key, value) => Indexes?.Copy(granularity, key, value), key => Indexes?.BeforeWrite(granularity, key))); + var writer = WriterOf(partition.Granularity, partition.Lease.Tree); for (; partition.NextRow < partition.Rows.Count; ++partition.NextRow) writer.Write(partition.Rows[partition.NextRow]); partition.Lease.Dispose(); @@ -96,6 +95,14 @@ public void Write(RollupWriteBatch batch) } } + /// + /// The writer of a partition tree, which every write of its rows goes through: flushes, and + /// the rebuilds and folds of background maintenance. + /// + public RollupWriter WriterOf(Granularity granularity, IZoneTree, RollupValue> tree) => + writers.GetValue(tree, tree => new RollupWriter(tree, + (key, value) => Indexes?.Copy(granularity, key, value), key => Indexes?.BeforeWrite(granularity, key))); + List PrepareWrites(Dictionary rows) { var groups = new Dictionary<(Granularity, string), RollupWriteBatch.PartitionWrites>(); @@ -115,8 +122,15 @@ public void Write(RollupWriteBatch batch) return [.. groups.Values]; } - /// Startup-only absolute replacements for an atomic backfill checkpoint. - public List BackfillReplacements(Dictionary rows) + /// + /// The rebuilt parts of the rows after adding reconstructed contributions to them. They are + /// absolute, so a checkpoint can write them again after a crash. + /// + /// + /// Only the reconstruction writes rebuilt parts, so they are read without the row locks: + /// concurrent flushes change the rest of the row. + /// + public List RebuiltParts(Dictionary rows) { var result = new List(rows.Count); var serializer = new RollupValueSerializer(); @@ -130,7 +144,7 @@ public List BackfillReplacements(Dictionary= toBucket) break; - visit(bucket, KeyCodec.ReadValue(key, prefix.Length), iterator.CurrentValue); + var value = iterator.CurrentValue; + if (!value.IsRebuiltOnly) + visit(bucket, KeyCodec.ReadValue(key, prefix.Length), value); } } } diff --git a/src/ZoneTree.Analytics/Storage/RollupValue.cs b/src/ZoneTree.Analytics/Storage/RollupValue.cs index 0dd7aab..9c2aad2 100644 --- a/src/ZoneTree.Analytics/Storage/RollupValue.cs +++ b/src/ZoneTree.Analytics/Storage/RollupValue.cs @@ -343,6 +343,13 @@ public sealed class RollupValue readonly RollupSection distinct; + // Contributions of a history reconstruction that runs while ingestion writes the same row, + // kept apart until they are folded in. Null for every other row. + readonly RollupValue rebuilt; + + /// A row without data, holding only a rebuilt part. + static readonly RollupValue Empty = new(0, null, [], [], null, null); + /// Creates a stored row. Arrays and sketches are retained without copying. public RollupValue( long count, @@ -358,7 +365,8 @@ public RollupValue( valueNames, values, details == null ? null : new(DetailEntries.Of(details)), - distinctNames is { Length: > 0 } ? new(DistinctEntries.Of(distinctNames, distinct ?? [])) : null) + distinctNames is { Length: > 0 } ? new(DistinctEntries.Of(distinctNames, distinct ?? [])) : null, + (RollupValue)null) { } @@ -368,7 +376,8 @@ internal RollupValue( string[] valueNames, ValueStats[] values, RollupSection details, - RollupSection distinct) + RollupSection distinct, + RollupValue rebuilt) { Count = count; ValueNames = valueNames ?? []; @@ -376,8 +385,35 @@ internal RollupValue( this.visitors = visitors; this.details = details; this.distinct = distinct; + this.rebuilt = rebuilt; + } + + /// + /// The contributions a history reconstruction has made to this row so far; null when it has none. + /// Readers ignore them until merges them into the row. + /// + /// + /// Ingestion merges its deltas into the row and keeps this part as it is, and a reconstruction + /// replaces this part and keeps the rest: each can write its absolute value again after a + /// failure without undoing the other's writes. + /// + internal RollupValue Rebuilt => rebuilt; + + /// Whether the row holds nothing but a rebuilt part. + internal bool IsRebuiltOnly => rebuilt != null && Count == 0 && ValueNames.Length == 0 && + visitors == null && details == null && distinct == null; + + /// The row with its rebuilt part replaced; a missing row gets one without data. + internal static RollupValue WithRebuilt(RollupValue row, RollupValue part) + { + row ??= Empty; + return new(row.Count, row.visitors, row.ValueNames, row.Values, row.details, row.distinct, part); } + /// The row with its rebuilt part merged in; the row itself when it has none. + internal RollupValue Folded() => + rebuilt == null ? this : Merge(new RollupValue(Count, visitors, ValueNames, Values, details, distinct, null), rebuilt); + /// The visitor sketch, or null when the row has no visitors. public HyperLogLog Visitors => visitors?.Value; @@ -422,7 +458,8 @@ internal RollupValue( /// Merges two rows without modifying either row, its arrays or sketches. /// /// A section, detail or distinct sketch only one row has is shared, so its stored bytes are - /// written back as they are. + /// written back as they are. The result keeps the rebuilt part of ; + /// a delta has none. /// public static RollupValue Merge(RollupValue existing, RollupValue delta) { @@ -457,7 +494,7 @@ public static RollupValue Merge(RollupValue existing, RollupValue delta) var distinct = delta.DistinctSection == null ? existing.distinct : existing.DistinctSection == null ? delta.distinct : new RollupSection(MergeDistinct(existing, delta)); - return new RollupValue(existing.Count + delta.Count, visitors, names, values, details, distinct); + return new RollupValue(existing.Count + delta.Count, visitors, names, values, details, distinct, existing.rebuilt); } /// @@ -1001,15 +1038,17 @@ public RollupValue ToValue() /// /// /// -/// header(1: 0 = deleted, otherwise Present | HasVisitors | HasDetails | HasDistinct) +/// header(1: 0 = deleted, otherwise Present | HasVisitors | HasDetails | HasDistinct | HasRebuilt) /// count(varint) /// nameCount(varint) [name(string) sum(double) count(varint) [min(double) max(double) [mean(double) stddev(double)]]]* /// [length(varint) visitor sketch] when HasVisitors /// [length(varint) detailCount(varint) [length(varint) detail]*] when HasDetails; length 0 for none /// [length(varint) distinctCount(varint) [dimension(string) length(varint) sketch]*] when HasDistinct +/// [length(varint) rebuilt part] when HasRebuilt /// /// Min and max are written when count is not zero, moments when count exceeds one. Sections, /// details and distinct sketches are length-prefixed and decoded only when read, one at a time. +/// The rebuilt part is a value in this layout without a rebuilt part of its own. /// A reader rejects header bits it does not know and trailing bytes. /// sealed class RollupValueSerializer : ISerializer @@ -1024,7 +1063,11 @@ sealed class RollupValueSerializer : ISerializer const byte HasDistinct = 8; - const byte KnownBits = Present | HasVisitors | HasDetails | HasDistinct; + const byte HasRebuilt = 16; + + const byte Sections = HasVisitors | HasDetails | HasDistinct | HasRebuilt; + + const byte KnownBits = Present | Sections; delegate T SectionReader(ref ByteReader reader); @@ -1157,17 +1200,18 @@ public RollupValue Deserialize(Memory bytes) } } - static RollupValue Read(ReadOnlyMemory bytes) + /// Reads a value; reads the rebuilt part of a value. + static RollupValue Read(ReadOnlyMemory bytes, bool rebuiltPart = false) { var reader = new ByteReader(bytes.Span); var header = reader.ReadByte(); - if (header == Deleted) + if (header == Deleted && !rebuiltPart) { if (reader.Remaining != 0) throw new FormatException("A deleted rollup value has trailing bytes."); return null; } - if ((header & ~KnownBits) != 0 || (header & Present) == 0) + if ((header & ~KnownBits) != 0 || (header & Present) == 0 || (rebuiltPart && (header & HasRebuilt) != 0)) throw new FormatException("The rollup value has an unknown layout."); var count = reader.ReadVarInt64(); // Every value takes at least a name length, its sum and its count. @@ -1193,7 +1237,7 @@ static RollupValue Read(ReadOnlyMemory bytes) } values[i] = new ValueStats(sum, valueCount, min, max, mean, deviation); } - if ((header & (HasVisitors | HasDetails | HasDistinct)) == 0) + if ((header & Sections) == 0) { if (reader.Remaining != 0) throw new FormatException("The rollup value has trailing bytes."); @@ -1202,9 +1246,10 @@ static RollupValue Read(ReadOnlyMemory bytes) // The sections follow the scalar values. They keep their stored bytes for as long as the row // lives, and ZoneTree reads values from decompressed disk blocks of megabytes: a slice of a // block would keep the whole block alive in caches and in merged rows, so the sections of a - // value that does not own its buffer are copied into one buffer of their own. + // value that does not own its buffer are copied into one buffer of their own. A rebuilt part + // lies in the buffer of its value. var start = bytes.Length - reader.Remaining; - var sections = OwnsBuffer(bytes) ? bytes[start..] : bytes[start..].ToArray(); + var sections = rebuiltPart || OwnsBuffer(bytes) ? bytes[start..] : bytes[start..].ToArray(); var sectionReader = new ByteReader(sections.Span); RollupSection visitors = null; RollupSection details = null; @@ -1222,9 +1267,12 @@ static RollupValue Read(ReadOnlyMemory bytes) } if ((header & HasDistinct) != 0) distinct = new(ReadLengthPrefixed(ref sectionReader, sections, allowEmpty: false), DecodeDistinct); + RollupValue rebuilt = null; + if ((header & HasRebuilt) != 0) + rebuilt = Read(ReadLengthPrefixed(ref sectionReader, sections, allowEmpty: false), rebuiltPart: true); if (sectionReader.Remaining != 0) throw new FormatException("The rollup value has trailing bytes."); - return new RollupValue(count, visitors, names, values, details, distinct); + return new RollupValue(count, visitors, names, values, details, distinct, rebuilt); } /// @@ -1247,6 +1295,8 @@ public Memory Serialize(in RollupValue entry) { if (entry == null) return new byte[] { Deleted }; + // Encoded first: it uses the same thread-local writers. + var rebuilt = entry.Rebuilt == null ? default : Serialize(entry.Rebuilt); var writer = ByteWriter.Reuse(ref threadWriter, 256); var visitors = entry.VisitorSection; var details = entry.DetailSection; @@ -1258,6 +1308,8 @@ public Memory Serialize(in RollupValue entry) header |= HasDetails; if (distinct != null) header |= HasDistinct; + if (entry.Rebuilt != null) + header |= HasRebuilt; writer.WriteByte(header); writer.WriteVarInt64(entry.Count); writer.WriteVarUInt64((ulong)entry.ValueNames.Length); @@ -1306,6 +1358,11 @@ public Memory Serialize(in RollupValue entry) } }); } + if (entry.Rebuilt != null) + { + writer.WriteVarUInt64((ulong)rebuilt.Length); + writer.WriteBytes(rebuilt.Span); + } return writer.ToArray(); } diff --git a/src/ZoneTree.Analytics/Storage/RollupWriteBatch.cs b/src/ZoneTree.Analytics/Storage/RollupWriteBatch.cs index f467284..68f8cf6 100644 --- a/src/ZoneTree.Analytics/Storage/RollupWriteBatch.cs +++ b/src/ZoneTree.Analytics/Storage/RollupWriteBatch.cs @@ -40,15 +40,36 @@ public void Dispose() } } -/// One additive contribution and whether its replacement has been written. -sealed class RollupWrite(Memory key, RollupValue delta) +/// How a write changes its row. +enum RollupWriteKind +{ + /// Merges an additive delta of ingestion into the row. + Delta, + + /// Replaces the rebuilt part of the row with the absolute value of a reconstruction. + Rebuild, + + /// Merges the rebuilt part into the row, and copies the row to its indexes. + Fold +} + +/// One change of a row and whether its replacement has been written. +/// The rollup key. +/// The delta, or the rebuilt part a writes; null for a fold. +/// How the write changes the row. +sealed class RollupWrite(Memory key, RollupValue delta, RollupWriteKind kind = RollupWriteKind.Delta) { public readonly Memory Key = key; public readonly RollupValue Delta = delta; + public readonly RollupWriteKind Kind = kind; + public RollupValue Replacement; + // A fold of a row without a rebuilt part only copies the row. + public bool Unchanged; + public bool Completed; } @@ -60,6 +81,11 @@ sealed class RollupWrite(Memory key, RollupValue delta) /// further delta. This also lets another worker finish the pending write without applying it /// twice. /// +/// +/// Background index builds and history reconstructions write through the same stripes, so they +/// read the current value of a row and never overwrite a concurrent flush. Only ingestion records +/// the days it writes for index recovery: background writes advance their own checkpoints. +/// sealed class RollupWriter(IZoneTree, RollupValue> tree, Action, RollupValue> copy = null, Action> beforeWrite = null) { @@ -102,7 +128,19 @@ public void Write(RollupWrite row) if (row.Completed) return; tree.TryGet(row.Key, out var existing); - row.Replacement = RollupValue.Merge(existing, row.Delta); + row.Replacement = row.Kind switch + { + RollupWriteKind.Delta => RollupValue.Merge(existing, row.Delta), + RollupWriteKind.Rebuild => RollupValue.WithRebuilt(existing, row.Delta), + _ => existing?.Folded() + }; + if (row.Replacement == null) + { + // A fold of a row that no longer exists. + row.Completed = true; + return; + } + row.Unchanged = ReferenceEquals(row.Replacement, existing); stripe.Pending = row; CompletePending(stripe); } @@ -115,10 +153,14 @@ void CompletePending(Stripe stripe) return; // Every writer of this row uses the same stripe. An upsert retry cannot overwrite another // worker's contribution, and repeating it cannot add the delta a second time. The index - // copy runs under the same lock, so the copies of a row are written in order. - beforeWrite?.Invoke(pending.Key); - tree.Upsert(pending.Key, pending.Replacement); - copy?.Invoke(pending.Key, pending.Replacement); + // copy runs under the same lock, so the copies of a row are written in order. A rebuilt part + // is invisible to readers: it reaches the indexes when it is folded in. + if (pending.Kind == RollupWriteKind.Delta) + beforeWrite?.Invoke(pending.Key); + if (!pending.Unchanged) + tree.Upsert(pending.Key, pending.Replacement); + if (pending.Kind != RollupWriteKind.Rebuild) + copy?.Invoke(pending.Key, pending.Replacement); pending.Completed = true; stripe.Pending = null; } diff --git a/src/ZoneTree.Analytics/Storage/StartupLog.cs b/src/ZoneTree.Analytics/Storage/StageLog.cs similarity index 66% rename from src/ZoneTree.Analytics/Storage/StartupLog.cs rename to src/ZoneTree.Analytics/Storage/StageLog.cs index e85484a..20c9b23 100644 --- a/src/ZoneTree.Analytics/Storage/StartupLog.cs +++ b/src/ZoneTree.Analytics/Storage/StageLog.cs @@ -3,8 +3,11 @@ namespace ZoneTree.Analytics.Storage; -/// Times startup stages without adding timers or logging to normal ingestion. -sealed class StartupLog : IDisposable +/// +/// Logs and times a stage of opening the engine, of background index maintenance or of +/// shutdown, without adding timers or logging to normal ingestion. +/// +sealed class StageLog : IDisposable { readonly ILogger logger; readonly string stage; @@ -12,12 +15,12 @@ sealed class StartupLog : IDisposable readonly long started = Stopwatch.GetTimestamp(); bool completed; - public StartupLog(ILogger logger, string stage, string directory) + public StageLog(ILogger logger, string stage, string directory, FormattableString metrics = null) { this.logger = logger; this.stage = stage; this.directory = directory; - Write("started"); + Write("started", metrics); } public void Progress(FormattableString metrics, long units, string unit) => @@ -47,3 +50,14 @@ public void Dispose() if (!completed) Write("failed"); } } + +/// Logs and times the disposal of a resource as a stage of shutdown. +sealed class StageDisposal(IDisposable resource, ILogger logger, string stage, string directory) : IDisposable +{ + public void Dispose() + { + using var log = new StageLog(logger, stage, directory); + resource.Dispose(); + log.Complete(null); + } +} diff --git a/yarn.lock b/yarn.lock index fb84e9c..5ece0e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -413,95 +413,98 @@ __metadata: languageName: node linkType: hard -"@purestack/ts-common@npm:^1.1.11, @purestack/ts-common@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-common@npm:1.1.12" +"@purestack/ts-common@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-common@npm:1.1.14" dependencies: - "@purestack/ts-html": "npm:^1.1.12" - "@purestack/ts-style": "npm:^1.1.12" - "@purestack/ts-util": "npm:^1.1.12" - checksum: 10c0/ecfeee66953264684c764d2dd884508f88024a20c16d164f5e2218ff5d94a7e72fdcfe29a238778aba93e222c9f5b3b4d2a2f2e6eb2f5cf6afe3ab9b04303082 + "@purestack/ts-html": "npm:^1.1.14" + "@purestack/ts-style": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.14" + checksum: 10c0/132802ad798a322a2b29ad3a3b5cbec6c22efe37e636e34656d0fa434de4323f90f2afc99d1996f112885e639cfecc233639850f1007edce60538b9d88506c19 languageName: node linkType: hard -"@purestack/ts-components@npm:^1.1.11, @purestack/ts-components@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-components@npm:1.1.12" +"@purestack/ts-components@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-components@npm:1.1.14" dependencies: - "@purestack/ts-common": "npm:^1.1.12" - "@purestack/ts-css": "npm:^1.1.12" - "@purestack/ts-page-scripts": "npm:^1.1.12" - "@purestack/ts-style": "npm:^1.1.12" - "@purestack/ts-util": "npm:^1.1.12" + "@purestack/ts-common": "npm:^1.1.14" + "@purestack/ts-css": "npm:^1.1.14" + "@purestack/ts-page-scripts": "npm:^1.1.14" + "@purestack/ts-style": "npm:^1.1.14" + "@purestack/ts-svg-icons": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.14" regor: "npm:^1.7.5" - checksum: 10c0/e970424c433d7e071fd420935de0a9b932c3f904141849e10141bb66767b730a7d4c1a2241ef4023dd3a0d021b545aae97b71e863e8518ab33a541bda8e9f691 + checksum: 10c0/865304a912caafe03e56b9ee2356928cdc83e43f48ea7961daf5c224c56eaf2f5419ed3ada05b1febf673f559dd2ce624280c05ff68b3120322f4cef585a72b7 languageName: node linkType: hard -"@purestack/ts-css@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-css@npm:1.1.12" +"@purestack/ts-css@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-css@npm:1.1.14" dependencies: - "@purestack/ts-util": "npm:^1.1.12" + "@purestack/ts-util": "npm:^1.1.14" prettier: "npm:^3.8.1" - checksum: 10c0/40f1712837e11b0e7a3f087f061f259548c98bd425206ae2d92fcacf727c0c355bbce3f00a9c4f24cb2b0cc9e0776fa4a1f1e140abf2e21385b8b86f946c2160 + checksum: 10c0/4996f71f9758dcd96a060e19a5c6792d258212d6d725cf0733cc4f271780d260ae2c16fb5aca21813bd02954351ab38d64320fa9410ed2ef86f9e1455e2dd48d languageName: node linkType: hard -"@purestack/ts-html@npm:^1.1.11, @purestack/ts-html@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-html@npm:1.1.12" +"@purestack/ts-html@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-html@npm:1.1.14" dependencies: - "@purestack/ts-util": "npm:^1.1.12" + "@purestack/ts-util": "npm:^1.1.14" prettier: "npm:^3.8.1" - checksum: 10c0/f46786738a712c4f2b6eff2bca76652730b9d142d048ef337479eb576f71e0ba0a252ce2aa716943e73b4e122bcb5d818be5fbbdbd743ae5ae834eea9eda9661 + checksum: 10c0/5f07904733f477238fedb8ec72d2141efcb1fef62870ff861190d03cf52ee07b6aa060f3dadc0a95e3177c4b28819166fb79464369e0cf80b7815faa7e316194 languageName: node linkType: hard -"@purestack/ts-minidom@npm:^1.1.11": - version: 1.1.11 - resolution: "@purestack/ts-minidom@npm:1.1.11" - checksum: 10c0/e247f06b0f83bd97754c10c736ee9e255a80e3059c4f5b81752828886895574169a3f6139c9cd5bb10152f9e065551c30313f25f72ddd984c4c57bacb6d822ab +"@purestack/ts-minidom@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-minidom@npm:1.1.14" + checksum: 10c0/8ba2840474dd173a22d28f93a108cf5d2adbef0fd1cbed263d82ac1785879924ec77d0b0f7d4a0ed1a1c3de3b5f994242c257ddb0e74eb824e6038e340074f2e languageName: node linkType: hard -"@purestack/ts-page-scripts@npm:^1.1.11, @purestack/ts-page-scripts@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-page-scripts@npm:1.1.12" +"@purestack/ts-page-scripts@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-page-scripts@npm:1.1.14" dependencies: - "@purestack/ts-common": "npm:^1.1.12" - checksum: 10c0/14ebb9544961112e4d789e285795e829d321f6ee7b6aa087d6e26ffc7f91ef8fb774df5c701d242978bd7a85bd11eb166595d656082022da3233cecc1da4f6d6 + "@purestack/ts-common": "npm:^1.1.14" + "@purestack/ts-html": "npm:^1.1.14" + checksum: 10c0/c872f719771b74767430705f55b337b1045bf93d0712cb9e80ba9e9d93dd19b5f20cd2d4a8ec70c3f69df042a72e0b5d13709e73ffd4f49d926d2823d0df3330 languageName: node linkType: hard -"@purestack/ts-render@npm:^1.1.11": - version: 1.1.11 - resolution: "@purestack/ts-render@npm:1.1.11" +"@purestack/ts-render@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-render@npm:1.1.14" dependencies: - "@purestack/ts-common": "npm:^1.1.11" - "@purestack/ts-minidom": "npm:^1.1.11" - "@purestack/ts-page-scripts": "npm:^1.1.11" - "@purestack/ts-util": "npm:^1.1.11" + "@purestack/ts-common": "npm:^1.1.14" + "@purestack/ts-minidom": "npm:^1.1.14" + "@purestack/ts-page-scripts": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.14" regor: "npm:^1.7.5" - checksum: 10c0/e66dbf2450f99c6c83a60cf34a692641eb34482d7fd1b7ec8b7cfc98b30449b29653c741a94569e91ffabba79977c5e19d565676a5f15c26073217435499f3b8 + checksum: 10c0/bc860f38827f3ce4ed0cbfa796ecbb5e9a6e97f4b31ddf24909b1d7bf5af6419bc9ca31e691a386551a0878fc40a4cfc114d28b1e40689d8a1c7ca7a2e82debb languageName: node linkType: hard -"@purestack/ts-ssg@npm:^1.1.11": - version: 1.1.11 - resolution: "@purestack/ts-ssg@npm:1.1.11" +"@purestack/ts-ssg@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-ssg@npm:1.1.14" dependencies: "@11ty/gray-matter": "npm:^3.0.0" - "@purestack/ts-common": "npm:^1.1.11" - "@purestack/ts-components": "npm:^1.1.11" - "@purestack/ts-html": "npm:^1.1.11" - "@purestack/ts-minidom": "npm:^1.1.11" - "@purestack/ts-page-scripts": "npm:^1.1.11" - "@purestack/ts-render": "npm:^1.1.11" - "@purestack/ts-style": "npm:^1.1.11" - "@purestack/ts-svg-icons": "npm:^1.1.11" - "@purestack/ts-util": "npm:^1.1.11" - "@purestack/ts-util-node": "npm:^1.1.11" + "@purestack/ts-common": "npm:^1.1.14" + "@purestack/ts-components": "npm:^1.1.14" + "@purestack/ts-html": "npm:^1.1.14" + "@purestack/ts-minidom": "npm:^1.1.14" + "@purestack/ts-page-scripts": "npm:^1.1.14" + "@purestack/ts-render": "npm:^1.1.14" + "@purestack/ts-style": "npm:^1.1.14" + "@purestack/ts-svg-icons": "npm:^1.1.14" + "@purestack/ts-theme": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.14" + "@purestack/ts-util-node": "npm:^1.1.14" esbuild: "npm:^0.28.0" hast-util-from-html: "npm:^2.0.3" hast-util-to-html: "npm:^9.0.5" @@ -520,38 +523,53 @@ __metadata: unified: "npm:^11.0.5" bin: ts-ssg: dist/cli.mjs - checksum: 10c0/ef1961171d1dfe1bdd9fc7bfda83305af93a1022e564b360055b1cbe935712683db01ceabc471b3c7be6cfc221c026decd1e04aa95d0c4439409ff042695cda9 + checksum: 10c0/08591fe129303336698d8e7927bc2a29ca983e46d6d6f2bfc879345054d03bf91ad24d88de0d733841760940bbf2010e62b9fe95fa8322362638b787b5fd064d languageName: node linkType: hard -"@purestack/ts-style@npm:^1.1.11, @purestack/ts-style@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-style@npm:1.1.12" +"@purestack/ts-style@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-style@npm:1.1.14" dependencies: - "@purestack/ts-css": "npm:^1.1.12" - "@purestack/ts-util": "npm:^1.1.12" - checksum: 10c0/c04cae88441239f721a5961a8c2ca81c5a18892f6011aa2ceefbe60f5474c7e2968914dea9622d05a451ce9d5da68cfad42f94a82b33827f43e8c9ba831dc0a5 + "@purestack/ts-css": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.14" + checksum: 10c0/c7a8ef003938e905ad6f282696c83d65af1c00348aabd9987df95b515a1fa17306d29e11fbc508f397d2e913fb9873be2f8b375606c8e0f1142b419a16f812f0 languageName: node linkType: hard -"@purestack/ts-svg-icons@npm:^1.1.11, @purestack/ts-svg-icons@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-svg-icons@npm:1.1.12" - checksum: 10c0/8822b8ceb3ed45e2d1b4f573414b811b0f97fbc17bc8b1dcfa52195c8422dd3f3682a0042e5063cda78e0a6a2a20a8d018fb1d4342adaa7f0b8812c4edd654de +"@purestack/ts-svg-icons@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-svg-icons@npm:1.1.14" + checksum: 10c0/e8d5c730e8ad93fd570592a2d7c5635144054459b02032ea35254a2dd067ec1350f3cee3eb519d8741d74a808057893c020c2f31658cbc5bfbbd00a265e0b9d6 languageName: node linkType: hard -"@purestack/ts-util-node@npm:^1.1.11": - version: 1.1.11 - resolution: "@purestack/ts-util-node@npm:1.1.11" - checksum: 10c0/e1c1e71558f4418e2b097676d963c028fe9a9df3904d7d9fecc94165f6ae81aaa0a51668bd01d178cd6fc2deeb8b31d997212046422c75f36ff63542e321d308 +"@purestack/ts-theme@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-theme@npm:1.1.14" + dependencies: + "@purestack/ts-common": "npm:^1.1.14" + "@purestack/ts-html": "npm:^1.1.14" + "@purestack/ts-page-scripts": "npm:^1.1.14" + "@purestack/ts-style": "npm:^1.1.14" + "@purestack/ts-util-node": "npm:^1.1.14" + highlight.js: "npm:^11.11.1" + logpot: "npm:^1.0.7" + checksum: 10c0/925281f830a7f3fd58d79fb544312a2a55e505f4b91473212ee930278224bf7b7ec3fa3ac7769a1e8ae4db00e19e2bbb265af1f140f22afd455c12ae9e20428f + languageName: node + linkType: hard + +"@purestack/ts-util-node@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-util-node@npm:1.1.14" + checksum: 10c0/4e736e5992cf76c433e8b4b8acc63bb862a2d6f5601351a7eec7e4f693b5329d9a655e8fe14097ec548f068701de55a401ff6fb13ed89e771158991d17b5811e languageName: node linkType: hard -"@purestack/ts-util@npm:^1.1.11, @purestack/ts-util@npm:^1.1.12": - version: 1.1.12 - resolution: "@purestack/ts-util@npm:1.1.12" - checksum: 10c0/2dc09f68747acb9c285433d952ef52b677fa928dac3c2ec0f922f73da68e427da2c378f5c5fc86fc3d0916c11bbeedddba3168aa1177bbdd9d97b260b2132e84 +"@purestack/ts-util@npm:^1.1.14": + version: 1.1.14 + resolution: "@purestack/ts-util@npm:1.1.14" + checksum: 10c0/f7a630ab7da8b2e65638fb96cd594c6fbcc0acb17a957b144312826b6d9a184b650f84573a6faecb4b171b71340f0013f60f151e6694b18ecdc3efc606aa0a90 languageName: node linkType: hard @@ -1010,9 +1028,9 @@ __metadata: version: 0.0.0-use.local resolution: "analytics.zonetree.dev@workspace:docs-site" dependencies: - "@purestack/ts-style": "npm:^1.1.12" - "@purestack/ts-util": "npm:^1.1.12" - purestack: "npm:^1.1.11" + "@purestack/ts-style": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.14" + purestack: "npm:^1.1.14" regor: "npm:^1.7.5" typescript: "npm:^7.0.2" languageName: unknown @@ -2520,14 +2538,14 @@ __metadata: languageName: node linkType: hard -"purestack@npm:^1.1.11": - version: 1.1.11 - resolution: "purestack@npm:1.1.11" +"purestack@npm:^1.1.14": + version: 1.1.14 + resolution: "purestack@npm:1.1.14" dependencies: - "@purestack/ts-ssg": "npm:^1.1.11" + "@purestack/ts-ssg": "npm:^1.1.14" bin: purestack: dist/cli.mjs - checksum: 10c0/8c60c2568939240702cddb8ae9fe5a04d4eb099c4ef101c2d21fa550ede7229f68dd3c361f15ac2c01717a98795caf6347a57fd8352bd5c14a10b2a6207acf81 + checksum: 10c0/244f87272518a446f17e09cc9bdd6bd099125cc3da1564e5447d71e1ad91d8192927d010620b817d97ceb3cd9c541714817619c66916cb1df32b4cb7e386e74d languageName: node linkType: hard @@ -3314,15 +3332,15 @@ __metadata: version: 0.0.0-use.local resolution: "zonetree-analytics-dashboard@workspace:apps/ZoneTree.Analytics.Dashboard/ui" dependencies: - "@purestack/ts-components": "npm:^1.1.12" - "@purestack/ts-css": "npm:^1.1.12" - "@purestack/ts-html": "npm:^1.1.12" - "@purestack/ts-style": "npm:^1.1.12" - "@purestack/ts-svg-icons": "npm:^1.1.12" - "@purestack/ts-util": "npm:^1.1.12" + "@purestack/ts-components": "npm:^1.1.14" + "@purestack/ts-css": "npm:^1.1.14" + "@purestack/ts-html": "npm:^1.1.14" + "@purestack/ts-style": "npm:^1.1.14" + "@purestack/ts-svg-icons": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.14" "@types/node": "npm:^22.0.0" playwright: "npm:^1.56.1" - purestack: "npm:^1.1.11" + purestack: "npm:^1.1.14" regor: "npm:^1.7.5" tsx: "npm:^4.20.6" typescript: "npm:^7.0.2"