From 15a15ed10dc9bb1314b5233e938e6f16ad2783d2 Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Sat, 17 Oct 2026 12:54:25 +0200 Subject: [PATCH 1/2] Simplifies the storage internals, refuses data directories it cannot load safely, and refreshes the benchmark numbers from Release builds. --- README.md | 6 +- .../server/BenchmarkReportStore.cs | 2 +- .../server/DashboardAnalyticsOptions.cs | 14 +- .../server/Program.cs | 3 +- .../server/SampleBenchmark.cs | 11 +- .../server/SampleDataJobs.cs | 2 +- .../ZoneTree.Analytics.Dashboard/ui/README.md | 6 +- .../ui/content/sample-data/benchmark.ts | 10 + .../ui/content/sample-data/jobs.ts | 2 + .../ui/package.json | 16 +- .../ui/tests/sample-data-browser.ts | 25 +- benchmarks/measure-overview.js | 49 + benchmarks/query-benchmark.ts | 227 ++++ .../performance/benchmarking/benchmarking.mdx | 2 +- docs-site/content/docs/performance/index.mdx | 2 +- .../query-benchmarks/query-benchmarks.mdx | 104 +- .../performance/query-benchmarks/results.json | 998 +++++++++--------- .../secondary-indexes/secondary-indexes.mdx | 2 +- docs-site/content/index.mdx | 8 +- docs-site/package.json | 8 +- internal-docs/storage-format.md | 28 +- package.json | 5 +- scripts/tsconfig.json | 2 +- src/Directory.Build.props | 2 +- .../BackfillLoggingTests.cs | 2 +- .../BackgroundIndexingTests.cs | 17 + .../DashboardAnalyticsOptionsTests.cs | 6 +- .../FlushRecoveryTests.cs | 2 + .../MetaStoreTests.cs | 12 +- .../ReleaseRegressionTests.cs | 2 +- .../SampleBenchmarkTests.cs | 24 +- .../StorageFormatRegressionTests.cs | 15 + src/ZoneTree.Analytics/AnalyticsEngine.cs | 17 +- src/ZoneTree.Analytics/AnalyticsStorage.cs | 8 +- .../Ingestion/IngestionWorker.cs | 22 +- .../Ingestion/RollupBuffer.cs | 11 + src/ZoneTree.Analytics/Internal/ByteWriter.cs | 6 +- src/ZoneTree.Analytics/Internal/Once.cs | 18 + .../Internal/RetentionCutoffs.cs | 10 + src/ZoneTree.Analytics/Internal/Text.cs | 3 +- .../Storage/AlternatingFiles.cs | 135 +++ .../Storage/BackgroundIndex.cs | 255 +++++ .../Storage/DimensionIndexScan.cs | 24 +- .../Storage/DimensionIndexStore.cs | 235 ++--- .../Storage/JointBackfillCheckpoint.cs | 120 +-- .../Storage/JointRollupReconstruction.cs | 4 +- src/ZoneTree.Analytics/Storage/MetaStore.cs | 113 +- .../Storage/RawEventStore.cs | 5 +- .../Storage/RollupIndexStore.cs | 230 ++-- src/ZoneTree.Analytics/Storage/RollupStore.cs | 3 +- src/ZoneTree.Analytics/Storage/RollupValue.cs | 36 +- .../Storage/StorageKeyCodec.cs | 4 +- .../Web/BuiltInUserAgentParser.cs | 23 +- .../Web/UserAgentBotDetector.cs | 26 +- src/ZoneTree.Analytics/Web/UserAgentCache.cs | 33 + yarn.lock | 244 +++-- 56 files changed, 1808 insertions(+), 1391 deletions(-) create mode 100644 benchmarks/measure-overview.js create mode 100644 benchmarks/query-benchmark.ts create mode 100644 src/ZoneTree.Analytics/Internal/Once.cs create mode 100644 src/ZoneTree.Analytics/Storage/AlternatingFiles.cs create mode 100644 src/ZoneTree.Analytics/Storage/BackgroundIndex.cs create mode 100644 src/ZoneTree.Analytics/Web/UserAgentCache.cs diff --git a/README.md b/README.md index 2a54a2a..b9e2f9f 100644 --- a/README.md +++ b/README.md @@ -196,10 +196,10 @@ The repository includes a [documented benchmark][benchmarks] over **5 million ge | Workload | Median | p95 | | --- | ---: | ---: | -| Complete overview: six reports, up to three requests in flight | **84.0 ms** | 87.0 ms | -| Country breakdown filtered to GitHub traffic | **13.5 ms** | 14.0 ms | +| Complete overview: six reports, up to three requests in flight | **33.3 ms** | 35.5 ms | +| Country breakdown filtered to GitHub traffic | **5.7 ms** | 6.5 ms | -Measured over 30 warm loads per scope on an Intel Core Ultra 7 265KF with 64 GiB RAM, Windows x64, and a .NET 10.0.6 Debug host. Times include loopback HTTP response transfer and JSON parsing; chart rendering is excluded. The dataset has low group cardinality, and filtered reports use configured covering rollup indexes. These figures describe that workload, rather than a latency guarantee. +Measured over 30 warm loads per scope on an Intel Core Ultra 7 265KF with 64 GiB RAM, Windows x64, and a .NET 10.0.6 Release host. Times include loopback HTTP response transfer and JSON parsing; chart rendering is excluded. The dataset has low group cardinality, and filtered reports use configured covering rollup indexes. These figures describe that workload, rather than a latency guarantee. Matching rollups answer recurring reports from aggregates. Segments, funnels, and unsupported aggregate combinations can require event scans. The [benchmark report][benchmarks] includes configuration, query plans, individual timings, and methodology; the [benchmarking guide][benchmarking] explains how to measure your workload. diff --git a/apps/ZoneTree.Analytics.Dashboard/server/BenchmarkReportStore.cs b/apps/ZoneTree.Analytics.Dashboard/server/BenchmarkReportStore.cs index 7b52f2f..143a0c8 100644 --- a/apps/ZoneTree.Analytics.Dashboard/server/BenchmarkReportStore.cs +++ b/apps/ZoneTree.Analytics.Dashboard/server/BenchmarkReportStore.cs @@ -52,7 +52,7 @@ public SampleEngineLease Acquire() if (current == null) throw new BenchmarkDataUnavailableException(); var path = Path.Combine(root, current.Directory); ValidateOwned(path); - engine ??= new AnalyticsEngine(DashboardAnalyticsOptions.CreateBenchmarkReport(path, current.TimeZone, current.Result.Percentiles, logger)); + engine ??= new AnalyticsEngine(DashboardAnalyticsOptions.CreateBenchmarkReport(path, current.TimeZone, current.Result.Percentiles, current.Result.Indexes, logger)); if (readers++ == 0) drained = new(TaskCreationOptions.RunContinuationsAsynchronously); return new(engine, () => { lock (sync) { if (--readers == 0) drained.TrySetResult(); } }); } diff --git a/apps/ZoneTree.Analytics.Dashboard/server/DashboardAnalyticsOptions.cs b/apps/ZoneTree.Analytics.Dashboard/server/DashboardAnalyticsOptions.cs index 2186517..569b3be 100644 --- a/apps/ZoneTree.Analytics.Dashboard/server/DashboardAnalyticsOptions.cs +++ b/apps/ZoneTree.Analytics.Dashboard/server/DashboardAnalyticsOptions.cs @@ -39,20 +39,26 @@ public static AnalyticsOptions CreatePerformanceSamples(string directory, TimeZo return options; } + // Without indexes, ingestion writes only raw events, sessions and rollups. public static AnalyticsOptions CreateBenchmark(string directory, TimeZoneInfo timeZone, - int workers, bool percentiles, ILogger logger = null) + int workers, bool percentiles, bool indexes, ILogger logger = null) { var options = Create(directory, BenchmarkSite, timeZone, logger); SetWorkers(options, workers); options.Ingestion.QueueFullBehavior = QueueFullBehavior.Wait; if (percentiles) options.Rollups.ValueAggregates["load_time"] = ValueAggregates.Percentiles; + if (!indexes) + { + options.Storage.DimensionIndexes.Dimensions.Clear(); + options.Storage.RollupIndexes.Definitions.Clear(); + } return options; } - // Reopening uses the saved dataset time zone and percentile setting, with one worker. + // Reopening uses the saved dataset time zone, percentile and index settings, with one worker. public static AnalyticsOptions CreateBenchmarkReport(string directory, string timeZoneId, - bool percentiles, ILogger logger = null) => - CreateBenchmark(directory, TimeZoneInfo.FindSystemTimeZoneById(timeZoneId), 1, percentiles, logger); + bool percentiles, bool indexes, ILogger logger = null) => + CreateBenchmark(directory, TimeZoneInfo.FindSystemTimeZoneById(timeZoneId), 1, percentiles, indexes, logger); // Generated ingestion runs as many rollup writers as workers, so the rows that every worker // buffers are merged and written in parallel at the same scale. diff --git a/apps/ZoneTree.Analytics.Dashboard/server/Program.cs b/apps/ZoneTree.Analytics.Dashboard/server/Program.cs index 81197c9..12950da 100644 --- a/apps/ZoneTree.Analytics.Dashboard/server/Program.cs +++ b/apps/ZoneTree.Analytics.Dashboard/server/Program.cs @@ -39,9 +39,10 @@ builder.Configuration["Analytics:DataDirectory"] ?? DashboardAnalyticsOptions.DefaultDataDirectory, timeZone)); var app = builder.Build(); +// Open the engine now: data it cannot load fails startup instead of every request. +var engine = app.Services.GetRequiredService(); if (builder.Configuration.GetValue("Analytics:SeedDemo")) { - var engine = app.Services.GetRequiredService(); if (engine.GetEventNames(DateRange.LastDays(60)).Count == 0) await SampleTraffic.SeedAsync(engine); } diff --git a/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs b/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs index 278bebb..e309fed 100644 --- a/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs +++ b/apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs @@ -4,13 +4,14 @@ using Microsoft.Extensions.Logging; using ZoneTree.Analytics; -sealed record SampleBenchmarkRequest(int Events = 5_000_000, int Days = 365, int Workers = 20, bool Percentiles = false); +sealed record SampleBenchmarkRequest(int Events = 5_000_000, int Days = 365, int Workers = 20, bool Percentiles = false, + bool Indexes = true); sealed record SampleBenchmarkResult(int Events, int Days, int Workers, bool Percentiles, double PreparationMs, double SubmissionMs, double IngestionMs, double EventsPerSecond, long Ingested, long Errors, long Dropped, long Rejected, string Build, string GarbageCollection, string TieredCompilation, - string FromDate = "", string ToDate = ""); + string FromDate = "", string ToDate = "", bool Indexes = true); sealed record SampleBenchmarkStatus(string State = "idle", int Requested = 0, int Prepared = 0, int Submitted = 0, - int Workers = 20, int Days = 365, bool Percentiles = false, string Error = null, SampleBenchmarkResult Result = null, + int Workers = 20, int Days = 365, bool Percentiles = false, bool Indexes = true, string Error = null, SampleBenchmarkResult Result = null, long Ingested = 0, double ElapsedMs = 0, double EventsPerSecond = 0); sealed record SampleBenchmarkProgress(string State, int Prepared = 0, int Submitted = 0, long Ingested = 0, double ElapsedMs = 0, double EventsPerSecond = 0); @@ -59,7 +60,7 @@ public async Task RunAsync(SampleBenchmarkRequest request async Task MeasureAsync(string directory, AnalyticsEvent[] events, SampleBenchmarkRequest request, DateTimeOffset end, double preparationMs, Action progress, CancellationToken cancellationToken, CancellationToken applicationStopping) { - var options = DashboardAnalyticsOptions.CreateBenchmark(directory, timeZone, request.Workers, request.Percentiles, logger); + var options = DashboardAnalyticsOptions.CreateBenchmark(directory, timeZone, request.Workers, request.Percentiles, request.Indexes, logger); await using var engine = new AnalyticsEngine(options); progress(new("running", Prepared: events.Length)); var stopwatch = Stopwatch.StartNew(); @@ -107,7 +108,7 @@ async Task MeasureAsync(string directory, AnalyticsEvent[ statistics.EventsProcessed, statistics.Errors, statistics.EventsDropped, statistics.EventsRejected, build, GCSettings.IsServerGC ? "Server" : "Workstation", tiered == "0" ? "Disabled" : tiered == "1" ? "Enabled" : "Default", TimeZoneInfo.ConvertTime(end.AddDays(-request.Days), timeZone).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), - TimeZoneInfo.ConvertTime(end, timeZone).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + TimeZoneInfo.ConvertTime(end, timeZone).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), request.Indexes); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/apps/ZoneTree.Analytics.Dashboard/server/SampleDataJobs.cs b/apps/ZoneTree.Analytics.Dashboard/server/SampleDataJobs.cs index 20697a0..f32c1c3 100644 --- a/apps/ZoneTree.Analytics.Dashboard/server/SampleDataJobs.cs +++ b/apps/ZoneTree.Analytics.Dashboard/server/SampleDataJobs.cs @@ -67,7 +67,7 @@ public bool Benchmark(SampleBenchmarkRequest request) var token = cancellation.Token; activeOperation = "benchmark"; benchmarkStatus = new("preparing", request.Events, Workers: request.Workers, Days: request.Days, - Percentiles: request.Percentiles, Result: benchmarkStatus.Result); + Percentiles: request.Percentiles, Indexes: request.Indexes, Result: benchmarkStatus.Result); operation = Task.Run(async () => { try diff --git a/apps/ZoneTree.Analytics.Dashboard/ui/README.md b/apps/ZoneTree.Analytics.Dashboard/ui/README.md index 6daa3ee..5da74d7 100644 --- a/apps/ZoneTree.Analytics.Dashboard/ui/README.md +++ b/apps/ZoneTree.Analytics.Dashboard/ui/README.md @@ -44,7 +44,7 @@ Edit `ConfigureCommon` for retention, storage, ingestion, queries, or rollups sh traffic, performance samples, new benchmarks and reopened benchmark reports. Raw events are retained for three years in all datasets. `ConfigurePrimary`, `CreatePerformanceSamples`, `CreateBenchmark` and `CreateBenchmarkReport` contain the per-engine settings; sample and -benchmark worker counts come from the dashboard, and the benchmark percentile setting and +benchmark worker counts come from the dashboard, and the benchmark percentile and index settings and reporting time zone are preserved when reopening a completed dataset. Each engine receives independent options and collections. Restart the host to apply edited engine settings. @@ -56,7 +56,7 @@ Select **View sample reports**, or choose **Performance samples** in a report's **Delete sample data** closes the sample engine and removes its events, rollups and metadata, then permits a fresh dataset. Primary traffic and `Analytics:DataDirectory` are preserved. Sample controls and endpoints are available only in Development mode, including when `Dashboard:Watch=false`; production hosts do not expose them. -**Benchmark ingestion** prepares the same realistic traffic in memory before timing, then feeds it to a fresh engine with the default per-worker limits. **Include load-time percentile aggregation** controls only percentile rollups; event values and traffic shape are retained. Progress polls lightweight engine counters every 250 ms and shows ingested events, queued events, elapsed time and average throughput so far. The progress bar follows processed events, then shows a separate saving-aggregates stage while the final flush finishes. The result separates preparation, submission, and ingestion-through-flush time. Throughput uses successfully processed events divided by the submission-through-flush interval, with preparation, store creation, dataset publication, disposal and cleanup excluded. +**Benchmark ingestion** prepares the same realistic traffic in memory before timing, then feeds it to a fresh engine with the default per-worker limits. **Include load-time percentile aggregation** controls only percentile rollups; event values and traffic shape are retained. Clear **Maintain dimension and rollup indexes** to measure ingestion without the dashboard's secondary indexes; that run's reports read rollups and raw events directly. Progress polls lightweight engine counters every 250 ms and shows ingested events, queued events, elapsed time and average throughput so far. The progress bar follows processed events, then shows a separate saving-aggregates stage while the final flush finishes. The result separates preparation, submission, and ingestion-through-flush time. Throughput uses successfully processed events divided by the submission-through-flush interval, with preparation, store creation, dataset publication, disposal and cleanup excluded. Completed sample generations and benchmarks compact their stores before reporting completion, moving remaining in-memory records into disk segments and draining their WALs. The dashboard @@ -72,7 +72,7 @@ $env:DOTNET_TieredCompilation = '0' dotnet run -c Release --project apps/ZoneTree.Analytics.Dashboard/server -- --Dashboard:Watch=false ``` -The dashboard server exposes `GET /_analytics/sample-data/status` and JSON `POST` requests to `/generate` (`{"events":1000000,"days":60,"workers":8}`), `/benchmark` (`{"events":5000000,"days":90,"workers":20,"percentiles":false}`), `/cancel` (`{}`), `/clear` (`{}`), and `/clear-benchmark` (`{}`). Generation workers default to 1; benchmark defaults are described above. Status includes the active operation, generation progress, benchmark preparation/ingestion progress, and the previous completed benchmark result. Its config response advertises both report sites and their endpoints; other sites continue using the configured reporting API. +The dashboard server exposes `GET /_analytics/sample-data/status` and JSON `POST` requests to `/generate` (`{"events":1000000,"days":60,"workers":8}`), `/benchmark` (`{"events":5000000,"days":90,"workers":20,"percentiles":false,"indexes":true}`), `/cancel` (`{}`), `/clear` (`{}`), and `/clear-benchmark` (`{}`). Generation workers default to 1; benchmark defaults are described above. Status includes the active operation, generation progress, benchmark preparation/ingestion progress, and the previous completed benchmark result. Its config response advertises both report sites and their endpoints; other sites continue using the configured reporting API. ## Build and host diff --git a/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/benchmark.ts b/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/benchmark.ts index 4cc7dd5..e2eee7c 100644 --- a/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/benchmark.ts +++ b/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/benchmark.ts @@ -17,6 +17,7 @@ export interface SampleBenchmarkPanel { days: SRef workers: SRef percentiles: SRef + indexes: SRef benchmark: ComputedRef run: (event: Event) => Promise seconds: (milliseconds: number) => string @@ -68,6 +69,11 @@ const template = html` label="Include load-time percentile aggregation" :checked="percentiles" :disabled="jobs.busy"/> +
{{ dashboard.number(benchmark.result.events) }} events · {{ benchmark.result.days }} days · {{ benchmark.result.workers }} workers · Percentiles {{ benchmark.result.percentiles ? 'on' : 'off' }} + · Indexes {{ benchmark.result.indexes ? 'on' : 'off' }}

@@ -221,12 +228,14 @@ export function defineSampleBenchmarkPanel() { const days = sref(365) const workers = sref(20) const percentiles = sref(false) + const indexes = sref(true) return { ...head.props, count, days, workers, percentiles, + indexes, benchmark: computed(() => jobs.status()?.benchmark ?? null), viewUrl: computed(() => { const model = unref(head.props.dashboard) @@ -267,6 +276,7 @@ export function defineSampleBenchmarkPanel() { days: span, workers: concurrency, percentiles: percentiles(), + indexes: indexes(), }) }, } diff --git a/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/jobs.ts b/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/jobs.ts index 5d539e9..f202704 100644 --- a/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/jobs.ts +++ b/apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/jobs.ts @@ -13,6 +13,7 @@ export interface SampleBenchmarkResult { days: number workers: number percentiles: boolean + indexes: boolean preparationMs: number submissionMs: number ingestionMs: number @@ -47,6 +48,7 @@ export interface SampleBenchmarkStatus { workers: number days: number percentiles: boolean + indexes: boolean error: string | null result: SampleBenchmarkResult | null ingested: number diff --git a/apps/ZoneTree.Analytics.Dashboard/ui/package.json b/apps/ZoneTree.Analytics.Dashboard/ui/package.json index 0bd0ade..c60597a 100644 --- a/apps/ZoneTree.Analytics.Dashboard/ui/package.json +++ b/apps/ZoneTree.Analytics.Dashboard/ui/package.json @@ -1,6 +1,6 @@ { "name": "zonetree-analytics-dashboard", - "version": "1.0.1", + "version": "1.0.2", "private": true, "type": "module", "scripts": { @@ -10,13 +10,13 @@ "publish": "purestack publish --content ./content" }, "dependencies": { - "@purestack/ts-components": "^1.1.14", - "@purestack/ts-css": "^1.1.14", - "@purestack/ts-html": "^1.1.14", - "@purestack/ts-style": "^1.1.14", - "@purestack/ts-svg-icons": "^1.1.14", - "@purestack/ts-util": "^1.1.14", - "purestack": "^1.1.14", + "@purestack/ts-components": "^1.1.15", + "@purestack/ts-css": "^1.1.15", + "@purestack/ts-html": "^1.1.15", + "@purestack/ts-style": "^1.1.15", + "@purestack/ts-svg-icons": "^1.1.15", + "@purestack/ts-util": "^1.1.15", + "purestack": "^1.1.15", "regor": "^1.7.5" }, "devDependencies": { diff --git a/apps/ZoneTree.Analytics.Dashboard/ui/tests/sample-data-browser.ts b/apps/ZoneTree.Analytics.Dashboard/ui/tests/sample-data-browser.ts index 45ba98e..9348c3a 100644 --- a/apps/ZoneTree.Analytics.Dashboard/ui/tests/sample-data-browser.ts +++ b/apps/ZoneTree.Analytics.Dashboard/ui/tests/sample-data-browser.ts @@ -14,6 +14,7 @@ export async function checkSampleDataTools(page: Page, base: string) { days: 7, workers: 2, percentiles: false, + indexes: true, preparationMs: 100, submissionMs: 200, ingestionMs: 500, @@ -48,6 +49,7 @@ export async function checkSampleDataTools(page: Page, base: string) { workers: 1, days: 30, percentiles: false, + indexes: true, error: null, result, ingested: 0, @@ -93,6 +95,7 @@ export async function checkSampleDataTools(page: Page, base: string) { workers: 2, days: 7, percentiles: true, + indexes: false, }, } } else if (endpoint === 'cancel') { @@ -131,6 +134,13 @@ export async function checkSampleDataTools(page: Page, base: string) { await page .getByText('Include load-time percentile aggregation', { exact: true }) .click() + assert.equal( + await page.getByLabel('Maintain dimension and rollup indexes').isChecked(), + true, + ) + await page + .getByText('Maintain dimension and rollup indexes', { exact: true }) + .click() assert.equal( await page .getByLabel('Include load-time percentile aggregation') @@ -140,7 +150,7 @@ export async function checkSampleDataTools(page: Page, base: string) { await page.getByRole('button', { name: 'Run benchmark', exact: true }).click() await page.getByText('Preparing 0 / 17 events', { exact: true }).waitFor() assert.deepEqual(submissions, [ - { events: 17, days: 7, workers: 2, percentiles: true }, + { events: 17, days: 7, workers: 2, percentiles: true, indexes: false }, ]) assert.equal( await page @@ -196,7 +206,7 @@ export async function checkSampleDataTools(page: Page, base: string) { benchmark: { ...status.benchmark, state: 'completed', - result: { ...result, percentiles: true }, + result: { ...result, percentiles: true, indexes: false }, }, } await page.getByText('Benchmark completed', { exact: true }).waitFor() @@ -213,9 +223,12 @@ export async function checkSampleDataTools(page: Page, base: string) { true, ) await page - .getByText('17 events · 7 days · 2 workers · Percentiles on', { - exact: true, - }) + .getByText( + '17 events · 7 days · 2 workers · Percentiles on · Indexes off', + { + exact: true, + }, + ) .waitFor() assert.equal( await page @@ -255,6 +268,6 @@ export async function checkSampleDataTools(page: Page, base: string) { fullPage: true, }) console.log( - 'Sample tools checks passed: benchmark inputs, percentile toggle, stages, shared busy state, cancellation, previous results and mobile layout (mock responses).', + 'Sample tools checks passed: benchmark inputs, percentile and index toggles, stages, shared busy state, cancellation, previous results and mobile layout (mock responses).', ) } diff --git a/benchmarks/measure-overview.js b/benchmarks/measure-overview.js new file mode 100644 index 0000000..f98aff0 --- /dev/null +++ b/benchmarks/measure-overview.js @@ -0,0 +1,49 @@ +// Runs in the browser page of query-benchmark.ts. A plain script, so the TypeScript +// loader cannot add helpers that the page does not have. +globalThis.measureOverview = async ({ + urls, + samples, + warmupBatches, + maxConcurrentRequests, +}) => { + // One overview load: each request is timed from its slot through JSON parsing. + async function load() { + const start = performance.now() + const responses = [] + let next = 0 + async function slot() { + while (next < urls.length) { + const index = next++ + const requested = performance.now() + const response = await fetch(urls[index], { cache: 'no-store' }) + const json = await response.json() + responses[index] = { + ms: performance.now() - requested, + status: response.status, + json, + } + } + } + await Promise.all(Array.from({ length: maxConcurrentRequests }, slot)) + return { ms: performance.now() - start, responses } + } + + const discarded = [] + for (let i = 0; i < warmupBatches; ++i) discarded.push((await load()).ms) + const loads = [] + for (let i = 0; i < samples; ++i) loads.push(await load()) + const batches = loads.map((x) => ({ + ms: x.ms, + requests: x.responses.map((y) => ({ ms: y.ms, status: y.status })), + })) + const shapes = loads[loads.length - 1].responses.map(({ json }) => ({ + from: json.from, + to: json.to, + plan: json.plan, + summary: json.summary, + rows: Array.isArray(json.rows) ? json.rows.length : 0, + totalRows: json.totalRows, + serializedResponseCharacters: JSON.stringify(json).length, + })) + return { discarded, batches, shapes } +} diff --git a/benchmarks/query-benchmark.ts b/benchmarks/query-benchmark.ts new file mode 100644 index 0000000..f7ca90a --- /dev/null +++ b/benchmarks/query-benchmark.ts @@ -0,0 +1,227 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { chromium } from 'playwright' + +// Measures the dashboard's annual overview reports over the latest ingestion benchmark. +// Start the dashboard in Release (yarn dashboard:release), run a benchmark from +// Tools → Sample data, then run from the repository root: +// yarn benchmark:queries [output.json] [origin] +// The output, artifacts/benchmarks/query-benchmark-results.json by default, has the layout of +// docs-site/content/docs/performance/query-benchmarks/results.json. +const output = + process.argv[2] ?? 'artifacts/benchmarks/query-benchmark-results.json' +const origin = process.argv[3] ?? 'http://127.0.0.1:5088' +const samples = 30 +const warmupBatches = 3 +const maxConcurrentRequests = 3 + +const query = 'site=benchmark-results&tz=UTC' +const range = 'range=this-year&compare=previous-period' +const api = `${origin}/_analytics/sample-data/benchmark-api` +const dashboardUrl = `${origin}/?${query}&${range}` + +// The six requests of one overview load, as the dashboard sends them. +const overview: { name: string; path: string }[] = [ + { name: 'summary', path: `summary?${query}` }, + { name: 'timeseries', path: `timeseries?${query}&interval=day` }, + { + name: 'pages', + path: `pages?${query}&limit=5&includePageFlowMetrics=false`, + }, + { + name: 'breakdown:source', + path: `breakdown?${query}&dimension=source&limit=5&includeSessionMetrics=false`, + }, + { + name: 'breakdown:country', + path: `breakdown?${query}&dimension=country&limit=5&includeSessionMetrics=false`, + }, + { + name: 'breakdown:device', + path: `breakdown?${query}&dimension=device&limit=10&includeSessionMetrics=false`, + }, +] + +const scenarios = [ + { id: 'unfiltered', label: 'No filter', filter: '' }, + { id: 'source-github', label: 'source = GitHub', filter: 'source==GitHub' }, + { id: 'country-us', label: 'country = US', filter: 'country==US' }, + { id: 'page-blog', label: 'page = /blog', filter: 'page==/blog' }, +] + +const reportUrl = (path: string, filter: string) => + `${api}/${path}&${range}${filter ? `&filter=${encodeURIComponent(filter)}` : ''}` + +interface Timing { + median: number + p95: number + min: number + max: number +} + +// Median of the middle two samples; p95 by nearest rank. +function stats(values: number[]): Timing { + const sorted = [...values].sort((a, b) => a - b) + const middle = sorted.length / 2 + const median = + sorted.length % 2 + ? sorted[Math.floor(middle)] + : (sorted[middle - 1] + sorted[middle]) / 2 + return { + median, + p95: sorted[Math.ceil(0.95 * sorted.length) - 1], + min: sorted[0], + max: sorted[sorted.length - 1], + } +} + +// The result of measureOverview in measure-overview.js. +interface OverviewRun { + discarded: number[] + batches: { ms: number; requests: { ms: number; status: number }[] }[] + shapes: { + from: string + to: string + plan: unknown + summary?: unknown + rows: number + totalRows?: number + serializedResponseCharacters: number + }[] +} + +async function status() { + const response = await fetch(`${origin}/_analytics/sample-data/status`) + if (!response.ok) + throw new Error(`The dashboard at ${origin} returned ${response.status}.`) + return await response.json() +} + +const before = await status() +if (before.benchmark?.state !== 'completed' || !before.benchmark.result) + throw new Error('Run an ingestion benchmark in the dashboard first.') + +const browser = await chromium.launch({ channel: 'chrome', headless: true }) +const page = await browser.newPage() + +// The dashboard's own first requests are recorded, not timed. +const initialReports: unknown[] = [] +let capturing = true +page.on('response', async (response) => { + const url = response.url() + if (!capturing || !url.includes('/benchmark-api/') || url.includes('/config')) + return + try { + const json = await response.json() + initialReports.push({ + url, + status: response.status(), + from: json.from, + to: json.to, + plan: json.plan, + }) + } catch { + // A response without a JSON body is not a report. + } +}) +await page.goto(dashboardUrl, { waitUntil: 'networkidle' }) +await page.waitForTimeout(1000) +capturing = false +// Measure from a same-origin page, so dashboard refresh timers send no requests. +await page.goto(`${origin}/_analytics/sample-data/status`) +await page.addScriptTag({ + path: fileURLToPath(new URL('measure-overview.js', import.meta.url)), +}) + +const results = [] +for (const scenario of scenarios) { + const urls = overview.map((report) => reportUrl(report.path, scenario.filter)) + const run = (await page.evaluate( + `measureOverview(${JSON.stringify({ urls, samples, warmupBatches, maxConcurrentRequests })})`, + )) as OverviewRun + + const failedRequests = run.batches + .flatMap((batch) => batch.requests) + .filter((request) => request.status !== 200).length + const batch = stats(run.batches.map((x) => x.ms)) + results.push({ + ...scenario, + batch, + failedRequests, + reports: overview.map((report, index) => { + const shape = run.shapes[index] + return { + name: report.name, + url: urls[index], + timing: stats(run.batches.map((x) => x.requests[index].ms)), + plan: shape.plan, + rows: shape.rows, + ...(shape.summary ? { summary: shape.summary } : {}), + ...(shape.totalRows !== undefined + ? { totalRows: shape.totalRows } + : {}), + from: shape.from, + to: shape.to, + samples: run.batches.map((x) => x.requests[index].ms), + serializedResponseCharacters: shape.serializedResponseCharacters, + } + }), + batchSamples: run.batches.map((x) => x.ms), + discardedWarmupBatches: run.discarded, + }) + console.log( + `${scenario.label}: ${batch.median.toFixed(1)} ms median, ${batch.p95.toFixed(1)} ms p95, ${failedRequests} failed`, + ) +} +const browserVersion = browser.version() +await browser.close() +const after = await status() + +const measuredAt = new Date() +const dataset = before.benchmark.result +mkdirSync(path.dirname(output), { recursive: true }) +writeFileSync( + output, + `${JSON.stringify( + { + measuredOn: measuredAt.toISOString().slice(0, 10), + measuredAtUtc: measuredAt.toISOString(), + browser: browserVersion, + url: dashboardUrl, + dataset, + methodology: { + samplesPerScenario: samples, + discardedWarmupBatches: warmupBatches, + requestsPerBatch: overview.length, + maxConcurrentRequests, + cache: 'no-store', + metric: 'Browser fetch through JSON body parsing', + percentiles: 'Median of middle two samples; p95 nearest rank', + firstVisitIsCold: false, + measuredRequests: samples * overview.length * scenarios.length, + measuredBatches: samples * scenarios.length, + initialNavigationExcluded: true, + engineCaches: 'Warm; no host restart or cache clearing', + }, + initialReports, + scenarios: results, + environment: { + cpu: os.cpus()[0].model.trim(), + logicalProcessors: os.cpus().length, + operatingSystem: `${os.platform() === 'win32' ? 'Windows' : os.type()} ${os.arch()}`, + hostBuild: dataset.build, + transport: 'HTTP over loopback', + storageProfile: 'Not recorded', + installedMemoryGiB: Math.round(os.totalmem() / 2 ** 30), + }, + stateUnchanged: + JSON.stringify(before.benchmark) === JSON.stringify(after.benchmark) && + before.state === after.state, + }, + null, + 2, + )}\n`, +) +console.log(`Wrote ${output}`) diff --git a/docs-site/content/docs/performance/benchmarking/benchmarking.mdx b/docs-site/content/docs/performance/benchmarking/benchmarking.mdx index ad8249f..3459d3d 100644 --- a/docs-site/content/docs/performance/benchmarking/benchmarking.mdx +++ b/docs-site/content/docs/performance/benchmarking/benchmarking.mdx @@ -11,7 +11,7 @@ Measure performance with a representative workload and record the conditions of ## Recorded results -The [5M-event query benchmark](../query-benchmarks/query-benchmarks.mdx) includes an annual overview completing in **84 ms median**, country and device breakdowns for GitHub traffic around **13 ms**, and p95 timings from 30 loads per scope. The page records the workload, query plans, original ingestion result, and environment so the numbers can be assessed and reproduced. +The [5M-event query benchmark](../query-benchmarks/query-benchmarks.mdx) includes an annual overview completing in **33 ms median**, country and device breakdowns for GitHub traffic around **6 ms**, and p95 timings from 30 loads per scope. The page records the workload, query plans, original ingestion result, and environment so the numbers can be assessed and reproduced. ## Choose a tool diff --git a/docs-site/content/docs/performance/index.mdx b/docs-site/content/docs/performance/index.mdx index 64eff25..95d9220 100644 --- a/docs-site/content/docs/performance/index.mdx +++ b/docs-site/content/docs/performance/index.mdx @@ -11,7 +11,7 @@ Use query plans and representative measurements to decide what to aggregate, ind ## Measured at scale -The [5M-event query benchmark](./query-benchmarks/query-benchmarks.mdx) records **84 ms median** for all six annual overview reports, with comparisons enabled. It includes filtered reports, p95 timings, query plans, and the raw samples from the running dashboard. +The [5M-event query benchmark](./query-benchmarks/query-benchmarks.mdx) records **33 ms median** for all six annual overview reports, with comparisons enabled. It includes filtered reports, p95 timings, query plans, and the raw samples from the running dashboard. ## Recommended investigation diff --git a/docs-site/content/docs/performance/query-benchmarks/query-benchmarks.mdx b/docs-site/content/docs/performance/query-benchmarks/query-benchmarks.mdx index de604b8..1fb0b4a 100644 --- a/docs-site/content/docs/performance/query-benchmarks/query-benchmarks.mdx +++ b/docs-site/content/docs/performance/query-benchmarks/query-benchmarks.mdx @@ -7,9 +7,9 @@ template: doc # 5M-event query benchmark -**Six annual overview reports completed in 84 ms median, with a 5-million-event dataset and previous-period comparisons enabled.** Filtered country and device breakdowns for GitHub traffic returned in about 13 ms median through covering rollup indexes. +**Six annual overview reports completed in 33 ms median, with a 5-million-event dataset and previous-period comparisons enabled.** Filtered country and device breakdowns for GitHub traffic returned in about 6 ms median through covering rollup indexes. -These results were collected from the running dashboard on **14 October 2026**. They measure browser/API response time, including response transfer and JSON parsing, in a .NET host. +These results were collected from the running dashboard on **17 October 2026**. They measure browser/API response time, including response transfer and JSON parsing, in a .NET host. @@ -38,12 +38,12 @@ Each overview load requests traffic totals, a daily time series, top pages, sour | Report scope | All six reports: median | All six reports: p95 | | --- | ---: | ---: | -| No filter | **84.0 ms** | 87.0 ms | -| source = GitHub | **60.0 ms** | 62.3 ms | -| country = US | **68.5 ms** | 69.7 ms | -| page = /blog | **62.2 ms** | 63.8 ms | +| No filter | **33.3 ms** | 35.5 ms | +| source = GitHub | **16.0 ms** | 17.8 ms | +| country = US | **26.5 ms** | 28.7 ms | +| page = /blog | **14.3 ms** | 15.3 ms | -The selected range was **1 January–31 December 2026**, compared with the previous period. Every daily time series returned **365 buckets**. The unfiltered current-period report contained **4,248,150 page views**; the retained dataset also includes earlier traffic and custom events. +The selected range was **1 January–31 December 2026**, compared with the previous period. Every daily time series returned **365 buckets**. The unfiltered current-period report contained **4,267,778 page views**; the retained dataset also includes earlier traffic and custom events. Batch time ends when all six JSON responses have been parsed. It includes waiting for a request slot and is measured separately from each request; adding individual medians does not reconstruct the batch time. Chart rendering and initial page assets are outside this measurement. @@ -55,51 +55,51 @@ Each cell below comes from **30 measured requests** for that report and scope. T | Report | Median | p95 | | --- | ---: | ---: | -| Traffic summary | 25.9 ms | 27.8 ms | -| Daily time series | 30.1 ms | 33.1 ms | -| Top pages | 63.6 ms | 65.7 ms | -| Sources | 32.4 ms | 34.4 ms | -| Countries | 27.2 ms | 28.0 ms | -| Devices | 26.8 ms | 27.8 ms | +| Traffic summary | 8.5 ms | 9.8 ms | +| Daily time series | 11.5 ms | 12.6 ms | +| Top pages | 25.9 ms | 27.5 ms | +| Sources | 14.0 ms | 15.2 ms | +| Countries | 11.5 ms | 12.6 ms | +| Devices | 11.1 ms | 12.0 ms | ### GitHub traffic -Filter: `source==GitHub`. The selected period contained **1,805,990 page views** from this source. +Filter: `source==GitHub`. The selected period contained **1,809,311 page views** from this source. | Report | Median | p95 | | --- | ---: | ---: | -| Traffic summary | 43.8 ms | 46.4 ms | -| Daily time series | 46.4 ms | 48.5 ms | -| Top pages | 29.3 ms | 30.0 ms | -| Sources | 24.3 ms | 26.0 ms | -| Countries | **13.5 ms** | **14.0 ms** | -| Devices | **13.3 ms** | **13.8 ms** | +| Traffic summary | 5.9 ms | 7.1 ms | +| Daily time series | 8.0 ms | 9.1 ms | +| Top pages | 12.5 ms | 13.9 ms | +| Sources | 4.5 ms | 6.2 ms | +| Countries | **5.7 ms** | **6.5 ms** | +| Devices | **5.6 ms** | **6.2 ms** | ### US traffic -Filter: `country==US`. The selected period contained **2,533,854 page views** from the US. +Filter: `country==US`. The selected period contained **2,543,624 page views** from the US. | Report | Median | p95 | | --- | ---: | ---: | -| Traffic summary | 39.4 ms | 41.0 ms | -| Daily time series | 42.3 ms | 43.8 ms | -| Top pages | 46.3 ms | 47.7 ms | -| Sources | 24.0 ms | 25.1 ms | -| Countries | 22.7 ms | 23.4 ms | -| Devices | 21.9 ms | 22.8 ms | +| Traffic summary | 8.6 ms | 10.3 ms | +| Daily time series | 10.5 ms | 11.7 ms | +| Top pages | 19.5 ms | 21.1 ms | +| Sources | 10.0 ms | 11.9 ms | +| Countries | 7.0 ms | 8.6 ms | +| Devices | 9.0 ms | 9.7 ms | ### Blog traffic -Filter: `page==/blog`. The selected period contained **1,077,885 page views** for this page. +Filter: `page==/blog`. The selected period contained **1,078,037 page views** for this page. | Report | Median | p95 | | --- | ---: | ---: | -| Traffic summary | 44.9 ms | 46.7 ms | -| Daily time series | 47.6 ms | 49.7 ms | -| Top pages | 45.0 ms | 47.0 ms | -| Sources | 17.2 ms | 18.3 ms | -| Countries | **13.4 ms** | **14.3 ms** | -| Devices | **13.4 ms** | **14.3 ms** | +| Traffic summary | 6.3 ms | 7.0 ms | +| Daily time series | 7.8 ms | 9.3 ms | +| Top pages | 6.0 ms | 7.2 ms | +| Sources | 7.7 ms | 8.4 ms | +| Countries | **6.2 ms** | **7.0 ms** | +| Devices | **6.3 ms** | **6.9 ms** | ## What the plans show @@ -126,16 +126,16 @@ See [rollup planning](../rollups/rollups.mdx) and [secondary indexes](../seconda | Setting | Recorded value | | --- | --- | | Retained input events | **5,000,000**, all successfully ingested | -| Generated history | 365 days; 14 October 2025–14 October 2026 | +| Generated history | 365 days; 17 October 2025–17 October 2026 | | Fixture | Generated website traffic with visits, sources, geography, devices, custom actions, and load-time values | | Unfiltered group counts | 6 page paths, 6 sources, 2 countries, 2 device groups | | Query site / calendar | benchmark-results / UTC | | Requested range | 2026-01-01 inclusive through 2027-01-01 exclusive | | Comparison | Previous period: the 2025 calendar year | -| Host | Windows x64, **.NET 10.0.6**, **Debug** build | +| Host | Windows x64, **.NET 10.0.6**, **Release** build | | CPU | Intel Core Ultra 7 265KF; 20 logical processors | | Installed memory | 64 GiB | -| Browser | Chrome 155.0.8059.41, headless | +| Browser | Chrome 155.0.8059.82, headless | | Transport | HTTP over loopback | | Numeric load-time percentile aggregation | Off | @@ -167,16 +167,18 @@ Use the [raw results](./results.json) to inspect every request URL, timing sampl The retained dataset also carries the original ingestion measurement. It was not regenerated for these query measurements. -| Measurement | Recorded value | -| --- | ---: | -| Input / successfully ingested events | 5,000,000 / 5,000,000 | -| Ingestion workers | 20 | -| Ingestion throughput | **38,542 events/s** | -| Ingestion plus final flush | 129.727 s | -| Submission | 95.334 s | -| Preparation | 2.315 s | -| Errors / dropped / rejected | **0 / 0 / 0** | -| Build / GC / tiered compilation | Debug / Server / Default | +| Measurement | Recorded value | Without indexes | +| --- | ---: | ---: | +| Input / successfully ingested events | 5,000,000 / 5,000,000 | 5,000,000 / 5,000,000 | +| Ingestion workers | 20 | 20 | +| Ingestion throughput | **113,334 events/s** | **260,044 events/s** | +| Ingestion plus final flush | 44.118 s | 19.228 s | +| Submission | 34.678 s | 15.048 s | +| Preparation | 2.432 s | 2.377 s | +| Errors / dropped / rejected | **0 / 0 / 0** | **0 / 0 / 0** | +| Build / GC / tiered compilation | Release / Server / Default | Release / Server / Default | + +The recorded run maintains the dashboard's 4 dimension indexes and 12 rollup indexes during ingestion. The run without indexes writes only raw events, sessions, and rollups; it shows the raw ingestion speed, but its filtered reports cannot use covering indexes. Throughput includes submission, processing, and the final flush. Preparation and the final compaction used to publish the report dataset are excluded. Submission overlaps processing and is already included in the ingestion interval. diff --git a/docs-site/content/docs/performance/query-benchmarks/results.json b/docs-site/content/docs/performance/query-benchmarks/results.json index c02476c..ebd4bf9 100644 --- a/docs-site/content/docs/performance/query-benchmarks/results.json +++ b/docs-site/content/docs/performance/query-benchmarks/results.json @@ -1,26 +1,26 @@ { - "measuredOn": "2026-10-14", - "measuredAtUtc": "2026-10-14T21:38:32.348Z", - "browser": "155.0.8059.41", - "url": "http://127.0.0.1:5088/?site=benchmark-results\u0026tz=UTC\u0026range=this-year\u0026compare=previous-period", + "measuredOn": "2026-10-17", + "measuredAtUtc": "2026-10-17T10:20:31.394Z", + "browser": "155.0.8059.82", + "url": "http://127.0.0.1:5088/?site=benchmark-results&tz=UTC&range=this-year&compare=previous-period", "dataset": { "events": 5000000, "days": 365, "workers": 20, "percentiles": false, - "preparationMs": 2315.4104, - "submissionMs": 95333.6618, - "ingestionMs": 129727.3037, - "eventsPerSecond": 38542.38743420364, + "preparationMs": 2431.9199, + "submissionMs": 34677.5676, + "ingestionMs": 44117.556, + "eventsPerSecond": 113333.56725381615, "ingested": 5000000, "errors": 0, "dropped": 0, "rejected": 0, - "build": "Debug", + "build": "Release", "garbageCollection": "Server", "tieredCompilation": "Default", - "fromDate": "2025-10-14", - "toDate": "2026-10-14" + "fromDate": "2025-10-17", + "toDate": "2026-10-17" }, "methodology": { "samplesPerScenario": 30, @@ -38,15 +38,14 @@ }, "initialReports": [ { - "name": "summary", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results\u0026tz=UTC\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results&tz=UTC&range=this-year&compare=previous-period", "status": 200, - "milliseconds": 25.80000000447035, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -71,15 +70,14 @@ } }, { - "name": "timeseries", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results\u0026tz=UTC\u0026interval=day\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results&tz=UTC&interval=day&range=this-year&compare=previous-period", "status": 200, - "milliseconds": 29.599999994039536, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -104,42 +102,34 @@ } }, { - "name": "pages", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results\u0026tz=UTC\u0026limit=5\u0026includePageFlowMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=country&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period", "status": 200, - "milliseconds": 61.69999999552965, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", "source": "DayRollups", "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00" - }, - { - "event": "$bot", - "source": "DayRollups", - "from": "2026-01-01T00:00:00+00:00", - "to": "2027-01-01T00:00:00+00:00" } ], "rollupSuggestions": [] } }, { - "name": "breakdown:source", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=source\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=source&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period", "status": 200, - "milliseconds": 31.899999998509884, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -152,36 +142,44 @@ } }, { - "name": "breakdown:country", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=country\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/realtime?site=benchmark-results", + "status": 200 + }, + { + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results&tz=UTC&limit=5&includePageFlowMetrics=false&range=this-year&compare=previous-period", "status": 200, - "milliseconds": 26.200000002980232, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", "source": "DayRollups", "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00" + }, + { + "event": "$bot", + "source": "DayRollups", + "from": "2026-01-01T00:00:00+00:00", + "to": "2027-01-01T00:00:00+00:00" } ], "rollupSuggestions": [] } }, { - "name": "breakdown:device", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=device\u0026limit=10\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=device&limit=10&includeSessionMetrics=false&range=this-year&compare=previous-period", "status": 200, - "milliseconds": 28.5, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -200,24 +198,26 @@ "label": "No filter", "filter": "", "batch": { - "median": 83.94999999925494, - "p95": 87, - "min": 81.89999999850988, - "max": 87.29999999701977 + "median": 33.349999979138374, + "p95": 35.5, + "min": 31.69999998807907, + "max": 36.599999994039536 }, + "failedRequests": 0, "reports": [ { "name": "summary", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results\u0026tz=UTC\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results&tz=UTC&range=this-year&compare=previous-period", "timing": { - "median": 25.900000002235174, - "p95": 27.799999997019768, - "min": 24.700000002980232, - "max": 30.100000001490116 + "median": 8.549999997019768, + "p95": 9.799999982118607, + "min": 7.5, + "max": 10 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -242,50 +242,51 @@ }, "rows": 0, "summary": { - "visitors": 1252269, - "pageViews": 4248150, - "sessions": 1455491, - "bounceRate": 0.3603375080986416, - "averageSessionDuration": 50.823987, - "pageViewsPerSession": 2.9187057838214048, - "botHits": 150599, + "visitors": 1238848, + "pageViews": 4267778, + "sessions": 1462364, + "bounceRate": 0.360501899663832, + "averageSessionDuration": 50.8062741, + "pageViewsPerSession": 2.9184101906228546, + "botHits": 151308, "previous": { - "visitors": 121140, - "pageViews": 411450, - "sessions": 150400, - "bounceRate": 0.42210771276595743, - "averageSessionDuration": 44.9160153, - "pageViewsPerSession": 2.7357047872340425, - "botHits": 15866 + "visitors": 114359, + "pageViews": 392339, + "sessions": 143407, + "bounceRate": 0.422113285962331, + "averageSessionDuration": 44.9812875, + "pageViewsPerSession": 2.7358427412887796, + "botHits": 15066 } }, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 25.80000000447035, 26.200000002980232, 26.100000001490116, - 25.399999998509884, 26.099999994039536, 26.399999998509884, - 24.899999998509884, 24.799999997019768, 30.100000001490116, - 25.200000002980232, 25.30000000447035, 24.700000002980232, 27.5, 25, - 26.80000000447035, 26.100000001490116, 25.700000002980232, - 26.100000001490116, 26.5, 27.799999997019768, 26.19999999552965, - 25.900000005960464, 26.399999998509884, 24.899999998509884, - 25.30000000447035, 25.700000002980232, 24.80000000447035, - 25.399999998509884, 26.700000002980232, 25.899999998509884 + 8.099999994039536, 7.899999976158142, 7.800000011920929, 8, + 9.400000005960464, 10, 8.800000011920929, 7.800000011920929, + 7.799999982118607, 7.9000000059604645, 9.099999994039536, 9, + 8.300000011920929, 8.800000011920929, 8.5, 7.899999976158142, + 7.699999988079071, 8.900000005960464, 9.599999994039536, + 7.5999999940395355, 8.699999988079071, 8.799999982118607, + 8.200000017881393, 8.099999994039536, 8.599999994039536, + 8.799999982118607, 9.799999982118607, 9.699999988079071, + 8.700000017881393, 7.5 ], - "serializedResponseCharacters": 859 + "serializedResponseCharacters": 870 }, { "name": "timeseries", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results\u0026tz=UTC\u0026interval=day\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results&tz=UTC&interval=day&range=this-year&compare=previous-period", "timing": { - "median": 30.050000000745058, - "p95": 33.099999994039536, - "min": 28.700000002980232, - "max": 34.600000001490116 + "median": 11.5, + "p95": 12.599999994039536, + "min": 10.600000023841858, + "max": 12.699999988079071 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -313,30 +314,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 29.299999997019768, 29.19999999552965, 31.399999998509884, - 29.69999999552965, 28.700000002980232, 30.69999999552965, - 29.599999994039536, 30.100000001490116, 34.600000001490116, 29.5, - 29.200000002980232, 29.80000000447035, 31, 29.899999998509884, - 31.700000002980232, 31, 30, 30.700000002980232, 29.799999997019768, - 33.099999994039536, 31.200000002980232, 30.200000002980232, - 30.399999998509884, 30.299999997019768, 29.600000001490116, - 30.100000001490116, 29.200000002980232, 30.200000002980232, 30, - 29.700000002980232 + 10.900000005960464, 11.099999994039536, 11.699999988079071, + 10.900000005960464, 12.199999988079071, 12.599999994039536, + 11.700000017881393, 10.900000005960464, 10.699999988079071, + 10.900000005960464, 12, 12.5, 11.400000005960464, + 11.900000005960464, 11.400000005960464, 10.699999988079071, + 10.800000011920929, 11.799999982118607, 12.699999988079071, + 11.400000005960464, 11.599999994039536, 11.699999988079071, + 10.600000023841858, 11.399999976158142, 11.699999988079071, + 12.199999988079071, 11.900000005960464, 12.5, 10.800000011920929, 11 ], - "serializedResponseCharacters": 129848 + "serializedResponseCharacters": 129820 }, { "name": "pages", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results\u0026tz=UTC\u0026limit=5\u0026includePageFlowMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results&tz=UTC&limit=5&includePageFlowMetrics=false&range=this-year&compare=previous-period", "timing": { - "median": 63.599999997764826, - "p95": 65.69999999552965, - "min": 61.30000000447035, - "max": 66 + "median": 25.900000005960464, + "p95": 27.5, + "min": 25, + "max": 28.5 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -358,28 +360,30 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 62.899999998509884, 64, 63.69999999552965, 62.69999999552965, - 65.39999999850988, 62.5, 62.20000000298023, 61.30000000447035, 66, - 63.899999998509884, 63.30000000447035, 61.80000000447035, 64.5, - 63.5, 63.29999999701977, 65.69999999552965, 63.29999999701977, 65.5, - 64.60000000149012, 65.5, 64.89999999850988, 65, 62.899999998509884, - 64.80000000447035, 63.70000000298023, 64, 63.20000000298023, 62, - 63.099999994039536, 62.399999998509884 + 25.900000005960464, 25.599999994039536, 25.19999998807907, + 25.700000017881393, 26.69999998807907, 26.599999994039536, + 26.099999994039536, 27, 25, 25.400000005960464, 26, + 26.200000017881393, 26.30000001192093, 25.600000023841858, 28.5, + 25.900000005960464, 25, 25.5, 27.5, 25, 25.200000017881393, + 25.69999998807907, 25.299999982118607, 26, 26.200000017881393, + 25.900000005960464, 27.5, 25.900000005960464, 26.900000005960464, + 25.400000005960464 ], - "serializedResponseCharacters": 1600 + "serializedResponseCharacters": 1613 }, { "name": "breakdown:source", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=source\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=source&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period", "timing": { - "median": 32.399999998509884, - "p95": 34.399999998509884, - "min": 31.100000001490116, - "max": 50.5 + "median": 14, + "p95": 15.200000017881393, + "min": 12.800000011920929, + "max": 15.5 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -395,29 +399,30 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 31.399999998509884, 32.79999999701977, 31.600000001490116, - 34.399999998509884, 33.20000000298023, 32.79999999701977, - 32.100000001490116, 31.100000001490116, 32, 32.399999998509884, - 32.29999999701977, 32.399999998509884, 31.600000001490116, 50.5, - 31.69999999552965, 32.899999998509884, 32.5, 31.200000002980232, - 33.70000000298023, 32.80000000447035, 33.5, 32, 32.5, 33.5, - 32.69999999552965, 32.20000000298023, 32.19999999552965, - 32.400000005960464, 31.299999997019768, 32.20000000298023 + 14.199999988079071, 14.400000005960464, 13.599999994039536, + 14.300000011920929, 14.099999994039536, 13.5, 13.199999988079071, + 14, 14, 14, 13.100000023841858, 14.300000011920929, + 14.199999988079071, 12.900000005960464, 14.599999994039536, + 14.400000005960464, 13.5, 14.800000011920929, 14.099999994039536, + 13.199999988079071, 12.800000011920929, 12.800000011920929, + 13.799999982118607, 14, 14, 12.900000005960464, 15.200000017881393, + 12.900000005960464, 15.5, 13.400000005960464 ], - "serializedResponseCharacters": 1547 + "serializedResponseCharacters": 1562 }, { "name": "breakdown:country", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=country\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=country&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period", "timing": { - "median": 27.200000002980232, - "p95": 28, - "min": 26.19999999552965, - "max": 28.19999999552965 + "median": 11.450000002980232, + "p95": 12.599999994039536, + "min": 10.5, + "max": 13 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -433,30 +438,30 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 27, 26.200000002980232, 26.69999999552965, 27.700000002980232, - 26.899999998509884, 26.399999998509884, 27.200000002980232, - 26.899999998509884, 26.30000000447035, 26.19999999552965, - 26.399999998509884, 27.200000002980232, 27.099999994039536, - 27.399999998509884, 27.299999997019768, 28.19999999552965, - 26.899999998509884, 27.599999994039536, 27.600000001490116, - 26.200000002980232, 28, 26.899999998509884, 27.5, 28, - 27.69999999552965, 27.5, 27.200000002980232, 27.80000000447035, - 27.399999998509884, 26.200000002980232 + 12.099999994039536, 11.900000005960464, 11, 11.5, 11.5, + 11.299999982118607, 10.900000005960464, 12.200000017881393, + 11.800000011920929, 11.699999988079071, 10.800000011920929, + 11.599999994039536, 11.099999994039536, 10.700000017881393, + 12.099999994039536, 12.099999994039536, 11.5, 11, + 11.100000023841858, 10.799999982118607, 11.400000005960464, 10.5, + 11.5, 11.300000011920929, 11.700000017881393, 10.800000011920929, + 13, 10.600000023841858, 12.599999994039536, 11.199999988079071 ], - "serializedResponseCharacters": 899 + "serializedResponseCharacters": 911 }, { "name": "breakdown:device", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=device\u0026limit=10\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=device&limit=10&includeSessionMetrics=false&range=this-year&compare=previous-period", "timing": { - "median": 26.75, - "p95": 27.799999997019768, - "min": 25.899999998509884, - "max": 28 + "median": 11.149999991059303, + "p95": 12, + "min": 10.5, + "max": 12.300000011920929 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -472,29 +477,30 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 27.600000001490116, 26.19999999552965, 25.899999998509884, - 26.100000001490116, 26.5, 26.80000000447035, 26.600000001490116, - 27.100000001490116, 26.19999999552965, 27.5, 27.299999997019768, 26, - 27.100000001490116, 27, 27.100000001490116, 28, 26.799999997019768, - 26.599999994039536, 26.5, 27.100000001490116, 27.100000001490116, - 26.600000001490116, 26.700000002980232, 27.799999997019768, 26, - 26.19999999552965, 26.899999998509884, 26.299999997019768, 26.5, - 26.899999998509884 + 11.700000017881393, 11, 11.800000011920929, 10.599999994039536, 12, + 11.699999988079071, 11, 11, 11.300000011920929, 10.699999988079071, + 10.699999988079071, 11.099999994039536, 11.099999994039536, + 11.199999988079071, 11.200000017881393, 10.900000005960464, 10.5, + 11.200000017881393, 10.700000017881393, 11.300000011920929, + 11.400000005960464, 11.699999988079071, 11, 11.700000017881393, + 10.900000005960464, 12.300000011920929, 11.699999988079071, 11, + 11.200000017881393, 11 ], - "serializedResponseCharacters": 915 + "serializedResponseCharacters": 929 } ], "batchSamples": [ - 84.10000000149012, 81.89999999850988, 83.60000000149012, - 83.70000000298023, 82.39999999850988, 84, 83.5, 83, 87.29999999701977, - 83.39999999850988, 83, 83.10000000149012, 85.5, 84.5, 85.60000000149012, - 87, 83.89999999850988, 83.89999999850988, 84.20000000298023, - 86.70000000298023, 86.59999999403954, 83.80000000447035, 85, - 86.19999999552965, 83.5, 84.10000000149012, 83.5, 84.10000000149012, - 84.29999999701977, 83.10000000149012 + 34, 33.39999997615814, 33.20000001788139, 33, 35.5, 35.19999998807907, + 33, 32.80000001192093, 33.099999994039536, 32.599999994039536, + 32.900000005960464, 34.400000005960464, 33.599999994039536, + 32.900000005960464, 34.30000001192093, 33.19999998807907, + 31.69999998807907, 34.20000001788139, 34.400000005960464, + 32.099999994039536, 32.900000005960464, 33.29999998211861, 33, + 33.80000001192093, 33.5, 34, 36.599999994039536, 33.599999994039536, + 34.70000001788139, 31.900000005960464 ], "discardedWarmupBatches": [ - 87.59999999403954, 85.20000000298023, 84.89999999850988 + 33.900000005960464, 33.599999994039536, 34.10000002384186 ] }, { @@ -502,24 +508,26 @@ "label": "source = GitHub", "filter": "source==GitHub", "batch": { - "median": 60, - "p95": 62.29999999701977, - "min": 58.30000000447035, - "max": 62.79999999701977 + "median": 15.950000002980232, + "p95": 17.799999982118607, + "min": 14.899999976158142, + "max": 17.80000001192093 }, + "failedRequests": 0, "reports": [ { "name": "summary", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results\u0026tz=UTC\u0026range=this-year\u0026compare=previous-period\u0026filter=source%3D%3DGitHub", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results&tz=UTC&range=this-year&compare=previous-period&filter=source%3D%3DGitHub", "timing": { - "median": 43.80000000447035, - "p95": 46.399999998509884, - "min": 42.099999994039536, - "max": 46.79999999701977 + "median": 5.850000008940697, + "p95": 7.100000023841858, + "min": 5, + "max": 7.299999982118607 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -538,50 +546,50 @@ }, "rows": 0, "summary": { - "visitors": 558555, - "pageViews": 1805990, - "sessions": 596996, - "bounceRate": 0.32530033702068356, - "averageSessionDuration": 54.3240084, - "pageViewsPerSession": 3.025129146593947, + "visitors": 559394, + "pageViews": 1809311, + "sessions": 597719, + "bounceRate": 0.3251511161599347, + "averageSessionDuration": 54.3763243, + "pageViewsPerSession": 3.027026077471186, "botHits": null, "previous": { - "visitors": 17226, - "pageViews": 48515, - "sessions": 17772, - "bounceRate": 0.42763898266936756, - "averageSessionDuration": 44.8775575, - "pageViewsPerSession": 2.729855953184785, + "visitors": 16521, + "pageViews": 46258, + "sessions": 16972, + "bounceRate": 0.42794013669573416, + "averageSessionDuration": 44.7884179, + "pageViewsPerSession": 2.7255479613481026, "botHits": null } }, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 43.5, 45.5, 46.79999999701977, 43.5, 43.900000005960464, - 46.399999998509884, 44.29999999701977, 42.29999999701977, - 42.80000000447035, 43.80000000447035, 43.79999999701977, - 42.20000000298023, 42.600000001490116, 43.400000005960464, - 43.80000000447035, 45.399999998509884, 43.80000000447035, - 43.899999998509884, 44, 44.600000001490116, 42.5, - 44.399999998509884, 44.899999998509884, 43.399999998509884, - 42.69999999552965, 42.099999994039536, 42.5, 45.20000000298023, - 44.79999999701977, 42.80000000447035 + 5.700000017881393, 6, 5.5, 5.199999988079071, 5, 6, + 6.299999982118607, 6.199999988079071, 5.300000011920929, + 5.300000011920929, 5.5999999940395355, 5.9000000059604645, 6.5, + 6.299999982118607, 5.800000011920929, 7.299999982118607, + 6.399999976158142, 5.200000017881393, 6.0999999940395355, + 5.199999988079071, 5.0999999940395355, 5.199999988079071, + 5.4000000059604645, 6.5, 7.100000023841858, 5.9000000059604645, + 5.199999988079071, 5.200000017881393, 7, 6.0999999940395355 ], - "serializedResponseCharacters": 744 + "serializedResponseCharacters": 757 }, { "name": "timeseries", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results\u0026tz=UTC\u0026interval=day\u0026range=this-year\u0026compare=previous-period\u0026filter=source%3D%3DGitHub", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results&tz=UTC&interval=day&range=this-year&compare=previous-period&filter=source%3D%3DGitHub", "timing": { - "median": 46.400000002235174, - "p95": 48.5, - "min": 44.399999998509884, - "max": 48.5 + "median": 7.950000002980232, + "p95": 9.099999994039536, + "min": 7.0999999940395355, + "max": 9.099999994039536 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -603,30 +611,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 47.399999998509884, 48.5, 46.30000000447035, 47.80000000447035, - 44.399999998509884, 45.899999998509884, 46.29999999701977, - 45.29999999701977, 46.899999998509884, 45.20000000298023, - 47.80000000447035, 46.19999999552965, 45.69999999552965, - 45.899999998509884, 47.70000000298023, 48.5, 46.79999999701977, - 45.29999999701977, 45.70000000298023, 46.79999999701977, - 46.30000000447035, 46.600000001490116, 44.899999998509884, 47, - 45.69999999552965, 46.80000000447035, 46.5, 48, 46.100000001490116, - 47.30000000447035 + 7.5, 7.9000000059604645, 8, 7.800000011920929, 7.299999982118607, + 7.699999988079071, 8.799999982118607, 8.5, 8.700000017881393, + 7.100000023841858, 8.599999994039536, 7.800000011920929, + 8.900000005960464, 9.099999994039536, 8.100000023841858, + 8.199999988079071, 7.699999988079071, 7.200000017881393, 8, + 7.4000000059604645, 7.800000011920929, 7.700000017881393, + 7.799999982118607, 8.300000011920929, 9.099999994039536, + 8.200000017881393, 7.0999999940395355, 7.0999999940395355, + 8.799999982118607, 8.099999994039536 ], - "serializedResponseCharacters": 130136 + "serializedResponseCharacters": 130091 }, { "name": "pages", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results\u0026tz=UTC\u0026limit=5\u0026includePageFlowMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=source%3D%3DGitHub", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results&tz=UTC&limit=5&includePageFlowMetrics=false&range=this-year&compare=previous-period&filter=source%3D%3DGitHub", "timing": { - "median": 29.25, - "p95": 30, - "min": 28, - "max": 32.5 + "median": 12.5, + "p95": 13.900000005960464, + "min": 12.199999988079071, + "max": 14.199999988079071 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -643,30 +652,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 28.799999997019768, 32.5, 29.600000001490116, 29.100000001490116, - 28.299999997019768, 29.400000005960464, 28.19999999552965, - 28.799999997019768, 29.399999998509884, 28.700000002980232, 29.5, - 29.899999998509884, 29.099999994039536, 30, 29.399999998509884, - 28.80000000447035, 29.899999998509884, 29.099999994039536, - 28.899999998509884, 28.399999998509884, 28.5, 29.899999998509884, - 29.400000005960464, 28.399999998509884, 29.899999998509884, - 28.900000005960464, 28, 29.899999998509884, 29.700000002980232, - 29.899999998509884 + 12.5, 12.400000005960464, 12.599999994039536, 12.300000011920929, + 12.199999988079071, 12.199999988079071, 12.399999976158142, + 12.599999994039536, 13.700000017881393, 12.800000011920929, + 13.900000005960464, 12.599999994039536, 13, 13, 12.700000017881393, + 14.199999988079071, 12.299999982118607, 12.399999976158142, + 12.400000005960464, 12.199999988079071, 12.199999988079071, + 12.300000011920929, 13.199999988079071, 13.099999994039536, + 12.699999988079071, 12.700000017881393, 12.299999982118607, + 12.199999988079071, 12.5, 12.399999976158142 ], - "serializedResponseCharacters": 1535 + "serializedResponseCharacters": 1516 }, { "name": "breakdown:source", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=source\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=source%3D%3DGitHub", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=source&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=source%3D%3DGitHub", "timing": { - "median": 24.299999997019768, - "p95": 26, - "min": 23.200000002980232, - "max": 26.900000005960464 + "median": 4.450000002980232, + "p95": 6.199999988079071, + "min": 3.9000000059604645, + "max": 6.699999988079071 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -682,30 +692,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 24.799999997019768, 24.799999997019768, 26, 24.299999997019768, - 26.900000005960464, 24, 24.899999998509884, 24.899999998509884, - 24.5, 24, 25.5, 23.200000002980232, 24.100000001490116, - 23.299999997019768, 23.700000002980232, 25.100000001490116, - 24.200000002980232, 24.200000002980232, 24.399999998509884, - 24.19999999552965, 23.700000002980232, 24.399999998509884, - 24.299999997019768, 24, 24.399999998509884, 25.799999997019768, - 24.80000000447035, 24.200000002980232, 23.600000001490116, - 23.600000001490116 + 4.799999982118607, 4, 4.5999999940395355, 4.700000017881393, + 4.799999982118607, 5.100000023841858, 4.300000011920929, + 4.0999999940395355, 6.699999988079071, 4.5999999940395355, + 5.199999988079071, 4.199999988079071, 4.4000000059604645, + 4.0999999940395355, 4.0999999940395355, 5, 4.100000023841858, + 5.299999982118607, 4.199999988079071, 5.300000011920929, + 4.5999999940395355, 4.5, 5.4000000059604645, 4.4000000059604645, + 4.0999999940395355, 6.199999988079071, 3.9000000059604645, + 4.0999999940395355, 4.200000017881393, 4.299999982118607 ], - "serializedResponseCharacters": 559 + "serializedResponseCharacters": 572 }, { "name": "breakdown:country", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=country\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=source%3D%3DGitHub", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=country&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=source%3D%3DGitHub", "timing": { - "median": 13.5, - "p95": 14, - "min": 12.799999997019768, - "max": 14.399999998509884 + "median": 5.699999988079071, + "p95": 6.5, + "min": 5.4000000059604645, + "max": 6.5 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -722,30 +733,30 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 13.299999997019768, 13.299999997019768, 13.900000005960464, - 13.399999998509884, 14.399999998509884, 13.799999997019768, - 13.900000005960464, 13.800000004470348, 13.699999995529652, - 12.899999998509884, 13.799999997019768, 13.699999995529652, - 13.699999995529652, 13.299999997019768, 13.099999994039536, - 13.600000001490116, 13.199999995529652, 13.600000001490116, 13.5, - 13.699999995529652, 13.199999995529652, 13.100000001490116, 13.5, - 13.200000002980232, 13.200000002980232, 14, 13.5, - 13.799999997019768, 13.200000002980232, 12.799999997019768 + 5.5, 5.5, 5.799999982118607, 5.5999999940395355, 5.5, + 5.4000000059604645, 5.600000023841858, 5.799999982118607, + 6.199999988079071, 6.0999999940395355, 6.5, 5.5999999940395355, + 5.700000017881393, 5.9000000059604645, 5.899999976158142, + 6.4000000059604645, 5.5, 5.699999988079071, 5.699999988079071, + 5.699999988079071, 5.5999999940395355, 5.5999999940395355, 6, + 5.699999988079071, 5.5, 6.5, 6, 6, 5.4000000059604645, + 6.299999982118607 ], - "serializedResponseCharacters": 919 + "serializedResponseCharacters": 931 }, { "name": "breakdown:device", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=device\u0026limit=10\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=source%3D%3DGitHub", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=device&limit=10&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=source%3D%3DGitHub", "timing": { - "median": 13.25, - "p95": 13.799999997019768, - "min": 12.700000002980232, - "max": 14.799999997019768 + "median": 5.5999999940395355, + "p95": 6.199999988079071, + "min": 5.4000000059604645, + "max": 6.300000011920929 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -762,57 +773,56 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 13.5, 13, 13.700000002980232, 13.099999994039536, - 13.700000002980232, 13.100000001490116, 13.600000001490116, - 13.600000001490116, 13.299999997019768, 13, 14.799999997019768, - 13.600000001490116, 13.199999995529652, 13, 13.200000002980232, - 13.600000001490116, 13.100000001490116, 13.799999997019768, - 12.700000002980232, 13.100000001490116, 13.299999997019768, - 13.399999998509884, 13.299999997019768, 13.300000004470348, - 13.200000002980232, 12.900000005960464, 12.700000002980232, - 13.399999998509884, 13.199999995529652, 13.199999995529652 + 5.5999999940395355, 5.5, 5.4000000059604645, 5.5999999940395355, + 5.5, 5.899999976158142, 5.4000000059604645, 5.5, 5.5, + 5.800000011920929, 5.800000011920929, 5.5, 5.800000011920929, 5.5, + 5.800000011920929, 5.5, 5.799999982118607, 6.300000011920929, + 5.600000023841858, 5.5999999940395355, 5.4000000059604645, 5.5, 6, + 5.699999988079071, 5.5999999940395355, 5.700000017881393, + 5.799999982118607, 6.199999988079071, 5.4000000059604645, + 5.4000000059604645 ], - "serializedResponseCharacters": 937 + "serializedResponseCharacters": 948 } ], "batchSamples": [ - 61, 61.70000000298023, 60.5, 61.099999994039536, 58.30000000447035, - 59.899999998509884, 60, 59.20000000298023, 60.599999994039536, - 58.400000005960464, 62.79999999701977, 60, 59.29999999701977, - 59.100000001490116, 61.20000000298023, 62.29999999701977, - 60.20000000298023, 59.19999999552965, 58.70000000298023, - 60.20000000298023, 59.69999999552965, 60.20000000298023, - 58.399999998509884, 60.600000001490116, 59.100000001490116, 60, - 59.29999999701977, 61.600000001490116, 59.69999999552965, - 60.600000001490116 + 16.099999994039536, 15.5, 15.600000023841858, 15.5, 15.299999982118607, + 17, 16, 15.799999982118607, 17.5, 15.800000011920929, + 16.599999994039536, 15.599999994039536, 16.700000017881393, + 15.899999976158142, 15.700000017881393, 17.799999982118607, + 16.400000005960464, 16.80000001192093, 15.900000005960464, + 16.099999994039536, 15.099999994039536, 15.199999988079071, + 16.80000001192093, 16.599999994039536, 16.80000001192093, + 17.80000001192093, 14.899999976158142, 15.5, 16.600000023841858, + 15.799999982118607 ], - "discardedWarmupBatches": [ - 77.09999999403954, 61.80000000447035, 59.399999998509884 - ] + "discardedWarmupBatches": [16, 15.900000005960464, 15.5] }, { "id": "country-us", "label": "country = US", "filter": "country==US", "batch": { - "median": 68.5, - "p95": 69.69999999552965, - "min": 66.79999999701977, - "max": 70.60000000149012 + "median": 26.5, + "p95": 28.700000017881393, + "min": 25.399999976158142, + "max": 28.900000005960464 }, + "failedRequests": 0, "reports": [ { "name": "summary", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results\u0026tz=UTC\u0026range=this-year\u0026compare=previous-period\u0026filter=country%3D%3DUS", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results&tz=UTC&range=this-year&compare=previous-period&filter=country%3D%3DUS", "timing": { - "median": 39.399999998509884, - "p95": 41, - "min": 37.5, - "max": 41.30000000447035 + "median": 8.600000008940697, + "p95": 10.299999982118607, + "min": 7.5, + "max": 10.699999988079071 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -831,50 +841,50 @@ }, "rows": 0, "summary": { - "visitors": 748253, - "pageViews": 2533854, - "sessions": 873394, - "bounceRate": 0.36627913633480425, - "averageSessionDuration": 50.3437384, - "pageViewsPerSession": 2.9011580111610566, + "visitors": 742232, + "pageViews": 2543624, + "sessions": 877150, + "bounceRate": 0.36679473294191417, + "averageSessionDuration": 50.3063407, + "pageViewsPerSession": 2.8998734537992363, "botHits": null, "previous": { - "visitors": 72730, - "pageViews": 244789, - "sessions": 90188, - "bounceRate": 0.42852707677296314, - "averageSessionDuration": 44.3125506, - "pageViewsPerSession": 2.7142080986384, + "visitors": 68898, + "pageViews": 233444, + "sessions": 86030, + "bounceRate": 0.4288968964314774, + "averageSessionDuration": 44.3044966, + "pageViewsPerSession": 2.713518540044171, "botHits": null } }, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 40.79999999701977, 37.70000000298023, 39.5, 39.19999999552965, - 38.899999998509884, 39.19999999552965, 40.100000001490116, - 40.600000001490116, 40.899999998509884, 39.900000005960464, 37.5, - 38.29999999701977, 40.80000000447035, 41.30000000447035, - 40.19999999552965, 40.79999999701977, 38.899999998509884, - 37.80000000447035, 39.600000001490116, 38.79999999701977, 39.5, - 39.399999998509884, 38.29999999701977, 39.399999998509884, 41, - 38.30000000447035, 39.30000000447035, 38.5, 38.400000005960464, - 39.69999999552965 + 7.5, 8.5, 8.599999994039536, 8.900000005960464, 7.9000000059604645, + 8.400000005960464, 7.899999976158142, 8.600000023841858, + 7.899999976158142, 8.5, 9.400000005960464, 10.699999988079071, + 8.900000005960464, 9.800000011920929, 7.700000017881393, + 8.399999976158142, 9.099999994039536, 8.900000005960464, + 7.899999976158142, 8.800000011920929, 9.099999994039536, 9.5, + 7.9000000059604645, 7.700000017881393, 7.799999982118607, 9, + 8.700000017881393, 7.5, 8.700000017881393, 10.299999982118607 ], - "serializedResponseCharacters": 744 + "serializedResponseCharacters": 758 }, { "name": "timeseries", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results\u0026tz=UTC\u0026interval=day\u0026range=this-year\u0026compare=previous-period\u0026filter=country%3D%3DUS", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results&tz=UTC&interval=day&range=this-year&compare=previous-period&filter=country%3D%3DUS", "timing": { - "median": 42.30000000074506, - "p95": 43.80000000447035, - "min": 40.399999998509884, - "max": 43.899999998509884 + "median": 10.5, + "p95": 11.700000017881393, + "min": 9.300000011920929, + "max": 11.800000011920929 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -896,30 +906,30 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 43, 42.70000000298023, 41.399999998509884, 40.399999998509884, - 41.600000001490116, 41, 42.69999999552965, 43.30000000447035, - 42.600000001490116, 43.20000000298023, 41.29999999701977, - 41.099999994039536, 42.5, 41.399999998509884, 41.5, 42.5, - 42.399999998509884, 41.70000000298023, 41.899999998509884, - 42.79999999701977, 42.30000000447035, 43.100000001490116, - 41.600000001490116, 42.29999999701977, 43.80000000447035, 41.5, - 42.70000000298023, 40.400000005960464, 43.899999998509884, - 41.600000001490116 + 9.5, 10.199999988079071, 11.5, 10.600000023841858, 9.5, + 10.300000011920929, 10.299999982118607, 10.5, 10.699999988079071, + 11.300000011920929, 11.199999988079071, 11.700000017881393, + 10.699999988079071, 11.200000017881393, 9.700000017881393, + 10.699999988079071, 10.599999994039536, 10.700000017881393, + 10.400000005960464, 10.5, 11.800000011920929, 11.100000023841858, + 10, 9.300000011920929, 9.699999988079071, 10.299999982118607, + 10.100000023841858, 10.199999988079071, 10.5, 10.799999982118607 ], - "serializedResponseCharacters": 130958 + "serializedResponseCharacters": 130941 }, { "name": "pages", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results\u0026tz=UTC\u0026limit=5\u0026includePageFlowMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=country%3D%3DUS", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results&tz=UTC&limit=5&includePageFlowMetrics=false&range=this-year&compare=previous-period&filter=country%3D%3DUS", "timing": { - "median": 46.25, - "p95": 47.70000000298023, - "min": 45.100000001490116, - "max": 48.30000000447035 + "median": 19.549999997019768, + "p95": 21.099999994039536, + "min": 18.400000005960464, + "max": 22.19999998807907 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -936,29 +946,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 46.20000000298023, 46, 46.399999998509884, 47.600000001490116, - 45.600000001490116, 47.399999998509884, 46.79999999701977, - 45.899999998509884, 47.399999998509884, 48.30000000447035, - 45.29999999701977, 46.399999998509884, 46.600000001490116, - 46.100000001490116, 46, 45.899999998509884, 45.600000001490116, - 46.20000000298023, 46.899999998509884, 47, 45.5, 46.099999994039536, - 46.20000000298023, 46.29999999701977, 47, 45.100000001490116, 46.5, - 46.80000000447035, 46, 47.70000000298023 + 18.899999976158142, 18.899999976158142, 19.400000005960464, + 20.799999982118607, 18.599999994039536, 20.900000005960464, 19, + 21.099999994039536, 19.299999982118607, 20.69999998807907, + 19.80000001192093, 20.30000001192093, 19.5, 22.19999998807907, + 18.899999976158142, 19.5, 19.599999994039536, 20.80000001192093, + 19.19999998807907, 19, 20.19999998807907, 19.600000023841858, + 20.400000005960464, 18.400000005960464, 18.69999998807907, + 19.400000005960464, 20.5, 18.900000005960464, 19.599999994039536, + 20.19999998807907 ], - "serializedResponseCharacters": 1511 + "serializedResponseCharacters": 1524 }, { "name": "breakdown:source", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=source\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=country%3D%3DUS", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=source&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=country%3D%3DUS", "timing": { - "median": 24, - "p95": 25.100000001490116, - "min": 22.899999998509884, - "max": 25.399999998509884 + "median": 10.049999997019768, + "p95": 11.900000005960464, + "min": 9.600000023841858, + "max": 12.099999994039536 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -975,29 +987,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 24.399999998509884, 23.600000001490116, 24, 24.5, 24.30000000447035, - 24.900000005960464, 23.600000001490116, 25.100000001490116, - 24.200000002980232, 22.899999998509884, 25.399999998509884, - 24.200000002980232, 24.5, 23.799999997019768, 24.700000002980232, - 23.5, 24.600000001490116, 24.399999998509884, 23.100000001490116, - 24.299999997019768, 23.399999998509884, 23.200000002980232, - 23.899999998509884, 24, 24, 23.69999999552965, 24, - 24.299999997019768, 23.599999994039536, 23.80000000447035 + 10.299999982118607, 9.900000005960464, 10.200000017881393, + 12.099999994039536, 10, 10.800000011920929, 10, 9.899999976158142, + 10.100000023841858, 10.600000023841858, 9.899999976158142, + 10.100000023841858, 9.900000005960464, 11.900000005960464, + 9.699999988079071, 10.600000023841858, 9.900000005960464, + 11.300000011920929, 10, 9.900000005960464, 10.099999994039536, + 9.800000011920929, 11.699999988079071, 9.899999976158142, + 10.400000005960464, 9.600000023841858, 11.299999982118607, + 9.900000005960464, 10.199999988079071, 9.900000005960464 ], - "serializedResponseCharacters": 1571 + "serializedResponseCharacters": 1582 }, { "name": "breakdown:country", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=country\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=country%3D%3DUS", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=country&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=country%3D%3DUS", "timing": { - "median": 22.650000002235174, - "p95": 23.400000005960464, - "min": 21.5, - "max": 23.899999998509884 + "median": 6.950000002980232, + "p95": 8.599999994039536, + "min": 6.5, + "max": 9 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1013,29 +1027,30 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 22.700000002980232, 22.30000000447035, 22.600000001490116, 23, 23, - 22.899999998509884, 22.600000001490116, 23.400000005960464, - 22.200000002980232, 22.399999998509884, 23.899999998509884, - 22.80000000447035, 22.700000002980232, 22.600000001490116, - 23.100000001490116, 22, 23.19999999552965, 22.19999999552965, 22, - 23.399999998509884, 22.400000005960464, 23, 22.5, - 22.200000002980232, 22.900000005960464, 22.30000000447035, - 22.799999997019768, 22.80000000447035, 21.5, 22.100000001490116 + 7.299999982118607, 6.5999999940395355, 7.300000011920929, + 8.599999994039536, 7.0999999940395355, 8.099999994039536, 6.5, + 7.0999999940395355, 6.699999988079071, 7.699999988079071, + 6.800000011920929, 6.5999999940395355, 6.5, 9, 6.699999988079071, + 7.9000000059604645, 6.800000011920929, 7.5999999940395355, + 6.799999982118607, 6.600000023841858, 7, 6.5999999940395355, + 6.9000000059604645, 7.199999988079071, 7.4000000059604645, + 6.700000017881393, 7.699999988079071, 6.5, 8.5, 6.700000017881393 ], - "serializedResponseCharacters": 552 + "serializedResponseCharacters": 565 }, { "name": "breakdown:device", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=device\u0026limit=10\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=country%3D%3DUS", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=device&limit=10&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=country%3D%3DUS", "timing": { - "median": 21.849999997764826, - "p95": 22.799999997019768, - "min": 21.100000001490116, - "max": 23.099999994039536 + "median": 9, + "p95": 9.700000017881393, + "min": 8.5, + "max": 10.200000017881393 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1052,54 +1067,57 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 21.399999998509884, 21.399999998509884, 22.100000001490116, - 21.899999998509884, 22.5, 21.399999998509884, 21.899999998509884, - 22.799999997019768, 21.19999999552965, 22.19999999552965, - 22.700000002980232, 21.19999999552965, 21.899999998509884, - 22.100000001490116, 21.700000002980232, 22.700000002980232, - 23.099999994039536, 21.19999999552965, 21.700000002980232, - 22.400000005960464, 21.100000001490116, 22.400000005960464, 21.5, - 21.899999998509884, 21.399999998509884, 21.799999997019768, - 21.700000002980232, 22.599999994039536, 21.200000002980232, 21.5 + 8.599999994039536, 8.900000005960464, 8.799999982118607, + 9.299999982118607, 9.699999988079071, 9.200000017881393, 9.5, + 9.299999982118607, 9, 9.700000017881393, 8.599999994039536, + 9.299999982118607, 8.699999988079071, 8.699999988079071, + 10.200000017881393, 8.699999988079071, 8.699999988079071, + 8.900000005960464, 9, 9, 9.299999982118607, 8.5, 9.699999988079071, + 9.600000023841858, 9.099999994039536, 8.699999988079071, 9, 9, + 8.599999994039536, 8.799999982118607 ], - "serializedResponseCharacters": 945 + "serializedResponseCharacters": 956 } ], "batchSamples": [ - 67.79999999701977, 67.5, 68.80000000447035, 69.69999999552965, - 68.20000000298023, 69, 68.69999999552965, 69.10000000149012, - 68.89999999850988, 70.60000000149012, 68.10000000149012, - 67.79999999701977, 68.5, 68.40000000596046, 67.89999999850988, - 68.89999999850988, 68.79999999701977, 67.70000000298023, - 68.70000000298023, 69.5, 66.79999999701977, 68.70000000298023, - 67.79999999701977, 68.29999999701977, 68.5, 67, 68.30000000447035, - 69.69999999552965, 67.40000000596046, 69.29999999701977 + 25.399999976158142, 25.80000001192093, 27.599999994039536, + 28.599999994039536, 26.400000005960464, 27.600000023841858, + 26.399999976158142, 27.100000023841858, 26.399999976158142, + 28.700000017881393, 26.599999994039536, 27.69999998807907, 26, + 28.900000005960464, 26.700000017881393, 27.5, 26.19999998807907, + 27.30000001192093, 26.299999982118607, 26.200000017881393, + 28.299999982118607, 26.30000001192093, 26.69999998807907, + 26.30000001192093, 26.299999982118607, 25.900000005960464, + 26.900000005960464, 25.69999998807907, 27.600000023841858, + 26.299999982118607 ], - "discardedWarmupBatches": [93.40000000596046, 67.5, 68.20000000298023] + "discardedWarmupBatches": [26.5, 28.30000001192093, 25.400000005960464] }, { "id": "page-blog", "label": "page = /blog", "filter": "page==/blog", "batch": { - "median": 62.149999998509884, - "p95": 63.79999999701977, - "min": 60.30000000447035, - "max": 64 + "median": 14.299999982118607, + "p95": 15.299999982118607, + "min": 13.300000011920929, + "max": 15.900000005960464 }, + "failedRequests": 0, "reports": [ { "name": "summary", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results\u0026tz=UTC\u0026range=this-year\u0026compare=previous-period\u0026filter=page%3D%3D%2Fblog", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/summary?site=benchmark-results&tz=UTC&range=this-year&compare=previous-period&filter=page%3D%3D%2Fblog", "timing": { - "median": 44.849999997764826, - "p95": 46.70000000298023, - "min": 43.100000001490116, - "max": 47.100000001490116 + "median": 6.299999997019768, + "p95": 7, + "min": 5.600000023841858, + "max": 7.899999976158142 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1118,50 +1136,51 @@ }, "rows": 0, "summary": { - "visitors": 793428, - "pageViews": 1077885, + "visitors": 795631, + "pageViews": 1078037, "sessions": null, "bounceRate": null, "averageSessionDuration": null, "pageViewsPerSession": null, - "botHits": 25230, + "botHits": 25295, "previous": { - "visitors": 23345, - "pageViews": 28941, + "visitors": 22394, + "pageViews": 27588, "sessions": null, "bounceRate": null, "averageSessionDuration": null, "pageViewsPerSession": null, - "botHits": 2662 + "botHits": 2543 } }, "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 45.19999999552965, 45.19999999552965, 43.100000001490116, - 46.399999998509884, 45, 44.29999999701977, 44.79999999701977, 44.5, - 44.600000001490116, 45.29999999701977, 44.899999998509884, - 44.600000001490116, 44.69999999552965, 45.80000000447035, 43.5, - 43.399999998509884, 47.100000001490116, 45.70000000298023, - 44.099999994039536, 43.79999999701977, 45.399999998509884, - 45.29999999701977, 44.70000000298023, 44.899999998509884, - 45.20000000298023, 44.600000001490116, 45.100000001490116, 44, - 46.70000000298023, 43.900000005960464 + 6.699999988079071, 6.300000011920929, 6.100000023841858, + 6.300000011920929, 6.0999999940395355, 6.300000011920929, + 5.9000000059604645, 5.799999982118607, 5.600000023841858, 7, + 6.699999988079071, 6.5999999940395355, 7, 5.700000017881393, 6, + 6.5999999940395355, 5.9000000059604645, 6.299999982118607, + 6.300000011920929, 7.899999976158142, 6.9000000059604645, + 6.699999988079071, 6.600000023841858, 5.700000017881393, + 5.9000000059604645, 5.9000000059604645, 5.9000000059604645, + 5.600000023841858, 5.699999988079071, 6.5 ], - "serializedResponseCharacters": 670 + "serializedResponseCharacters": 683 }, { "name": "timeseries", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results\u0026tz=UTC\u0026interval=day\u0026range=this-year\u0026compare=previous-period\u0026filter=page%3D%3D%2Fblog", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/timeseries?site=benchmark-results&tz=UTC&interval=day&range=this-year&compare=previous-period&filter=page%3D%3D%2Fblog", "timing": { - "median": 47.55000000074506, - "p95": 49.69999999552965, - "min": 46.19999999552965, - "max": 50.100000001490116 + "median": 7.800000011920929, + "p95": 9.299999982118607, + "min": 7.299999982118607, + "max": 9.400000005960464 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1183,30 +1202,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 47.600000001490116, 48.5, 46.399999998509884, 47.5, - 50.100000001490116, 49.600000001490116, 46.80000000447035, - 49.69999999552965, 46.20000000298023, 47.20000000298023, - 47.600000001490116, 47.29999999701977, 46.5, 47.20000000298023, - 46.79999999701977, 47.29999999701977, 49.399999998509884, - 46.19999999552965, 47.80000000447035, 47.70000000298023, - 46.69999999552965, 48.400000005960464, 48.5, 49.29999999701977, - 49.100000001490116, 49.399999998509884, 46.70000000298023, - 46.69999999552965, 48.899999998509884, 46.899999998509884 + 8.199999988079071, 7.800000011920929, 7.899999976158142, + 7.699999988079071, 7.9000000059604645, 7.5999999940395355, + 7.899999976158142, 7.5, 7.399999976158142, 8.5, 8.900000005960464, + 8.400000005960464, 8.800000011920929, 7.300000011920929, + 7.4000000059604645, 9.400000005960464, 7.5999999940395355, + 8.099999994039536, 7.800000011920929, 9.299999982118607, + 8.900000005960464, 8.300000011920929, 8, 7.5, 7.800000011920929, + 7.299999982118607, 7.300000011920929, 7.5, 7.4000000059604645, + 7.700000017881393 ], - "serializedResponseCharacters": 120865 + "serializedResponseCharacters": 120879 }, { "name": "pages", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results\u0026tz=UTC\u0026limit=5\u0026includePageFlowMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=page%3D%3D%2Fblog", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/pages?site=benchmark-results&tz=UTC&limit=5&includePageFlowMetrics=false&range=this-year&compare=previous-period&filter=page%3D%3D%2Fblog", "timing": { - "median": 45, - "p95": 47, - "min": 42.600000001490116, - "max": 48 + "median": 6.049999997019768, + "p95": 7.199999988079071, + "min": 5.399999976158142, + "max": 7.799999982118607 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1228,29 +1248,29 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 45.20000000298023, 44.600000001490116, 43.100000001490116, - 44.30000000447035, 45.399999998509884, 43.79999999701977, - 45.399999998509884, 45.100000001490116, 46.899999998509884, 47, - 44.5, 44.69999999552965, 44.100000001490116, 44.79999999701977, 46, - 44.69999999552965, 48, 45.599999994039536, 42.600000001490116, - 43.30000000447035, 47, 47, 45.79999999701977, 46, - 44.600000001490116, 45, 44.600000001490116, 45.099999994039536, 45, - 43.79999999701977 + 6.800000011920929, 6.4000000059604645, 6, 6, 6.0999999940395355, 6, + 5.699999988079071, 5.699999988079071, 5.5, 6.9000000059604645, + 6.5999999940395355, 7.199999988079071, 7, 5.5, 6, + 6.4000000059604645, 5.9000000059604645, 6.300000011920929, + 6.200000017881393, 7.799999982118607, 6.700000017881393, 6.5, 6.5, + 5.399999976158142, 5.800000011920929, 5.799999982118607, + 5.700000017881393, 5.399999976158142, 6.199999988079071, 5.5 ], - "serializedResponseCharacters": 623 + "serializedResponseCharacters": 636 }, { "name": "breakdown:source", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=source\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=page%3D%3D%2Fblog", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=source&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=page%3D%3D%2Fblog", "timing": { - "median": 17.200000002980232, - "p95": 18.299999997019768, - "min": 16.700000002980232, - "max": 18.600000001490116 + "median": 7.699999988079071, + "p95": 8.399999976158142, + "min": 7.299999982118607, + "max": 8.800000011920929 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1267,30 +1287,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 17.200000002980232, 17.5, 17.200000002980232, 17.399999998509884, - 17.200000002980232, 17.299999997019768, 16.700000002980232, - 17.299999997019768, 17.30000000447035, 17.600000001490116, - 17.100000001490116, 17.5, 17, 16.700000002980232, 17.69999999552965, - 17.200000002980232, 16.899999998509884, 17, 18.299999997019768, - 17.100000001490116, 17.400000005960464, 16.899999998509884, - 17.299999997019768, 17.899999998509884, 17.099999994039536, - 18.600000001490116, 17.399999998509884, 16.80000000447035, - 17.19999999552965, 16.899999998509884 + 7.4000000059604645, 8.299999982118607, 7.899999976158142, + 8.299999982118607, 7.9000000059604645, 8.399999976158142, 8, + 7.9000000059604645, 7.299999982118607, 7.300000011920929, + 7.300000011920929, 7.5999999940395355, 7.300000011920929, + 8.800000011920929, 7.5999999940395355, 8.099999994039536, + 7.4000000059604645, 8, 7.299999982118607, 7.4000000059604645, + 7.4000000059604645, 7.300000011920929, 7.5, 8.100000023841858, + 7.5999999940395355, 8, 7.799999982118607, 7.800000011920929, + 8.200000017881393, 7.4000000059604645 ], - "serializedResponseCharacters": 1558 + "serializedResponseCharacters": 1572 }, { "name": "breakdown:country", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=country\u0026limit=5\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=page%3D%3D%2Fblog", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=country&limit=5&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=page%3D%3D%2Fblog", "timing": { - "median": 13.399999998509884, - "p95": 14.300000004470348, - "min": 12.900000005960464, - "max": 14.699999995529652 + "median": 6.199999988079071, + "p95": 7, + "min": 5.699999988079071, + "max": 7.200000017881393 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1307,30 +1328,31 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 13.600000001490116, 12.900000005960464, 13.399999998509884, - 13.299999997019768, 13.100000001490116, 13.700000002980232, - 13.600000001490116, 13.299999997019768, 13.300000004470348, - 13.899999998509884, 13.299999997019768, 13.200000002980232, - 13.200000002980232, 13, 14.200000002980232, 13.200000002980232, - 13.5, 13.5, 14.300000004470348, 13.100000001490116, - 13.700000002980232, 13.200000002980232, 13.200000002980232, - 14.299999997019768, 13.5, 14.699999995529652, 13.5, - 13.600000001490116, 13.399999998509884, 13.300000004470348 + 6.199999988079071, 7, 6.799999982118607, 6.799999982118607, + 6.4000000059604645, 6.4000000059604645, 6.200000017881393, + 6.0999999940395355, 5.699999988079071, 5.9000000059604645, + 5.800000011920929, 5.700000017881393, 5.799999982118607, + 6.899999976158142, 6.199999988079071, 6.700000017881393, + 6.4000000059604645, 6.5999999940395355, 5.799999982118607, 6, + 5.9000000059604645, 6, 6, 6.199999988079071, 6.300000011920929, + 6.300000011920929, 6.5, 5.799999982118607, 7.200000017881393, + 5.699999988079071 ], - "serializedResponseCharacters": 915 + "serializedResponseCharacters": 928 }, { "name": "breakdown:device", - "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results\u0026tz=UTC\u0026dimension=device\u0026limit=10\u0026includeSessionMetrics=false\u0026range=this-year\u0026compare=previous-period\u0026filter=page%3D%3D%2Fblog", + "url": "http://127.0.0.1:5088/_analytics/sample-data/benchmark-api/breakdown?site=benchmark-results&tz=UTC&dimension=device&limit=10&includeSessionMetrics=false&range=this-year&compare=previous-period&filter=page%3D%3D%2Fblog", "timing": { - "median": 13.399999998509884, - "p95": 14.299999997019768, - "min": 12.899999998509884, - "max": 14.5 + "median": 6.299999982118607, + "p95": 6.9000000059604645, + "min": 5.5999999940395355, + "max": 6.9000000059604645 }, "plan": { "complete": true, "warnings": [], + "notices": [], "steps": [ { "event": "pageview", @@ -1347,31 +1369,32 @@ "from": "2026-01-01T00:00:00+00:00", "to": "2027-01-01T00:00:00+00:00", "samples": [ - 14.200000002980232, 13.399999998509884, 13.800000004470348, - 13.399999998509884, 13.200000002980232, 13.399999998509884, - 13.299999997019768, 13.899999998509884, 13.400000005960464, - 13.599999994039536, 13.200000002980232, 13.300000004470348, 13.5, - 13.100000001490116, 14, 13.100000001490116, 13.600000001490116, - 13.200000002980232, 14.100000001490116, 13.299999997019768, - 13.300000004470348, 13.300000004470348, 13.5, 14.299999997019768, - 13.299999997019768, 13.600000001490116, 13.399999998509884, 14.5, - 13, 12.899999998509884 + 5.700000017881393, 6.699999988079071, 6.4000000059604645, + 6.9000000059604645, 6.299999982118607, 6.799999982118607, + 6.600000023841858, 6.299999982118607, 5.800000011920929, + 5.700000017881393, 5.5999999940395355, 5.799999982118607, + 5.5999999940395355, 6.5, 6.199999988079071, 6.5, 6.300000011920929, + 6.5, 6.4000000059604645, 5.9000000059604645, 5.800000011920929, + 5.899999976158142, 5.9000000059604645, 6.9000000059604645, + 6.0999999940395355, 6.4000000059604645, 6.299999982118607, + 6.200000017881393, 6.5999999940395355, 6.199999988079071 ], - "serializedResponseCharacters": 935 + "serializedResponseCharacters": 948 } ], "batchSamples": [ - 62.399999998509884, 62.29999999701977, 60.30000000447035, - 61.79999999701977, 63.5, 63, 61.5, 63.69999999552965, - 61.900000005960464, 62.899999998509884, 61.899999998509884, - 62.100000001490116, 61.29999999701977, 61.70000000298023, - 61.29999999701977, 60.600000001490116, 64, 62.70000000298023, 62, - 61.19999999552965, 62.80000000447035, 62.29999999701977, 62, - 63.79999999701977, 62.600000001490116, 63.30000000447035, - 62.19999999552965, 61.5, 62.399999998509884, 60.900000005960464 + 14.099999994039536, 14.599999994039536, 14.400000005960464, + 14.800000011920929, 14.299999982118607, 14.5, 14.600000023841858, + 13.899999976158142, 13.300000011920929, 14.300000011920929, + 14.599999994039536, 14.199999988079071, 14.5, 14.400000005960464, + 13.599999994039536, 15.900000005960464, 13.900000005960464, + 14.699999988079071, 14.200000017881393, 15.299999982118607, + 14.800000011920929, 14.299999982118607, 14, 14.400000005960464, 14, + 13.900000005960464, 13.699999988079071, 13.700000017881393, + 14.099999994039536, 14 ], "discardedWarmupBatches": [ - 73.19999999552965, 62.100000001490116, 61.30000000447035 + 15.799999982118607, 14.900000005960464, 14.900000005960464 ] } ], @@ -1380,9 +1403,10 @@ "logicalProcessors": 20, "operatingSystem": "Windows x64", "runtime": ".NET 10.0.6", - "hostBuild": "Debug", + "hostBuild": "Release", "transport": "HTTP over loopback", "storageProfile": "Not recorded", "installedMemoryGiB": 64 - } + }, + "stateUnchanged": true } diff --git a/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx b/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx index de1243f..491b2d9 100644 --- a/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx +++ b/docs-site/content/docs/performance/secondary-indexes/secondary-indexes.mdx @@ -18,7 +18,7 @@ Use secondary indexes when important filtered reports need a more selective orde Both are optional and disabled in the core defaults. Each adds storage and ingestion work, so choose from actual report patterns. -The [5M-event query benchmark](../query-benchmarks/query-benchmarks.mdx#what-the-plans-show) shows these paths in use: country and device breakdowns for GitHub traffic returned in about **13 ms median** with source-leading covering indexes. +The [5M-event query benchmark](../query-benchmarks/query-benchmarks.mdx#what-the-plans-show) shows these paths in use: country and device breakdowns for GitHub traffic returned in about **6 ms median** with source-leading covering indexes. ## Index an event dimension diff --git a/docs-site/content/index.mdx b/docs-site/content/index.mdx index 48d58cc..f19a900 100644 --- a/docs-site/content/index.mdx +++ b/docs-site/content/index.mdx @@ -38,7 +38,7 @@ layout: ## Measured on 5 million events -Six annual overview reports completed in **84 ms median**, with previous-period comparisons enabled. The retained benchmark spans **365 days** of generated traffic. +Six annual overview reports completed in **33 ms median**, with previous-period comparisons enabled. The retained benchmark spans **365 days** of generated traffic. diff --git a/docs-site/package.json b/docs-site/package.json index ea67e7e..cc2ff1b 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -2,7 +2,7 @@ "private": true, "type": "module", "name": "analytics.zonetree.dev", - "version": "1.0.1", + "version": "1.0.2", "scripts": { "check": "tsc --noEmit", "build": "purestack build --content ./content --clean", @@ -11,9 +11,9 @@ "deploy": "pgdeploy deploy --host https://analytics.zonetree.dev --package ./dist/web --allow-path 'pagefind/*,.ts-ssg/manifest.json' --skip-path 'docs/*.json,*.cs,*.http,*.ps1,*.sh'" }, "dependencies": { - "@purestack/ts-style": "^1.1.14", - "@purestack/ts-util": "^1.1.14", - "purestack": "^1.1.14", + "@purestack/ts-style": "^1.1.15", + "@purestack/ts-util": "^1.1.15", + "purestack": "^1.1.15", "regor": "^1.7.5" }, "devDependencies": { diff --git a/internal-docs/storage-format.md b/internal-docs/storage-format.md index 21c48e1..1912fd8 100644 --- a/internal-docs/storage-format.md +++ b/internal-docs/storage-format.md @@ -38,7 +38,7 @@ Paths in this table are relative to `AnalyticsOptions.DataDirectory`. A partitio | Path | Contents | | --- | --- | | `engine.lock` | Exclusive handle held for the engine's lifetime | -| `meta/meta-a.json` and `meta/meta-b.json` | Alternating generations of dataset metadata | +| `meta/meta-a.bin` and `meta/meta-b.bin` | Alternating generations of dataset metadata | | `events/yyyy-MM-dd/` | Raw event partitions, when `StoreRawEvents` is enabled | | `rollups/minute/yyyy-MM-dd/` | Minute aggregate partitions, when enabled | | `rollups/hour/yyyy-MM-dd/` | Hour aggregate partitions, when enabled | @@ -351,9 +351,17 @@ A missing rollup dimension is converted to the `01` sentinel string before encod ## Metadata file format -Each `meta-a.json` or `meta-b.json` is a UTF-8 header line, a line-feed byte, and an indented JSON object. Despite the extension, the complete file is not a standalone JSON document. +Each `meta-a.bin` or `meta-b.bin` is a binary header followed by an indented UTF-8 JSON object. The historical joint-rollup journal uses the same header with its own magic: -The header consists of three space-separated tokens: `zonetree-analytics-meta`, a positive decimal generation, and the uppercase hexadecimal SHA-256 digest of **all bytes after the first LF**. The digest covers the JSON bytes exactly as written, including whitespace. It does not cover the header. +| Order | Field | Encoding | +| --- | --- | --- | +| 1 | Magic | The 8 ASCII bytes `ZTAMETA1` (metadata) or `ZTAJRNL1` (journal) | +| 2 | Generation | `u64le`, positive | +| 3 | Body length | `u64le` | +| 4 | Body checksum | 32-byte SHA-256 of the body | +| 5 | Body | The JSON object (metadata) or the checkpoint (journal) | + +The header is written last, so a torn write never validates. The checksum covers the body bytes exactly as written, including whitespace. The JSON body maps string keys to string values, with keys emitted in ordinal order. Numbers, booleans, manifests, and base64 are represented as strings at this layer. The writer escapes some characters, for example `+` in base64 as `+`, the `1F` separator as `\u001F`, and non-ASCII text, so read the file with a JSON parser rather than by text search. @@ -385,7 +393,7 @@ The JSON body maps string keys to string values, with keys emitted in ordinal or ### Generations and writes -A metadata update creates the next generation by replacing the older file in place, then flushing the stream to disk. Generation N goes to `meta-a.json` when N is even and `meta-b.json` when N is odd. Startup validates available headers and checksums and selects the valid file with the highest generation. A missing or checksum-invalid generation can therefore leave the other one usable; existing metadata files with no valid generation cause startup to fail rather than create a new identity secret or reset format settings. +A metadata update creates the next generation by replacing the older file in place, then flushing the stream to disk. Generation N goes to `meta-a.bin` when N is even and `meta-b.bin` when N is odd. Startup validates available headers and checksums and selects the valid file with the highest generation. A missing or checksum-invalid generation can therefore leave the other one usable; existing metadata files with no valid generation cause startup to fail rather than create a new identity secret or reset format settings. When keys are removed, the store writes two successive generations so the removed entries disappear from both files. This applies to expired salts as well as completed recovery checkpoints. It is removal from the live metadata files, not secure erasure of historical backups or underlying storage media. @@ -420,17 +428,7 @@ Ingestion keeps writing its own copies while a build runs. A dimension-index bui Copy backfills use metadata keys ending in `.config`, `.phase`, `.partition`, `.key`, and `.count`. Phase is `clear` or `copy`; the cursor key is base64, and the count is decimal text. When a new build has dropped the old copies, the index format, configuration, and empty partition manifest are recorded with the index marked unclean, so an interrupted build resumes. Checkpoints advance after their writes return, every 100,000 input records and at partition boundaries. Absolute upserts make replay of an unfinished chunk idempotent. With the default asynchronous write-ahead log, copy and reconstruction writes can still be queued when the checkpoint advances. A crash can lose them even when their source raw events were already durable, leaving historical aggregates without those contributions. Synchronous logs for the stores involved make checkpoint recovery exact. -Historical reconstruction of joint rollups has a separate binary journal, `backfill/joint-rollups/journal-a.bin` and `journal-b.bin`, written alternately like the metadata files: - -| Order | Field | Encoding | -| --- | --- | --- | -| 1 | Magic | The 8 ASCII bytes `ZTAJRNL1` | -| 2 | Generation | `u64le`, positive | -| 3 | Body length | `u64le` | -| 4 | Body checksum | 32-byte SHA-256 of the body | -| 5 | Body | The checkpoint | - -The header is written last, so a torn write never validates; startup reads the valid file with the highest generation. The body holds the configuration signature, coverage, the fence, the raw partitions when the reconstruction began, the site/partition/key cursor, progress counters, active reconstructed sessions, and a redo list of absolute rebuilt parts. Redo records include granularity, partition, primary key bytes, and the serialized rebuilt part, stored as bytes rather than text. +Historical reconstruction of joint rollups has a separate binary journal, `backfill/joint-rollups/journal-a.bin` and `journal-b.bin`, written alternately in the [metadata file format](#metadata-file-format) with the magic `ZTAJRNL1`. Startup reads the valid file with the highest generation. The body holds the configuration signature, coverage, the fence, the raw partitions when the reconstruction began, the site/partition/key cursor, progress counters, active reconstructed sessions, and a redo list of absolute rebuilt parts. Redo records include granularity, partition, primary key bytes, and the serialized rebuilt part, stored as bytes rather than text. The reconstruction runs while ingestion writes to the same rows. It reads the raw events whose sequence is at most the fence, the raw event sequence when it began; ingestion writes every later event, late ones included, so each event counts once. Its contributions go to the rows' rebuilt parts under the row locks of the flushes. Replaying the absolute rebuilt parts avoids adding the same contributions twice after a crash and never undoes a flush. When the reconstruction begins, it deletes the rows the definitions kept from an earlier activation, before ingestion starts. Raw retention does not pass the coverage start until it completes. Then the journal files are deleted, the definitions are listed in `rollup-index.folding`, and the rollup index build folds the rebuilt parts in before the coverage is published. diff --git a/package.json b/package.json index 140f62f..4a73d5e 100644 --- a/package.json +++ b/package.json @@ -27,12 +27,15 @@ "test:dotnet": "dotnet test src/ZoneTree.Analytics.sln --configuration Release", "test": "yarn test:web && yarn test:dotnet", "dashboard": "dotnet run --project apps/ZoneTree.Analytics.Dashboard/server --", + "dashboard:release": "dotnet run -c release --project apps/ZoneTree.Analytics.Dashboard/server --", "dashboard:dev": "yarn workspace zonetree-analytics-dashboard dev", - "docs:dev": "yarn workspace analytics.zonetree.dev serve" + "docs:dev": "yarn workspace analytics.zonetree.dev serve", + "benchmark:queries": "tsx benchmarks/query-benchmark.ts" }, "devDependencies": { "@biomejs/biome": "^2.5.15", "@types/node": "^22.0.0", + "playwright": "^1.56.1", "tsx": "^4.20.6", "typescript": "^7.0.2", "vite": "8.3.4", diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index b36bed8..26ac988 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -8,5 +8,5 @@ "noEmit": true, "skipLibCheck": true }, - "include": ["*.ts", "../vitest.config.ts"] + "include": ["*.ts", "../benchmarks/*.ts", "../vitest.config.ts"] } diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 614ed86..76d7430 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -12,7 +12,7 @@ matching ZoneTree.Analytics. Pass VersionSuffix (for example preview.1) to pack a pre-release. --> - 1.0.1 + 1.0.2 false Ahmed Yasin Koculu ZoneTree diff --git a/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs b/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs index c1a9cb9..1a6da81 100644 --- a/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/BackfillLoggingTests.cs @@ -133,7 +133,7 @@ public void HostRegistrationAndManuallyCreatedSampleEnginesUseHostLoggingFilters Assert.That(provider.Messages.Any(log => log.Category == "ZoneTree.Analytics" && log.Message.StartsWith("Rollup index backfill started;", StringComparison.Ordinal)), Is.True); } provider.Messages.Clear(); - var options = DashboardAnalyticsOptions.CreateBenchmark(directory, TimeZoneInfo.Utc, 1, false, factory.CreateLogger("SampleBenchmark")); + var options = DashboardAnalyticsOptions.CreateBenchmark(directory, TimeZoneInfo.Utc, 1, false, true, factory.CreateLogger("SampleBenchmark")); TestEngine.UseTestDefaults(options); using (var engine = new AnalyticsEngine(options)) Assert.That(provider.Messages.Any(log => log.Category == "SampleBenchmark" && log.Message.Contains("backfill check completed;", StringComparison.Ordinal)), Is.True); diff --git a/src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs b/src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs index 83bdade..36e1c1c 100644 --- a/src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/BackgroundIndexingTests.cs @@ -368,6 +368,23 @@ public void ABuildWhoseCopyFailedIsNotRecordedAsComplete() Assert.That(result.Totals[Metric.Events], Is.EqualTo(3)); } + [Test] + public void RemovedIndexesForgetTheirRepairIntents() + { + using (var indexed = TestEngine.Create(directory, DimensionIndex)) + { + View(indexed, "a", Start, "/docs", "GitHub"); + indexed.Flush(); + } + // A crash left an intent; the dimension index is removed before the next open. + var meta = new MetaStore(Path.Combine(directory, "meta")); + meta.Set("dimension-index.dirty." + Start.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture), "true"); + using (var removed = TestEngine.Create(directory)) + Assert.That(removed.GetStatistics().IndexMaintenance, Is.Null); + // Before: a repair without indexes never ran, so the intent stayed for good. + Assert.That(new MetaStore(Path.Combine(directory, "meta")).ListByPrefix("dimension-index.dirty."), Is.Empty); + } + [Test] public void QueriesSeeTheRawEventsRetentionHolds() { diff --git a/src/ZoneTree.Analytics.UnitTests/DashboardAnalyticsOptionsTests.cs b/src/ZoneTree.Analytics.UnitTests/DashboardAnalyticsOptionsTests.cs index f50ae9a..c929e1e 100644 --- a/src/ZoneTree.Analytics.UnitTests/DashboardAnalyticsOptionsTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/DashboardAnalyticsOptionsTests.cs @@ -12,8 +12,8 @@ public void ProfilesShareRetentionAndIndexCoverageWithoutSharingMutableOptions() var primary = new AnalyticsOptions(); DashboardAnalyticsOptions.ConfigurePrimary(primary, "primary", zone); var samples = DashboardAnalyticsOptions.CreatePerformanceSamples("samples", zone, 4, NullLogger.Instance); - var benchmark = DashboardAnalyticsOptions.CreateBenchmark("benchmark", zone, 20, false, NullLogger.Instance); - var reports = DashboardAnalyticsOptions.CreateBenchmarkReport("retained", zone.Id, true, NullLogger.Instance); + var benchmark = DashboardAnalyticsOptions.CreateBenchmark("benchmark", zone, 20, false, true, NullLogger.Instance); + var reports = DashboardAnalyticsOptions.CreateBenchmarkReport("retained", zone.Id, true, true, NullLogger.Instance); foreach (var options in new[] { primary, samples, benchmark, reports }) { Assert.That(options.Retention.RawEvents, Is.EqualTo(TimeSpan.FromDays(365 * 3))); @@ -72,7 +72,7 @@ public void EveryEngineRetainsGeneratedHistoryOlderThanNinetyDays(string profile var options = profile switch { "samples" => DashboardAnalyticsOptions.CreatePerformanceSamples(directory, TimeZoneInfo.Utc, 1, NullLogger.Instance), - "benchmark" => DashboardAnalyticsOptions.CreateBenchmark(directory, TimeZoneInfo.Utc, 1, false, NullLogger.Instance), + "benchmark" => DashboardAnalyticsOptions.CreateBenchmark(directory, TimeZoneInfo.Utc, 1, false, true, NullLogger.Instance), _ => new AnalyticsOptions() }; if (profile == "primary") DashboardAnalyticsOptions.ConfigurePrimary(options, directory, TimeZoneInfo.Utc); diff --git a/src/ZoneTree.Analytics.UnitTests/FlushRecoveryTests.cs b/src/ZoneTree.Analytics.UnitTests/FlushRecoveryTests.cs index 0e131e5..6c92f61 100644 --- a/src/ZoneTree.Analytics.UnitTests/FlushRecoveryTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/FlushRecoveryTests.cs @@ -436,6 +436,8 @@ public WorkerHarness(string directory, Action configure = null o.Storage.StoreRawEvents = false; configure?.Invoke(o); }); + // An engine creates the directory's metadata, so engines can reopen what the worker wrote. + using (TestEngine.Create(directory, o => o.Storage.StoreRawEvents = false)) { } var calendar = new ReportingCalendar(options.TimeZone, options.FirstDayOfWeek); var meta = new MetaStore(Path.Combine(directory, "meta")); var hasher = new VisitorHasher(options.Privacy.VisitorIdentification, null, meta); diff --git a/src/ZoneTree.Analytics.UnitTests/MetaStoreTests.cs b/src/ZoneTree.Analytics.UnitTests/MetaStoreTests.cs index 91a6415..45e0bbb 100644 --- a/src/ZoneTree.Analytics.UnitTests/MetaStoreTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/MetaStoreTests.cs @@ -1,14 +1,13 @@ -using System.Globalization; using ZoneTree.Analytics.Storage; namespace ZoneTree.Analytics.UnitTests; public sealed class MetaStoreTests : DirectoryTest { - string[] Files => [Path.Combine(directory, "meta-a.json"), Path.Combine(directory, "meta-b.json")]; + string[] Files => [Path.Combine(directory, "meta-a.bin"), Path.Combine(directory, "meta-b.bin")]; - static long GenerationOf(string path) => - long.Parse(File.ReadLines(path).First().Split(' ')[1], CultureInfo.InvariantCulture); + // The generation follows the 8-byte magic. + static long GenerationOf(string path) => BitConverter.ToInt64(File.ReadAllBytes(path), 8); string NewestFile => Files.Where(File.Exists).OrderByDescending(GenerationOf).First(); @@ -54,7 +53,10 @@ public void AChangedFileIsDetected() meta.Set("time-zone", "UTC"); meta.Set("raw-sequence", "1"); var newest = NewestFile; - File.WriteAllText(newest, File.ReadAllText(newest).Replace("UTC", "CET", StringComparison.Ordinal)); + var bytes = File.ReadAllBytes(newest); + var at = bytes.AsSpan().IndexOf("UTC"u8); + "CET"u8.CopyTo(bytes.AsSpan(at)); + File.WriteAllBytes(newest, bytes); // The checksum rejects the edited file; the previous generation still has the setting. var reopened = new MetaStore(directory); diff --git a/src/ZoneTree.Analytics.UnitTests/ReleaseRegressionTests.cs b/src/ZoneTree.Analytics.UnitTests/ReleaseRegressionTests.cs index 9e441e0..0b7f151 100644 --- a/src/ZoneTree.Analytics.UnitTests/ReleaseRegressionTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/ReleaseRegressionTests.cs @@ -200,7 +200,7 @@ public void DisposeReleasesStorageWhenMetadataWriteFails() { var engine = TestEngine.Create(directory); engine.Track(new AnalyticsEvent("purchase") { Timestamp = Utc(Day0.AddHours(1)) }); - var paths = new[] { Path.Combine(directory, "meta", "meta-a.json"), Path.Combine(directory, "meta", "meta-b.json") }; + var paths = new[] { Path.Combine(directory, "meta", "meta-a.bin"), Path.Combine(directory, "meta", "meta-b.bin") }; foreach (var path in paths) { File.Move(path, path + ".bak"); diff --git a/src/ZoneTree.Analytics.UnitTests/SampleBenchmarkTests.cs b/src/ZoneTree.Analytics.UnitTests/SampleBenchmarkTests.cs index aa27fb5..e6a1043 100644 --- a/src/ZoneTree.Analytics.UnitTests/SampleBenchmarkTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/SampleBenchmarkTests.cs @@ -21,11 +21,11 @@ public void PreparationPreservesTheRealisticDataShape(bool percentiles) } } - [TestCase(1, false)] - [TestCase(4, true)] - public void BenchmarkUsesDefaultPerWorkerLimitsAndOptionalPercentiles(int workers, bool percentiles) + [TestCase(1, false, true)] + [TestCase(4, true, false)] + public void BenchmarkUsesDefaultPerWorkerLimitsAndOptionalPercentilesAndIndexes(int workers, bool percentiles, bool indexes) { - var options = DashboardAnalyticsOptions.CreateBenchmark(directory, TimeZoneInfo.Utc, workers, percentiles); + var options = DashboardAnalyticsOptions.CreateBenchmark(directory, TimeZoneInfo.Utc, workers, percentiles, indexes); var defaults = new AnalyticsOptions(); Assert.That(options.Ingestion.WorkerCount, Is.EqualTo(workers)); Assert.That(options.Ingestion.QueueCapacity, Is.EqualTo(defaults.Ingestion.QueueCapacity)); @@ -33,6 +33,22 @@ public void BenchmarkUsesDefaultPerWorkerLimitsAndOptionalPercentiles(int worker Assert.That(options.Sessions.MaxCachedSessions, Is.EqualTo(defaults.Sessions.MaxCachedSessions)); Assert.That(options.Ingestion.QueueFullBehavior, Is.EqualTo(QueueFullBehavior.Wait)); Assert.That(options.Rollups.ValueAggregates.ContainsKey("load_time"), Is.EqualTo(percentiles)); + Assert.That(options.Storage.DimensionIndexes.Dimensions, indexes ? Is.Not.Empty : Is.Empty); + Assert.That(options.Storage.RollupIndexes.Definitions, indexes ? Is.Not.Empty : Is.Empty); + } + + [Test] + public async Task ABenchmarkWithoutIndexesRetainsBrowsableReportsWithoutIndexes() + { + await using var benchmark = new SampleBenchmark(directory, TimeZoneInfo.Utc); + var result = await benchmark.RunAsync(new(17, 7, 2, Indexes: false), _ => { }, + CancellationToken.None, CancellationToken.None); + Assert.That(result.Indexes, Is.False); + Assert.That(result.Ingested, Is.EqualTo(17)); + using var lease = benchmark.Reports.Acquire(); + var report = lease.Engine.Web.GetSummary(new() { Site = BenchmarkReportStore.Site, Range = DateRange.LastDays(8) }); + Assert.That(report.Summary.PageViews, Is.GreaterThan(0)); + Assert.That(lease.Engine.RollupIndexes.AllStores, Is.Empty, "reopened reports build no indexes"); } [Test] diff --git a/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs b/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs index bf92e2d..c6d0f2c 100644 --- a/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs +++ b/src/ZoneTree.Analytics.UnitTests/StorageFormatRegressionTests.cs @@ -52,6 +52,21 @@ public void DirectoriesOfOtherStorageFormatsAreRefused(string key, string stored Assert.That(error.Message, Does.Contain(explanation)); } + [TestCase(true)] + [TestCase(false)] + public void DataWithoutMetadataIsRefused(bool olderMetadataFile) + { + using (var engine = TestEngine.Create(directory)) + View(engine, First); + var meta = Path.Combine(directory, "meta"); + foreach (var path in Directory.GetFiles(meta)) + File.Delete(path); + if (olderMetadataFile) + File.WriteAllText(Path.Combine(meta, "meta-a.json"), "zonetree-analytics-meta 1 00\n{}"); + var error = Assert.Throws(() => TestEngine.Create(directory).Dispose()); + Assert.That(error.Message, Does.Contain("holds data but no valid metadata")); + } + [Test] public void RolloverNeverHidesRecordsWrittenAfterItsClock() { diff --git a/src/ZoneTree.Analytics/AnalyticsEngine.cs b/src/ZoneTree.Analytics/AnalyticsEngine.cs index b4f93ce..a9415a3 100644 --- a/src/ZoneTree.Analytics/AnalyticsEngine.cs +++ b/src/ZoneTree.Analytics/AnalyticsEngine.cs @@ -362,6 +362,9 @@ internal void ReportIntegrationError(Exception exception) bool IsDisposed => Volatile.Read(ref disposed) != 0; + /// The directories of the data root that hold stores or checkpoints. + static readonly string[] DataDirectories = ["events", "rollups", "sessions", "indexes", "rollup-indexes", "backfill"]; + /// /// Persists the settings that define the storage format when the directory is created, and /// requires the same settings on reopen. Returns true for a new directory. @@ -379,6 +382,13 @@ bool ValidateConfiguration() ]; if (meta.Get(StorageFormats.DatasetKey) is not { } format) { + // Data without metadata is not a new directory: starting over would replace its visitor + // secret and salts and claim rollup coverage it does not have. + if (DataDirectories.Any(x => Directory.Exists(Path.Combine(dataRoot, x))) || + Directory.EnumerateFiles(Path.Combine(dataRoot, "meta")).Any()) + throw new AnalyticsConfigurationException( + $"The analytics data directory '{dataRoot}' holds data but no valid metadata: an older release wrote it, " + + "or its metadata files were removed. Start with a fresh analytics data directory."); meta.Update(entries => { foreach (var (key, value, _) in settings) @@ -899,12 +909,7 @@ void RemoveExpiredRecords(RetentionCutoffs cutoffs) continue; rollups.GetStore(granularity).RemoveExpired( cutoff, tree => ExpiredRecords.DeleteFirstOfExpiredHeads(tree, KeyCodec.RollupHead, key => retentionClock.IsRollupExpired(granularity, key))); - removedUpTo = granularity switch - { - Granularity.Minute => removedUpTo with { MinuteRollups = cutoff }, - Granularity.Hour => removedUpTo with { HourRollups = cutoff }, - _ => removedUpTo with { DayRollups = cutoffs.DayRollups, DayRollupsLocalDay = cutoff } - }; + removedUpTo = removedUpTo.WithRollup(granularity, cutoffs); } } diff --git a/src/ZoneTree.Analytics/AnalyticsStorage.cs b/src/ZoneTree.Analytics/AnalyticsStorage.cs index 3274a57..5a05c31 100644 --- a/src/ZoneTree.Analytics/AnalyticsStorage.cs +++ b/src/ZoneTree.Analytics/AnalyticsStorage.cs @@ -111,13 +111,7 @@ public IMaintainer SessionsMaintainer public AnalyticsPartitionedStorage, RollupValue> GetRollups(Granularity granularity) { ensureOpen(); - return rollups[granularity switch - { - Granularity.Minute => 0, - Granularity.Hour => 1, - Granularity.Day => 2, - _ => throw new ArgumentOutOfRangeException(nameof(granularity)) - }]; + return rollups[RollupStore.IndexOf(granularity)]; } } diff --git a/src/ZoneTree.Analytics/Ingestion/IngestionWorker.cs b/src/ZoneTree.Analytics/Ingestion/IngestionWorker.cs index 8092647..cad7389 100644 --- a/src/ZoneTree.Analytics/Ingestion/IngestionWorker.cs +++ b/src/ZoneTree.Analytics/Ingestion/IngestionWorker.cs @@ -117,8 +117,6 @@ sealed class IngestionWorker : IAsyncDisposable // keeps nothing. readonly SessionDimensions sessionDimensions; - readonly PageDimensions pageDimensions = new(); - readonly long sessionTimeoutTicks; readonly long flushIntervalTimestamp; @@ -468,15 +466,7 @@ void UpdateSession(EnrichedEvent ev, SessionState session, bool isNewSession, Li { valueChangesForSession.Clear(); valueChangesForSession.Add(new ValueChange(StandardValues.Exits, -1, IsDelta: true)); - buffer.Contribute( - ev.Site, - StandardEvents.PageView, - session.LastPageTicks, - pageDimensions.Of(dimensionsOfSession, session.LastPage, session.LastHostname), - 0, - 0, - valueChangesForSession, - DimensionScope.PageView); + buffer.ContributePageMark(ev.Site, session.LastPageTicks, dimensionsOfSession, session.LastPage, session.LastHostname, valueChangesForSession); } var isEntrance = isNewSession && session.Events == 1; if (ev.HasCustomDimensions) @@ -486,15 +476,7 @@ void UpdateSession(EnrichedEvent ev, SessionState session, bool isNewSession, Li valueChangesForSession.Add(new ValueChange(StandardValues.Exits, 1, IsDelta: true)); if (isEntrance) valueChangesForSession.Add(new ValueChange(StandardValues.Entrances, 1, IsDelta: true)); - buffer.Contribute( - ev.Site, - StandardEvents.PageView, - ev.Ticks, - pageDimensions.Of(dimensionsOfSession, ev.Page, ev.Hostname), - 0, - 0, - valueChangesForSession, - DimensionScope.PageView); + buffer.ContributePageMark(ev.Site, ev.Ticks, dimensionsOfSession, ev.Page, ev.Hostname, valueChangesForSession); } else { diff --git a/src/ZoneTree.Analytics/Ingestion/RollupBuffer.cs b/src/ZoneTree.Analytics/Ingestion/RollupBuffer.cs index 711fced..1816dd2 100644 --- a/src/ZoneTree.Analytics/Ingestion/RollupBuffer.cs +++ b/src/ZoneTree.Analytics/Ingestion/RollupBuffer.cs @@ -120,6 +120,8 @@ sealed class RollupBuffer readonly List<(string Dimension, ulong Hash)> eventDistinct = []; + readonly PageDimensions pageDimensions = new(); + // Per scope, per definition: whether contributions of the scope reach the definition. readonly bool[][] accepts; @@ -173,6 +175,15 @@ public Dictionary Swap() _ => dayCache.LocalDay(ticks) }; + /// + /// Adds an entrance or an exit to the page view rows of a page: the session's dimensions with + /// the page and its hostname, without counting a page view. + /// + public void ContributePageMark(string site, long ticks, IReadOnlyDictionary sessionDimensions, + string page, string hostname, List changes) => + Contribute(site, StandardEvents.PageView, ticks, pageDimensions.Of(sessionDimensions, page, hostname), 0, 0, + changes, DimensionScope.PageView); + /// /// Adds a contribution to every applicable rollup row. /// A rollup applies when the event name matches and every one of its dimensions is present. diff --git a/src/ZoneTree.Analytics/Internal/ByteWriter.cs b/src/ZoneTree.Analytics/Internal/ByteWriter.cs index 6d5618d..1ed75f2 100644 --- a/src/ZoneTree.Analytics/Internal/ByteWriter.cs +++ b/src/ZoneTree.Analytics/Internal/ByteWriter.cs @@ -126,8 +126,6 @@ public void WriteRawUtf8(string value) /// ref struct ByteReader { - static readonly UTF8Encoding StrictUtf8 = new(false, true); - readonly ReadOnlySpan span; readonly bool strict; @@ -194,7 +192,7 @@ public string ReadString() var byteCount = checked((int)(lengthPlusOne - 1)); if (byteCount == 0) return string.Empty; - var result = (strict ? StrictUtf8 : Encoding.UTF8).GetString(span.Slice(position, byteCount)); + var result = (strict ? Text.StrictUtf8 : Encoding.UTF8).GetString(span.Slice(position, byteCount)); position += byteCount; return result; } @@ -211,7 +209,7 @@ public string ReadString(Utf8StringCache cache) return null; var byteCount = checked((int)(lengthPlusOne - 1)); if (strict) - StrictUtf8.GetCharCount(span.Slice(position, byteCount)); + Text.StrictUtf8.GetCharCount(span.Slice(position, byteCount)); var result = cache.Get(span.Slice(position, byteCount)); position += byteCount; return result; diff --git a/src/ZoneTree.Analytics/Internal/Once.cs b/src/ZoneTree.Analytics/Internal/Once.cs new file mode 100644 index 0000000..e83577f --- /dev/null +++ b/src/ZoneTree.Analytics/Internal/Once.cs @@ -0,0 +1,18 @@ +namespace ZoneTree.Analytics.Internal; + +/// +/// Thread-safe lazy fields without locks: concurrent first reads may each create a value, and +/// all of them get the first one stored. +/// +static class Once +{ + /// The value of a field, stored from when the field is null. + public static T Get(ref T field, TState state, Func create) where T : class + { + var current = Volatile.Read(ref field); + if (current != null) + return current; + var created = create(state); + return Interlocked.CompareExchange(ref field, created, null) ?? created; + } +} diff --git a/src/ZoneTree.Analytics/Internal/RetentionCutoffs.cs b/src/ZoneTree.Analytics/Internal/RetentionCutoffs.cs index f90d879..8662b1e 100644 --- a/src/ZoneTree.Analytics/Internal/RetentionCutoffs.cs +++ b/src/ZoneTree.Analytics/Internal/RetentionCutoffs.cs @@ -72,6 +72,16 @@ static long Cutoff(TimeSpan? retention, long nowUtcTicks) public long RollupCutoff(Granularity granularity) => granularity == Granularity.Day ? DayRollupsLocalDay : Of(granularity); + /// + /// These cutoffs with the cutoff of a rollup granularity taken from . + /// + public RetentionCutoffs WithRollup(Granularity granularity, RetentionCutoffs source) => granularity switch + { + Granularity.Minute => this with { MinuteRollups = source.MinuteRollups }, + Granularity.Hour => this with { HourRollups = source.HourRollups }, + _ => this with { DayRollups = source.DayRollups, DayRollupsLocalDay = source.DayRollupsLocalDay } + }; + /// /// True when a rollup row is within the retention of its granularity. Minute and hour rows /// are compared by event time, day rows by their local day bucket. diff --git a/src/ZoneTree.Analytics/Internal/Text.cs b/src/ZoneTree.Analytics/Internal/Text.cs index d5b20fe..d1def31 100644 --- a/src/ZoneTree.Analytics/Internal/Text.cs +++ b/src/ZoneTree.Analytics/Internal/Text.cs @@ -7,7 +7,8 @@ namespace ZoneTree.Analytics.Internal; /// static class Text { - static readonly UTF8Encoding StrictUtf8 = new(false, true); + /// UTF-8 that throws on malformed input instead of replacing it. + public static readonly UTF8Encoding StrictUtf8 = new(false, true); /// Rejects names that cannot be represented unambiguously in storage keys. public static void ValidateStoredName(string value, string parameter) diff --git a/src/ZoneTree.Analytics/Storage/AlternatingFiles.cs b/src/ZoneTree.Analytics/Storage/AlternatingFiles.cs new file mode 100644 index 0000000..68f862f --- /dev/null +++ b/src/ZoneTree.Analytics/Storage/AlternatingFiles.cs @@ -0,0 +1,135 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; + +namespace ZoneTree.Analytics.Storage; + +/// +/// Two files written in turn, each holding a generation and the checksum of its body. A write +/// replaces the older file, so a crash can only damage the file being written; loading picks the +/// valid file with the highest generation. No rename and no directory flush are involved, which +/// makes the scheme equally durable on every platform. +/// +/// +/// File layout: magic(8) generation(8, LE) body length(8, LE) sha256 of body(32) body. +/// The header is written last, so a torn write never carries a matching checksum. +/// +sealed class AlternatingFiles +{ + const int HeaderLength = 56; + + readonly byte[] magic; + + readonly string[] paths; + + long generation; + + /// The file of even generations. + /// The file of odd generations. + /// The 8 bytes that start every file of this kind. + public AlternatingFiles(string pathA, string pathB, ReadOnlySpan magic) + { + this.magic = magic.ToArray(); + paths = [pathA, pathB]; + } + + /// Whether either file exists, valid or not. + public bool Exist => paths.Any(File.Exists); + + /// + /// Reads the body of the valid file with the highest generation; the default value when no + /// file is valid. Later writes go after that generation. + /// + public T Load(Func read) + { + string newest = null; + foreach (var path in paths) + { + if (!File.Exists(path)) + continue; + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16); + if (TryReadHeader(stream, out var fileGeneration) && fileGeneration > generation) + { + generation = fileGeneration; + newest = path; + } + } + if (newest == null) + return default; + using var body = new FileStream(newest, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16); + body.Position = HeaderLength; + return read(body); + } + + /// Writes the next generation over the older file and flushes it to disk. + public void Save(Action write) + { + var path = paths[(generation + 1) % 2]; + using (var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 16)) + { + stream.Write(new byte[HeaderLength]); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + using (var body = new HashingStream(stream, hash)) + write(body); + var length = stream.Position - HeaderLength; + var header = new byte[HeaderLength]; + magic.CopyTo(header, 0); + BinaryPrimitives.WriteInt64LittleEndian(header.AsSpan(8), generation + 1); + BinaryPrimitives.WriteInt64LittleEndian(header.AsSpan(16), length); + hash.GetHashAndReset().CopyTo(header, 24); + stream.Position = 0; + stream.Write(header); + stream.Flush(flushToDisk: true); + } + ++generation; + } + + /// Deletes both files. + public void Delete() + { + foreach (var path in paths) + File.Delete(path); + generation = 0; + } + + /// Validates the header and the checksum of the whole body. + bool TryReadHeader(FileStream stream, out long fileGeneration) + { + fileGeneration = 0; + var header = new byte[HeaderLength]; + if (stream.Length < HeaderLength || stream.ReadAtLeast(header, HeaderLength, throwOnEndOfStream: false) != HeaderLength || + !header.AsSpan(0, 8).SequenceEqual(magic)) + return false; + fileGeneration = BinaryPrimitives.ReadInt64LittleEndian(header.AsSpan(8)); + var length = BinaryPrimitives.ReadInt64LittleEndian(header.AsSpan(16)); + if (fileGeneration < 1 || length != stream.Length - HeaderLength) + return false; + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var buffer = new byte[1 << 16]; + int read; + while ((read = stream.Read(buffer)) > 0) + hash.AppendData(buffer, 0, read); + return hash.GetHashAndReset().AsSpan().SequenceEqual(header.AsSpan(24, 32)); + } + + /// Forwards writes to a stream and hashes them. + sealed class HashingStream(Stream inner, IncrementalHash hash) : Stream + { + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() => inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + + public override void Write(ReadOnlySpan buffer) + { + hash.AppendData(buffer); + inner.Write(buffer); + } + } +} diff --git a/src/ZoneTree.Analytics/Storage/BackgroundIndex.cs b/src/ZoneTree.Analytics/Storage/BackgroundIndex.cs new file mode 100644 index 0000000..be4df9f --- /dev/null +++ b/src/ZoneTree.Analytics/Storage/BackgroundIndex.cs @@ -0,0 +1,255 @@ +using ZoneTree.Analytics.Internal; +using ZoneTree.Logger; + +namespace ZoneTree.Analytics.Storage; + +/// +/// The lifecycle of optional index copies of authoritative data, which background index +/// maintenance builds and repairs while the engine ingests and answers queries. +/// +/// +/// +/// Opening decides whether the copies can be reused, need a repair of the days whose writes may +/// have been interrupted, or need a rebuild. Only an interrupted build of the same configuration +/// resumes from its checkpoint, and then repairs the days written since it began; any other +/// reason drops the copies, before ingestion writes new ones, and builds them again. +/// +/// +/// A completed build, and an orderly shutdown, record the coverage: format, configuration, +/// partitions and a clean checkpoint. Once a copy fails, the copies after it are skipped: no +/// checkpoint or coverage may claim them, so the build stops until the engine opens again. +/// +/// +abstract class BackgroundIndex : IDisposable +{ + protected const string UncleanReason = "missing or unclean checkpoint"; + + readonly string stage; + readonly string cleanKey; + readonly string configurationKey; + readonly string checkpointPrefix; + readonly string formatKey; + readonly int format; + readonly string formatDescription; + string manifest; + Dictionary dirtyDays = []; + int invalid; + long revision; + volatile bool ready; + + /// The engine metadata. + /// Receives the stage logs. + /// The directory of the index stores. + /// The name that starts the stage logs, such as "Dimension index". + /// The prefix of the clean and checkpoint metadata keys. + /// The metadata key of the configuration manifest. + /// The metadata key of the index format. + /// The current index format. + /// What the format describes, for its errors. + protected BackgroundIndex(MetaStore meta, ILogger logger, string root, string stage, string keyPrefix, + string configurationKey, string formatKey, int format, string formatDescription) + { + Meta = meta; + Logger = logger; + Root = root; + this.stage = stage; + cleanKey = keyPrefix + ".clean"; + checkpointPrefix = keyPrefix + ".backfill"; + this.configurationKey = configurationKey; + this.formatKey = formatKey; + this.format = format; + this.formatDescription = formatDescription; + } + + protected MetaStore Meta { get; } + + protected ILogger Logger { get; } + + /// The directory of the index stores. + protected string Root { get; } + + /// The stores and other resources the index owns. + protected List Resources { get; } = []; + + /// Whether opening decided to rebuild the copies. + protected bool Rebuilding { get; private set; } + + /// The days to repair, by intent journal; empty when there is no repair. + protected IReadOnlyDictionary RepairDays { get; private set; } = new Dictionary(); + + /// Whether a build or a repair has completed, or none was needed. + protected bool IsReady => ready; + + /// Changes whenever queries must not keep reading the copies they selected. + protected long Revision => Interlocked.Read(ref revision); + + public bool IsHealthy => Volatile.Read(ref invalid) == 0; + + /// Whether the copies are being built or repaired; queries read the authoritative data meanwhile. + public bool IsBuilding => IndexCount > 0 && !ready && IsHealthy; + + /// The number of configured logical indexes. + protected abstract int IndexCount { get; } + + /// The number of physical index stores. + protected abstract int StoreCount { get; } + + /// Why the index is disabled, for the logs. + protected abstract string DisabledReason { get; } + + /// Whether both the source and the copies have a write-ahead log. + protected abstract bool Durable { get; } + + /// The intents of the days whose writes may have been interrupted. + protected abstract IEnumerable Journals { get; } + + /// Whether there is no source data to copy, so a new build completes at once. + protected abstract bool NothingToCopy { get; } + + /// + /// Opens the stores of the configured indexes and drops those of removed ones. Adds the + /// reasons to rebuild that the stores show, and returns their number of partitions. + /// + protected abstract int OpenStores(List reasons); + + /// Drops every copy, before a new build. + protected abstract void DropCopies(); + + /// Records the partitions of the stores in the coverage. + protected abstract void SavePartitions(Dictionary entries); + + /// The failure of a build that cannot succeed until the engine opens again. + protected abstract AnalyticsStorageException BuildFailed(Exception cause); + + /// Runs the build or repair that opening decided, on the background maintenance thread. + public abstract void Build(CancellationToken cancellationToken); + + /// Whether an interrupted build may resume despite a reason to rebuild. + protected virtual bool Resumes(string reason) => reason == UncleanReason; + + /// Runs when a build completes, before its coverage is recorded. + protected virtual void Completing() + { + } + + /// Marks the copies unusable until the engine opens again. + public void Invalidate() + { + Volatile.Write(ref invalid, 1); + Interlocked.Increment(ref revision); + } + + /// Makes queries that selected the copies stop reading them. + protected void ChangeRevision() => Interlocked.Increment(ref revision); + + /// + /// Opens the index and decides its work, which then runs in the + /// background. A new build drops the old copies first. + /// + /// The configuration of the indexes. + /// Reasons to rebuild that only the store knows. + protected void Open(string manifest, IEnumerable reasons) + { + this.manifest = manifest; + using var startup = new StageLog(Logger, stage + " initialization", Root); + using var check = new StageLog(Logger, stage + " backfill check", Root); + var stored = Meta.Get(formatKey); + if (stored != null) + StorageFormats.Require(stored, format, formatDescription); + var found = new List(); + // Indexes that were never initialized have no checkpoint, manifest or format to compare. + if (stored == null) found.Add("new indexes"); + else + { + if (Meta.Get(cleanKey) != "true") found.Add(UncleanReason); + if (Meta.Get(configurationKey) != manifest) found.Add("configuration changed"); + } + if (!Durable) found.Add("write-ahead logging disabled"); + found.AddRange(reasons); + var partitions = OpenStores(found); + Rebuilding = found.Count > 0 && IndexCount > 0; + var checkpoint = Checkpoint(); + // Only an interrupted build of the same configuration resumes; any other reason starts over. + var resuming = Rebuilding && checkpoint.Copying && found.All(Resumes); + dirtyDays = Journals.ToDictionary(journal => journal, journal => journal.Days); + // A new build copies every day; one that resumes repairs the days written since it began. + // Without indexes there is nothing to repair: their intents are cleared. + RepairDays = IndexCount == 0 || (Rebuilding && !resuming) + ? new Dictionary() : dirtyDays.Where(pair => pair.Value.Length > 0).ToDictionary(); + var dirty = dirtyDays.Values.Sum(days => days.Length); + var decision = IndexCount == 0 ? "disabled" : Rebuilding ? "rebuild required" : dirty > 0 ? "repair affected days" : "reuse existing indexes"; + var reason = IndexCount == 0 ? DisabledReason : Rebuilding ? string.Join(", ", found) : dirty > 0 ? "durable write intents require reconciliation" : "clean checkpoint and matching manifests"; + check.Complete($"decision={decision}; indexes={IndexCount}; physical stores={StoreCount}; partitions={partitions}; dirty days={dirty}; resuming={resuming}; reasons={reason}"); + if (Rebuilding && !resuming) + { + Meta.Set(cleanKey, "false"); + checkpoint.Begin(); + DropCopies(); + // An interrupted build of the new copies resumes at the next open, while they stay unclean. + Meta.Update(entries => + { + SaveCoverage(entries); + entries[cleanKey] = "false"; + }); + checkpoint.Cleared(); + if (NothingToCopy) + Build(CancellationToken.None); + } + if (!Rebuilding && RepairDays.Count == 0) + { + ready = true; + MarkClean(); + } + startup.Complete($"decision={(ready ? IndexCount == 0 ? "disabled" : Rebuilding ? "rebuild completed" : "reuse existing indexes" : Rebuilding ? "rebuild in background" : "repair in background")}; indexes={IndexCount}; physical stores={StoreCount}"); + } + + protected IndexBackfillCheckpoint Checkpoint() => new(Meta, checkpointPrefix, manifest); + + /// Stops a build once a copy has failed. + protected void RequireHealthy() + { + if (!IsHealthy) + throw BuildFailed(null); + } + + /// Records the coverage of a completed build, and makes the copies available to queries. + protected void Complete(IndexBackfillCheckpoint checkpoint) + { + RequireHealthy(); + Completing(); + Meta.Update(entries => + { + SaveCoverage(entries); + checkpoint.Clear(entries); + }); + foreach (var (journal, days) in dirtyDays) + journal.ClearRepaired(days); + ready = true; + Interlocked.Increment(ref revision); + } + + /// + /// Records the coverage at an orderly shutdown, when the copies are complete: no day needs a + /// repair then. An unfinished build keeps its checkpoint and resumes at the next open. + /// + public void MarkClean() + { + if (!ready || !IsHealthy) return; + Meta.Update(entries => + { + SaveCoverage(entries); + foreach (var journal in Journals) journal.Clear(entries); + Checkpoint().Clear(entries); + }); + } + + void SaveCoverage(Dictionary entries) + { + entries[cleanKey] = Durable ? "true" : "false"; + entries[formatKey] = StorageFormats.Text(format); + entries[configurationKey] = manifest; + SavePartitions(entries); + } + + public void Dispose() => Disposal.DisposeAll(Resources); +} diff --git a/src/ZoneTree.Analytics/Storage/DimensionIndexScan.cs b/src/ZoneTree.Analytics/Storage/DimensionIndexScan.cs index 83deb60..c335829 100644 --- a/src/ZoneTree.Analytics/Storage/DimensionIndexScan.cs +++ b/src/ZoneTree.Analytics/Storage/DimensionIndexScan.cs @@ -1,12 +1,11 @@ using ZoneTree.Analytics.Internal; +using Slice = ZoneTree.Analytics.Storage.RawEventStore.Slice; namespace ZoneTree.Analytics.Storage; /// Ordered covering reads, including k-way merging of distinct equality values. static class DimensionIndexScan { - readonly record struct Slice(IZoneTree, Memory> Tree, long From, long To); - sealed class Cursor : IDisposable { readonly IZoneTreeIterator, Memory> iterator; @@ -76,7 +75,7 @@ public static TState[] Scan(DimensionIndexStore owner, DimensionIndexSel leases.Add(lease); ranges.Add(new(lease.Tree, Math.Max(from, partition.From), Math.Min(to, partition.To))); } - var slices = parallelism == 1 ? ranges : Split(ranges, parallelism * 4); + var slices = parallelism == 1 ? ranges : RawEventStore.Split(ranges, parallelism * 4); var prefixes = selection.Values.Select(value => DimensionIndexKeyCodec.Prefix(selection.Index.KeyPrefix, site, value)).ToArray(); var states = new TState[slices.Count]; ParallelWork.For(slices.Count, parallelism, i => @@ -92,25 +91,6 @@ public static TState[] Scan(DimensionIndexStore owner, DimensionIndexSel finally { foreach (var lease in leases) lease.Dispose(); } } - static List Split(List ranges, int target) - { - var total = ranges.Sum(x => x.To - x.From); - var slices = new List(); - foreach (var range in ranges) - { - var length = range.To - range.From; - var count = (int)Math.Clamp(Math.Round((double)target * length / total), 1, Math.Max(1, length / TimeSpan.TicksPerMinute)); - var from = range.From; - for (var i = 1; i <= count; ++i) - { - var to = i == count ? range.To : range.From + (long)((Int128)length * i / count); - slices.Add(range with { From = from, To = to }); - from = to; - } - } - return slices; - } - static void Read(Slice slice, byte[][] prefixes, TState state, RawEventStore.SliceVisitor visit, CancellationToken cancellationToken) { diff --git a/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs b/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs index f455ae4..8c122ae 100644 --- a/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs +++ b/src/ZoneTree.Analytics/Storage/DimensionIndexStore.cs @@ -1,4 +1,3 @@ -using ZoneTree.Analytics.Internal; using ZoneTree.Logger; namespace ZoneTree.Analytics.Storage; @@ -20,193 +19,116 @@ sealed class DimensionIndexUnavailableException : Exception; /// /// Builds and repairs run on the background maintenance thread while ingestion writes its own /// copies: copying an event again writes the same key and value. Queries scan raw events until -/// the indexes are ready. A build resumes from its checkpoint after a restart; the days that -/// ingestion wrote meanwhile are repaired. Erasure and the build's chunks hold -/// one at a time, and every chunk reads the raw events again, so a build never copies an -/// erased event back. +/// the indexes are ready. Erasure and the build's chunks hold one at a time, +/// and every chunk reads the raw events again, so a build never copies an erased event back. /// -sealed class DimensionIndexStore : IDisposable +sealed class DimensionIndexStore : BackgroundIndex { - const string DimensionsKey = "dimension-index.dimensions"; - const string CleanKey = "dimension-index.clean"; const string PartitionsKey = "dimension-index.partitions.shared"; - const string UncleanReason = "missing or unclean checkpoint"; const int ProbeLimit = 1024; const int ChunkSize = 4096; - readonly MetaStore meta; - readonly string manifest; - readonly string directory; readonly DimensionIndexOptions options; readonly RawEventStore rawEvents; readonly StoreFactory factory; - readonly ILogger logger; readonly IndexRecovery recovery; readonly List indexes = []; - readonly List resources = []; - int invalid; int suspended; - long revision; - volatile bool ready; - bool rebuild; long fence; - long[] dirtyDays = []; - long[] repairDays = []; public IReadOnlyList Indexes => indexes; public PartitionedStore, Memory> Store { get; private set; } public IEnumerable, Memory>> AllStores => Store == null ? [] : [Store]; - public bool IsAvailable => ready && Volatile.Read(ref invalid) == 0 && Volatile.Read(ref suspended) == 0; - public bool IsHealthy => Volatile.Read(ref invalid) == 0; - - /// Whether the indexes are being built or repaired; queries scan raw events meanwhile. - public bool IsBuilding => indexes.Count > 0 && !ready && IsHealthy; + public bool IsAvailable => IsReady && IsHealthy && Volatile.Read(ref suspended) == 0; /// Held by erasure and by each chunk of a build, so that a build never copies an erased event. public object Gate { get; } = new(); public DimensionIndexStore(string directory, DimensionIndexOptions options, RawEventStore rawEvents, StoreFactory factory, MetaStore meta, ILogger logger) + : base(meta, logger, directory, "Dimension index", "dimension-index", "dimension-index.dimensions", + StorageFormats.DimensionIndexKey, StorageFormats.DimensionIndex, "The dimension indexes of the analytics data directory") { - this.meta = meta; - this.directory = directory; this.options = options; this.rawEvents = rawEvents; this.factory = factory; - this.logger = logger; recovery = new(meta, "dimension-index.dirty."); - manifest = string.Join('\n', options.Dimensions.Order(StringComparer.Ordinal)); } /// - /// Opens the indexes and decides whether they need a build or a repair, which - /// then runs in the background. A new build drops the old copies first, - /// before ingestion writes new ones. Builds copy the raw events up to , - /// the last sequence stored before ingestion starts: ingestion copies the later ones. + /// Opens the indexes and decides their work. Builds copy the raw events up to + /// , the last sequence stored before ingestion starts: ingestion copies + /// the later ones. /// public void Initialize(long fence) { this.fence = fence; - using var startup = new StageLog(logger, "Dimension index initialization", directory); - using var check = new StageLog(logger, "Dimension index backfill check", directory); - var format = meta.Get(StorageFormats.DimensionIndexKey); - if (format != null) - StorageFormats.Require(format, StorageFormats.DimensionIndex, "The dimension indexes of the analytics data directory"); - var reasons = new List(); - // Indexes that were never initialized have no checkpoint, manifest or format to compare. - if (format == null) reasons.Add("new indexes"); - else - { - if (meta.Get(CleanKey) != "true") reasons.Add(UncleanReason); - if (meta.Get(DimensionsKey) != manifest) reasons.Add("configuration changed"); - } - if (!factory.RawEventsDurable || !factory.DimensionIndexesDurable) reasons.Add("write-ahead logging disabled"); - dirtyDays = recovery.Days; - var partitions = 0; - if (options.Dimensions.Count > 0) - { - Store = Open(); - foreach (var dimension in options.Dimensions) indexes.Add(new(dimension, Store)); - var names = Store.ListPartitions(); - partitions = names.Count; - var expected = (meta.Get(PartitionsKey) ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries); - if (IndexRecoveryScan.MissingRetainedPartitions(expected, names, factory.RawRetentionCutoff)) - reasons.Add("partition manifest changed"); - } - rebuild = reasons.Count > 0 && indexes.Count > 0; - var checkpoint = Checkpoint(); - // Only an interrupted build of the same configuration resumes; any other reason starts over. - var resuming = rebuild && checkpoint.Copying && reasons.All(reason => reason == UncleanReason); - // A new build copies every day; one that resumes repairs the days written since it began. - repairDays = rebuild && !resuming ? [] : dirtyDays; - var decision = indexes.Count == 0 ? "disabled" : rebuild ? "rebuild required" : dirtyDays.Length > 0 ? "repair affected days" : "reuse existing indexes"; - var reason = indexes.Count == 0 ? "no configured dimensions" : rebuild ? string.Join(", ", reasons) : dirtyDays.Length > 0 ? "durable write intents require reconciliation" : "clean checkpoint and matching manifests"; - check.Complete($"decision={decision}; indexes={indexes.Count}; physical stores={(Store == null ? 0 : 1)}; partitions={partitions}; dirty days={dirtyDays.Length}; resuming={resuming}; reasons={reason}"); - // Configuration changes rebuild the shared tree, removing discarded namespaces. - // Disabling every dimension drops its partitions; interrupted cleanup is retried. - if (Store == null && Directory.Exists(directory)) - { - var obsolete = Open(); - foreach (var partition in obsolete.ListPartitions()) obsolete.Drop(partition); - } - if (rebuild && !resuming) - { - meta.Set(CleanKey, "false"); - checkpoint.Begin(); - foreach (var partition in Store.ListPartitions()) Store.Drop(partition); - Started(checkpoint); - // A new directory has nothing to copy: the build completes at once. - if (rawEvents.Store.ListPartitions().Count == 0) - Build(CancellationToken.None); - } - if (!rebuild && repairDays.Length == 0) - { - ready = true; - MarkClean(); - } - startup.Complete($"decision={(ready ? rebuild ? "rebuild completed" : decision : rebuild ? "rebuild in background" : "repair in background")}; indexes={indexes.Count}; physical stores={(Store == null ? 0 : 1)}"); + Open(string.Join('\n', options.Dimensions.Order(StringComparer.Ordinal)), []); } - IndexBackfillCheckpoint Checkpoint() => new(meta, "dimension-index.backfill", manifest); + protected override int IndexCount => indexes.Count; - /// - /// Stops a build once a copy has failed: the copies after it were skipped, so no checkpoint - /// or coverage may claim them. - /// - void RequireHealthy() - { - if (!IsHealthy) - throw Failed(null); - } + protected override int StoreCount => Store == null ? 0 : 1; - /// The failure of a build that cannot succeed until the engine opens again. - static AnalyticsStorageException Failed(Exception cause) => new( - "The dimension indexes could not be built" + (cause == null ? ": a copy failed." : ": " + cause.Message) + - " Queries scan raw events without them until the engine opens again, which resumes the build.", cause); + protected override string DisabledReason => "no configured dimensions"; + + protected override bool Durable => factory.RawEventsDurable && factory.DimensionIndexesDurable; + + protected override IEnumerable Journals => [recovery]; - /// Records that the indexes cover the raw events, and makes them available to queries. - void Complete(IndexBackfillCheckpoint checkpoint) + protected override bool NothingToCopy => rawEvents.Store.ListPartitions().Count == 0; + + protected override int OpenStores(List reasons) { - RequireHealthy(); - meta.Update(entries => + if (options.Dimensions.Count == 0) { - SaveCoverage(entries); - checkpoint.Clear(entries); - }); - recovery.ClearRepaired(dirtyDays); - ready = true; - Interlocked.Increment(ref revision); + // Disabling every dimension drops its partitions; interrupted cleanup is retried. + if (Directory.Exists(Root)) + { + var obsolete = Open(); + foreach (var partition in obsolete.ListPartitions()) obsolete.Drop(partition); + } + return 0; + } + // One shared tree holds every dimension: a configuration change rebuilds it, removing + // discarded namespaces. + Store = Open(); + foreach (var dimension in options.Dimensions) indexes.Add(new(dimension, Store)); + var names = Store.ListPartitions(); + var expected = (Meta.Get(PartitionsKey) ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries); + if (IndexRecoveryScan.MissingRetainedPartitions(expected, names, factory.RawRetentionCutoff)) + reasons.Add("partition manifest changed"); + return names.Count; } - /// - /// Records the configuration and format of a build that has dropped the old copies: an - /// interrupted build of them resumes at the next open, while the indexes stay unclean. - /// - void Started(IndexBackfillCheckpoint checkpoint) + protected override void DropCopies() { - meta.Update(entries => - { - SaveCoverage(entries); - entries[CleanKey] = "false"; - }); - checkpoint.Cleared(); + foreach (var partition in Store.ListPartitions()) Store.Drop(partition); } + protected override void SavePartitions(Dictionary entries) => + entries[PartitionsKey] = Store == null ? "" : string.Join(',', Store.ListPartitions()); + + protected override AnalyticsStorageException BuildFailed(Exception cause) => new( + "The dimension indexes could not be built" + (cause == null ? ": a copy failed." : ": " + cause.Message) + + " Queries scan raw events without them until the engine opens again, which resumes the build.", cause); + /// /// Runs the build or repair that decided, on the background /// maintenance thread, then makes the indexes available to queries. Every attempt resumes /// from the checkpoint, so a failed attempt can simply be repeated. /// - public void Build(CancellationToken cancellationToken) + public override void Build(CancellationToken cancellationToken) { - if (ready) return; + if (IsReady) return; // Indexes that failed wait for the next open, which resumes them from their last checkpoint. RequireHealthy(); - using var build = new StageLog(logger, "Dimension index backfill", directory); + using var build = new StageLog(Logger, "Dimension index backfill", Root); var checkpoint = Checkpoint(); + var repairDays = RepairDays.GetValueOrDefault(recovery) ?? []; var record = new RawEvent(); var selector = new DimensionSelector(options.Dimensions); - long records = rebuild ? checkpoint.Count : 0; + long records = Rebuilding ? checkpoint.Count : 0; var resumed = records; long rows = 0; long bytes = 0; @@ -230,7 +152,7 @@ void Copy(Memory key, Memory value) catch (Exception exception) { // The writer has invalidated the indexes: no retry in this run can complete them. - throw Failed(exception); + throw BuildFailed(exception); } rows += copies; bytes += (long)copies * value.Length; @@ -257,7 +179,7 @@ void Save(string partition, Memory key) RequireHealthy(); checkpoint.Save(partition, key, records); } - if (rebuild) + if (Rebuilding) { var resumePartition = checkpoint.Partition; var resumeKey = checkpoint.Key; @@ -310,10 +232,10 @@ void Save(string partition, Memory key) PartitionedStore, Memory> Open() { - var store = new PartitionedStore, Memory>(directory, - options.RecordsPerPartition, factory.OpenDimensionIndex, logger, + var store = new PartitionedStore, Memory>(Root, + options.RecordsPerPartition, factory.OpenDimensionIndex, Logger, factory.CreateDimensionIndexMaintainer, factory.WriteAheadLogDirectoryOf); - resources.Add(store); + Resources.Add(store); return store; } @@ -324,16 +246,10 @@ public void BeforeWrite(long ticks) public DimensionIndexWriter CreateWriter() => new(this); - public void Invalidate() - { - Volatile.Write(ref invalid, 1); - Interlocked.Increment(ref revision); - } - public IDisposable Suspend() { Interlocked.Increment(ref suspended); - Interlocked.Increment(ref revision); + ChangeRevision(); return new Suspension(this); } @@ -344,19 +260,19 @@ public void Dispose() { var released = Interlocked.Exchange(ref current, null); if (released == null) return; - Interlocked.Increment(ref released.revision); + released.ChangeRevision(); Interlocked.Decrement(ref released.suspended); } } public void Check(DimensionIndexSelection selection) { - if (!IsAvailable || selection.Revision != Interlocked.Read(ref revision)) + if (!CanUse(selection)) throw new DimensionIndexUnavailableException(); } public bool CanUse(DimensionIndexSelection selection) => - IsAvailable && selection.Revision == Interlocked.Read(ref revision); + IsAvailable && selection.Revision == Revision; /// The query plan notice of a scan that could not read the index of a dimension yet. public static string UnbuiltNotice(string dimension) => @@ -386,7 +302,7 @@ public DimensionIndexSelection Select(string site, long from, long to, IEnumerab Func allowed, CancellationToken cancellationToken) { if (indexes.Count == 0 || !IsAvailable) return null; - var currentRevision = Interlocked.Read(ref revision); + var currentRevision = Revision; DimensionIndexSelection best = null; var bestCount = int.MaxValue; foreach (var filter in Mandatory(filters)) @@ -400,7 +316,7 @@ public DimensionIndexSelection Select(string site, long from, long to, IEnumerab if (count < bestCount) { best = selection; bestCount = count; } if (count == 0) break; } - return IsAvailable && currentRevision == Interlocked.Read(ref revision) ? best : null; + return IsAvailable && currentRevision == Revision ? best : null; } internal static IEnumerable Mandatory(IEnumerable filters) @@ -434,29 +350,4 @@ internal static bool TryEquality(Filter filter, out string dimension, out string values = [.. combined]; return true; } - - /// - /// Records that the indexes cover the raw events, when they do: after an orderly shutdown no - /// day needs a repair. An unfinished build keeps its checkpoint and resumes at the next open. - /// - public void MarkClean() - { - if (!ready || Volatile.Read(ref invalid) != 0) return; - meta.Update(entries => - { - SaveCoverage(entries); - recovery.Clear(entries); - Checkpoint().Clear(entries); - }); - } - - void SaveCoverage(Dictionary entries) - { - entries[DimensionsKey] = manifest; - entries[StorageFormats.DimensionIndexKey] = StorageFormats.Text(StorageFormats.DimensionIndex); - entries[PartitionsKey] = Store == null ? "" : string.Join(',', Store.ListPartitions()); - entries[CleanKey] = factory.RawEventsDurable && factory.DimensionIndexesDurable ? "true" : "false"; - } - - public void Dispose() => Disposal.DisposeAll(resources); } diff --git a/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs b/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs index af047c6..5810870 100644 --- a/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs +++ b/src/ZoneTree.Analytics/Storage/JointBackfillCheckpoint.cs @@ -1,5 +1,3 @@ -using System.Buffers.Binary; -using System.Security.Cryptography; using System.Text; using ZoneTree.Analytics.Query; @@ -41,23 +39,12 @@ sealed class JointBackfillReplacement /// a rebuilt part is apart from what ingestion writes, so replaying it never undoes a flush. /// /// -/// The journal is binary: a flush of replacements holds serialized rollup rows, whose dense -/// sketches are 16 KiB each. Two files are written alternately, like , -/// so a crash can only damage the file being written; loading picks the valid file with the -/// highest generation. File layout: -/// -/// magic(8) generation(8, LE) body length(8, LE) sha256 of body(32) body -/// +/// The journal is binary, in : a flush of replacements holds +/// serialized rollup rows, whose dense sketches are 16 KiB each. /// sealed class JointBackfillCheckpoint { - const int HeaderLength = 56; - - static readonly byte[] Magic = "ZTAJRNL1"u8.ToArray(); - - readonly string[] paths; - - long generation; + readonly AlternatingFiles files; public JointBackfillState State { get; } public bool Resuming { get; } @@ -71,8 +58,12 @@ public JointBackfillCheckpoint(string root, string configuration, Func + { + using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true); + return Read(reader); + }); Resuming = resume && loaded?.Configuration == configuration; if (Resuming) State = loaded; @@ -84,28 +75,11 @@ public JointBackfillCheckpoint(string root, string configuration, Func files.Save(stream => { - var path = paths[(generation + 1) % 2]; - using (var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 16)) - { - // The header is written last: a torn write never carries a matching checksum. - stream.Write(new byte[HeaderLength]); - using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); - using (var body = new BinaryWriter(new HashingStream(stream, hash), Encoding.UTF8, leaveOpen: true)) - Write(body, State); - var length = stream.Position - HeaderLength; - var header = new byte[HeaderLength]; - Magic.CopyTo(header, 0); - BinaryPrimitives.WriteInt64LittleEndian(header.AsSpan(8), generation + 1); - BinaryPrimitives.WriteInt64LittleEndian(header.AsSpan(16), length); - hash.GetHashAndReset().CopyTo(header, 24); - stream.Position = 0; - stream.Write(header); - stream.Flush(flushToDisk: true); - } - ++generation; - } + using var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true); + Write(writer, State); + }); /// /// Writes the rebuilt parts of the checkpoint, then commits that they are written. @@ -126,51 +100,7 @@ public void Replay(RollupStore rollups) Save(); } - public void Complete() - { - foreach (var path in paths) - File.Delete(path); - generation = 0; - } - - JointBackfillState Load() - { - JointBackfillState newest = null; - foreach (var path in paths) - { - if (!File.Exists(path)) - continue; - using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16); - if (TryReadHeader(stream, out var fileGeneration) && fileGeneration > generation) - { - stream.Position = HeaderLength; - using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true); - newest = Read(reader); - generation = fileGeneration; - } - } - return newest; - } - - /// Validates the header and the checksum of the whole body. - static bool TryReadHeader(FileStream stream, out long fileGeneration) - { - fileGeneration = 0; - var header = new byte[HeaderLength]; - if (stream.Length < HeaderLength || stream.ReadAtLeast(header, HeaderLength, throwOnEndOfStream: false) != HeaderLength || - !header.AsSpan(0, 8).SequenceEqual(Magic)) - return false; - fileGeneration = BinaryPrimitives.ReadInt64LittleEndian(header.AsSpan(8)); - var length = BinaryPrimitives.ReadInt64LittleEndian(header.AsSpan(16)); - if (fileGeneration < 1 || length != stream.Length - HeaderLength) - return false; - using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); - var buffer = new byte[1 << 16]; - int read; - while ((read = stream.Read(buffer)) > 0) - hash.AppendData(buffer, 0, read); - return hash.GetHashAndReset().AsSpan().SequenceEqual(header.AsSpan(24, 32)); - } + public void Complete() => files.Delete(); static void Write(BinaryWriter writer, JointBackfillState state) { @@ -321,26 +251,4 @@ static byte[] ReadBytes(BinaryReader reader) var length = reader.ReadInt32(); return length < 0 ? null : reader.ReadBytes(length); } - - /// Forwards writes to a stream and hashes them. - sealed class HashingStream(Stream inner, IncrementalHash hash) : Stream - { - public override bool CanRead => false; - public override bool CanSeek => false; - public override bool CanWrite => true; - public override long Length => throw new NotSupportedException(); - public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } - public override void Flush() => inner.Flush(); - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - public override void SetLength(long value) => throw new NotSupportedException(); - - public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); - - public override void Write(ReadOnlySpan buffer) - { - hash.AppendData(buffer); - inner.Write(buffer); - } - } } diff --git a/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs b/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs index bf94130..f544031 100644 --- a/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs +++ b/src/ZoneTree.Analytics/Storage/JointRollupReconstruction.cs @@ -162,7 +162,6 @@ public void Run(CancellationToken cancellationToken) var record = new RawEvent(); var dimensions = new Dictionary(StringComparer.Ordinal); var changes = new List(); - var pageDimensions = new PageDimensions(); var collector = new RawSessionReader(rawEvents, schema, options); var timeout = options.Sessions.Timeout.Ticks; var resumed = state.Events; @@ -205,8 +204,7 @@ void ContributePageMark(string site, RawSession session, long ticks, string page return; changes.Clear(); changes.Add(new(value, 1, true)); - buffer.Contribute(site, StandardEvents.PageView, ticks, - pageDimensions.Of(session.Dimensions, page, hostname), 0, 0, changes, DimensionScope.PageView); + buffer.ContributePageMark(site, ticks, session.Dimensions, page, hostname, changes); } // One chronological stream per site keeps only sessions within an inactivity window. // Original session IDs/starts preserve attribution; events are never re-ingested or re-enriched. diff --git a/src/ZoneTree.Analytics/Storage/MetaStore.cs b/src/ZoneTree.Analytics/Storage/MetaStore.cs index c48d9b1..417be65 100644 --- a/src/ZoneTree.Analytics/Storage/MetaStore.cs +++ b/src/ZoneTree.Analytics/Storage/MetaStore.cs @@ -1,7 +1,3 @@ -using System.Buffers; -using System.Globalization; -using System.Security.Cryptography; -using System.Text; using System.Text.Json; namespace ZoneTree.Analytics.Storage; @@ -11,35 +7,28 @@ namespace ZoneTree.Analytics.Storage; /// times and the raw event sequence. A few kilobytes of string pairs, changed rarely. /// /// -/// -/// The entries are kept in memory and every change persists all of them. Two files are written -/// alternately, each in place and flushed to disk, and each records a generation number and a -/// checksum. A write always replaces the older file, so a crash can only damage the file being -/// written; loading picks the valid file with the highest generation. No rename and no directory -/// flush are involved, which makes the scheme equally durable on every platform. -/// -/// -/// File layout: a header line zonetree-analytics-meta {generation} {sha256 of the rest} -/// followed by the entries as an indented JSON object with sorted keys. -/// +/// The entries are kept in memory and every change persists all of them, as an indented JSON +/// object with sorted keys in . /// sealed class MetaStore { - const string Header = "zonetree-analytics-meta"; - readonly object sync = new(); - readonly string[] paths; + readonly AlternatingFiles files; Dictionary entries; - long generation; - public MetaStore(string directory) { Directory.CreateDirectory(directory); - paths = [Path.Combine(directory, "meta-a.json"), Path.Combine(directory, "meta-b.json")]; - (entries, generation) = Load(); + files = new(Path.Combine(directory, "meta-a.bin"), Path.Combine(directory, "meta-b.bin"), "ZTAMETA1"u8); + entries = files.Load(Read); + // Files exist but none is readable: refuse to start over, which would replace the visitor + // secret and salts and reset the storage format settings of a directory that holds data. + if (entries == null && files.Exist) + throw new AnalyticsConfigurationException( + $"The analytics metadata in '{directory}' is damaged: no metadata file is valid."); + entries ??= new(StringComparer.Ordinal); } public string Get(string key) @@ -83,80 +72,28 @@ public void Update(Action> change) if (!removed && next.Count == entries.Count && next.All(x => entries.TryGetValue(x.Key, out var value) && value == x.Value)) return; - WriteNextGeneration(next); + files.Save(stream => Write(stream, next)); entries = next; if (removed) - WriteNextGeneration(next); + files.Save(stream => Write(stream, next)); } } - void WriteNextGeneration(Dictionary content) + static Dictionary Read(Stream stream) { - Write(paths[(generation + 1) % 2], generation + 1, content); - ++generation; + var result = new Dictionary(StringComparer.Ordinal); + using var document = JsonDocument.Parse(stream); + foreach (var property in document.RootElement.EnumerateObject()) + result[property.Name] = property.Value.GetString(); + return result; } - (Dictionary Entries, long Generation) Load() + static void Write(Stream stream, Dictionary entries) { - var found = false; - (Dictionary Entries, long Generation) newest = (new(StringComparer.Ordinal), 0); - foreach (var path in paths) - { - if (!File.Exists(path)) - continue; - found = true; - if (TryRead(path, out var candidate) && candidate.Generation > newest.Generation) - newest = candidate; - } - // Files exist but none is readable: refuse to start over, which would replace the visitor - // secret and salts and reset the storage format settings of a directory that holds data. - if (found && newest.Generation == 0) - throw new AnalyticsConfigurationException( - $"The analytics metadata in '{Path.GetDirectoryName(paths[0])}' is damaged: no metadata file is valid."); - return newest; + using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true }); + writer.WriteStartObject(); + foreach (var (key, value) in entries.OrderBy(x => x.Key, StringComparer.Ordinal)) + writer.WriteString(key, value); + writer.WriteEndObject(); } - - static bool TryRead(string path, out (Dictionary Entries, long Generation) result) - { - result = default; - var bytes = File.ReadAllBytes(path); - var newline = Array.IndexOf(bytes, (byte)'\n'); - if (newline < 0) - return false; - var header = Encoding.UTF8.GetString(bytes, 0, newline).Split(' '); - var body = bytes.AsSpan(newline + 1); - if (header.Length != 3 || header[0] != Header || - !long.TryParse(header[1], NumberStyles.None, CultureInfo.InvariantCulture, out var fileGeneration) || - fileGeneration < 1 || - !string.Equals(header[2], Checksum(body), StringComparison.Ordinal)) - return false; - var entries = new Dictionary(StringComparer.Ordinal); - using (var document = JsonDocument.Parse(body.ToArray())) - { - foreach (var property in document.RootElement.EnumerateObject()) - entries[property.Name] = property.Value.GetString(); - } - result = (entries, fileGeneration); - return true; - } - - static void Write(string path, long fileGeneration, Dictionary entries) - { - var body = new ArrayBufferWriter(); - using (var writer = new Utf8JsonWriter(body, new JsonWriterOptions { Indented = true })) - { - writer.WriteStartObject(); - foreach (var (key, value) in entries.OrderBy(x => x.Key, StringComparer.Ordinal)) - writer.WriteString(key, value); - writer.WriteEndObject(); - } - var header = Encoding.UTF8.GetBytes(string.Create( - CultureInfo.InvariantCulture, $"{Header} {fileGeneration} {Checksum(body.WrittenSpan)}\n")); - using var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None); - stream.Write(header); - stream.Write(body.WrittenSpan); - stream.Flush(flushToDisk: true); - } - - static string Checksum(ReadOnlySpan body) => Convert.ToHexString(SHA256.HashData(body)); } diff --git a/src/ZoneTree.Analytics/Storage/RawEventStore.cs b/src/ZoneTree.Analytics/Storage/RawEventStore.cs index bbd9b04..eec02c3 100644 --- a/src/ZoneTree.Analytics/Storage/RawEventStore.cs +++ b/src/ZoneTree.Analytics/Storage/RawEventStore.cs @@ -63,7 +63,8 @@ public TState[] ScanIndexed(DimensionIndexSelection selection, string si /// public delegate void SliceVisitor(TState state, long ticks, ReadOnlySpan value); - readonly record struct Slice(IZoneTree, Memory> Tree, long From, long To); + /// A time range of one partition tree, scanned on its own. + internal readonly record struct Slice(IZoneTree, Memory> Tree, long From, long To); /// /// Scans the events of a site with timestamps in [fromTicks, toTicks). @@ -119,7 +120,7 @@ public TState[] Scan( /// /// Splits partition ranges into about slices of similar duration. /// - static List Split(List ranges, int targetCount) + internal static List Split(List ranges, int targetCount) { long total = 0; foreach (var range in ranges) diff --git a/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs b/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs index 918e2de..82a802e 100644 --- a/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs +++ b/src/ZoneTree.Analytics/Storage/RollupIndexStore.cs @@ -12,116 +12,88 @@ sealed record RollupIndex(string Dimension, string DefinitionId, Granularity Gra /// Recoverable, absolute copies of authoritative joint rollup rows. /// /// Builds and repairs run on the background maintenance thread while flushes write their own -/// copies. Queries read the primary rollups until the indexes are ready. A build resumes from -/// its checkpoint after a restart; the days that flushes wrote meanwhile are repaired. +/// copies. Queries read the primary rollups until the indexes are ready. A build also folds in +/// the history that a joint rollup reconstruction rebuilt, then publishes it. /// -sealed class RollupIndexStore : IDisposable +sealed class RollupIndexStore : BackgroundIndex { - const string CleanKey = "rollup-index.clean"; + const string KeyPrefix = "rollup-index"; /// The prefix of the build checkpoint, which a new history reconstruction clears. - public const string CheckpointPrefix = "rollup-index.backfill"; + public const string CheckpointPrefix = KeyPrefix + ".backfill"; - const string UncleanReason = "missing or unclean checkpoint"; const string FoldingReason = "rebuilt history to fold in"; const int BackgroundBatch = 4096; - readonly string root; readonly AnalyticsOptions options; readonly RollupStore rollups; readonly StoreFactory factory; - readonly MetaStore meta; - readonly ILogger logger; readonly List indexes = []; - readonly List resources = []; readonly Dictionary, RollupValue>> stores = []; Dictionary<(Granularity Granularity, string DefinitionId), RollupIndex[]> copies = []; readonly Dictionary removed = []; readonly Dictionary recovery = []; - int invalid; - long revision; - string manifest; RollupActivations activations; - volatile bool ready; - bool rebuild; - Dictionary dirtyDays = []; - Dictionary repairDays = []; public RollupIndexStore(string root, AnalyticsOptions options, RollupStore rollups, StoreFactory factory, MetaStore meta, ILogger logger) + : base(meta, logger, root, "Rollup index", KeyPrefix, "rollup-index.config", + StorageFormats.RollupIndexKey, StorageFormats.RollupIndex, "The rollup indexes of the analytics data directory") { - this.root = root; this.options = options; this.rollups = rollups; this.factory = factory; - this.meta = meta; - this.logger = logger; } public IReadOnlyList Indexes => indexes; - public bool IsHealthy => Volatile.Read(ref invalid) == 0; - - /// Whether the indexes are being built or repaired; queries read the primary rollups meanwhile. - public bool IsBuilding => indexes.Count > 0 && !ready && IsHealthy; public IEnumerable, RollupValue>> AllStores => stores.Values; static string Registration(string dimension, string definitionId, Granularity granularity) => string.Join(KeyCodec.ValueSeparator, dimension, definitionId, granularity.ToString()); - /// - /// Opens the indexes and decides whether they need a build or a repair, which - /// then runs in the background. A new build drops the old copies first, - /// before ingestion writes new ones. - /// + /// Opens the indexes and decides their work. public void Initialize(RollupActivations activations) { this.activations = activations; - using var startup = new StageLog(logger, "Rollup index initialization", root); - using var check = new StageLog(logger, "Rollup index backfill check", root); - var format = meta.Get(StorageFormats.RollupIndexKey); - if (format != null) - StorageFormats.Require(format, StorageFormats.RollupIndex, "The rollup indexes of the analytics data directory"); var configured = options.Storage.RollupIndexes.Definitions.SelectMany(definition => RollupStore.Granularities .Where(granularity => rollups.IsEnabled(granularity)) .Select(granularity => Registration(definition.Dimension, definition.Definition.Id, granularity))) - .Order(StringComparer.Ordinal).ToArray(); - manifest = string.Join('\n', configured); - var reasons = new List(); - // Indexes that were never initialized have no checkpoint, manifest or format to compare. - if (format == null) reasons.Add("new indexes"); - else - { - if (meta.Get(CleanKey) != "true") reasons.Add(UncleanReason); - if (meta.Get("rollup-index.config") != manifest) reasons.Add("configuration changed"); - } - if (!factory.RollupsDurable || !factory.RollupIndexesDurable) reasons.Add("write-ahead logging disabled"); - // Rows of a completed history reconstruction wait for this build to fold them in. - if (JointRollupReconstruction.HasFolding(meta)) reasons.Add(FoldingReason); + .Order(StringComparer.Ordinal); + // Rows of a completed history reconstruction wait for a build to fold them in. + Open(string.Join('\n', configured), JointRollupReconstruction.HasFolding(Meta) ? [FoldingReason] : []); + } + + protected override int IndexCount => indexes.Count; + + protected override int StoreCount => stores.Count; + + protected override string DisabledReason => "no configured combinations or enabled tiers"; + + protected override bool Durable => factory.RollupsDurable && factory.RollupIndexesDurable; + + protected override IEnumerable Journals => recovery.Values; + + protected override bool NothingToCopy => + rollups.PrimaryStores.All(store => store.ListPartitions().Count == 0) && !JointRollupReconstruction.IsPending(Meta); + + protected override bool Resumes(string reason) => reason is UncleanReason or FoldingReason; + + protected override int OpenStores(List reasons) + { var partitions = 0; foreach (var granularity in RollupStore.Granularities.Where(granularity => rollups.IsEnabled(granularity) && options.Storage.RollupIndexes.Definitions.Count > 0)) { var store = Open(granularity); stores.Add(granularity, store); - recovery.Add(granularity, new(meta, "rollup-index.dirty." + granularity + ".")); + recovery.Add(granularity, new(Meta, "rollup-index.dirty." + granularity + ".")); foreach (var definition in options.Storage.RollupIndexes.Definitions) indexes.Add(new(definition.Dimension, definition.Definition.Id, granularity, store)); var names = store.ListPartitions(); partitions += names.Count; - var expected = (meta.Get(PartitionsKey(granularity)) ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries); + var expected = (Meta.Get(PartitionsKey(granularity)) ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries); if (IndexRecoveryScan.MissingRetainedPartitions(expected, names, factory.RollupRetentionCutoff(granularity))) reasons.Add($"partition manifest changed: {granularity}"); } copies = indexes.GroupBy(index => (index.Granularity, index.DefinitionId)).ToDictionary(group => group.Key, group => group.ToArray()); - rebuild = reasons.Count > 0 && indexes.Count > 0; - var checkpoint = Checkpoint(); - // Only an interrupted build of the same configuration resumes; any other reason starts over. - var resuming = rebuild && checkpoint.Copying && reasons.All(reason => reason is UncleanReason or FoldingReason); - dirtyDays = recovery.ToDictionary(pair => pair.Key, pair => pair.Value.Days); - // A new build copies every day; one that resumes repairs the days written since it began. - repairDays = rebuild && !resuming ? [] : dirtyDays.Where(pair => pair.Value.Length > 0).ToDictionary(); - var dirtyCount = dirtyDays.Values.Sum(days => days.Length); - var decision = indexes.Count == 0 ? "disabled" : rebuild ? "rebuild required" : dirtyCount > 0 ? "repair affected days" : "reuse existing indexes unless historical rollups change"; - var reason = indexes.Count == 0 ? "no configured combinations or enabled tiers" : rebuild ? string.Join(", ", reasons) : dirtyCount > 0 ? "durable write intents require reconciliation" : "clean checkpoint and matching manifests"; - check.Complete($"decision={decision}; indexes={indexes.Count}; physical stores={stores.Count}; partitions={partitions}; dirty days={dirtyCount}; resuming={resuming}; reasons={reason}"); // Disabled tiers must not retain copies. Only the three known tier directories are owned here. foreach (var granularity in RollupStore.Granularities.Where(granularity => !stores.ContainsKey(granularity))) { @@ -129,69 +101,26 @@ public void Initialize(RollupActivations activations) var obsolete = Open(granularity); foreach (var partition in obsolete.ListPartitions()) obsolete.Drop(partition); } - if (rebuild && !resuming) - { - meta.Set(CleanKey, "false"); - checkpoint.Begin(); - foreach (var store in stores.Values) - foreach (var partition in store.ListPartitions()) store.Drop(partition); - Started(checkpoint); - // A new directory has nothing to copy: the build completes at once. - if (rollups.PrimaryStores.All(store => store.ListPartitions().Count == 0) && !JointRollupReconstruction.IsPending(meta)) - Build(CancellationToken.None); - } - if (!rebuild && repairDays.Count == 0) - { - ready = true; - MarkClean(); - } - startup.Complete($"decision={(ready ? indexes.Count == 0 ? "disabled" : rebuild ? "rebuild completed" : "reuse existing indexes" : rebuild ? "rebuild in background" : "repair in background")}; indexes={indexes.Count}; physical stores={stores.Count}"); + return partitions; } - IndexBackfillCheckpoint Checkpoint() => new(meta, CheckpointPrefix, manifest); - - /// - /// Stops a build once a copy has failed: the copies after it were skipped, so no checkpoint, - /// coverage or published history may claim them. - /// - void RequireHealthy() + protected override void DropCopies() { - if (!IsHealthy) - throw new AnalyticsStorageException( - "The rollup indexes could not be built: a copy failed. Queries read the primary rollups until the engine opens again, which resumes the build."); + foreach (var store in stores.Values) + foreach (var partition in store.ListPartitions()) store.Drop(partition); } - /// - /// Publishes the history the build folded in, records that the indexes copy the rollups, and - /// makes them available to queries. - /// - void Complete(IndexBackfillCheckpoint checkpoint) + protected override void SavePartitions(Dictionary entries) { - RequireHealthy(); - JointRollupReconstruction.PublishFolded(meta, activations, options.Rollups.Definitions); - meta.Update(entries => - { - SaveCoverage(entries); - checkpoint.Clear(entries); - }); - foreach (var (granularity, journal) in recovery) journal.ClearRepaired(dirtyDays[granularity]); - ready = true; - Interlocked.Increment(ref revision); + foreach (var (granularity, store) in stores) entries[PartitionsKey(granularity)] = string.Join(',', store.ListPartitions()); } - /// - /// Records the configuration and format of a build that has dropped the old copies: an - /// interrupted build of them resumes at the next open, while the indexes stay unclean. - /// - void Started(IndexBackfillCheckpoint checkpoint) - { - meta.Update(entries => - { - SaveCoverage(entries); - entries[CleanKey] = "false"; - }); - checkpoint.Cleared(); - } + protected override AnalyticsStorageException BuildFailed(Exception cause) => new( + "The rollup indexes could not be built: a copy failed. Queries read the primary rollups until the engine opens again, which resumes the build.", cause); + + /// Publishes the history the build folded in. + protected override void Completing() => + JointRollupReconstruction.PublishFolded(Meta, activations, options.Rollups.Definitions); /// /// Runs the build or repair that decided, on the background @@ -204,17 +133,17 @@ void Started(IndexBackfillCheckpoint checkpoint) /// flushes: the fold merges the row's rebuilt part in, and the row's current value is copied. /// Flushes copy the rows they write themselves, so a copy is never older than its row. /// - public void Build(CancellationToken cancellationToken) + public override void Build(CancellationToken cancellationToken) { - if (ready) return; + if (IsReady) return; // Indexes that failed wait for the next open, which resumes them from their last checkpoint. RequireHealthy(); - using var build = new StageLog(logger, "Rollup index backfill", root); + using var build = new StageLog(Logger, "Rollup index backfill", Root); var checkpoint = Checkpoint(); - var historical = JointRollupReconstruction.HasFolding(meta); + var historical = JointRollupReconstruction.HasFolding(Meta); // Rebuilt rows wait for this fold even when their definition is no longer indexed. - var folding = JointRollupReconstruction.FoldingDefinitions(meta, options.Rollups.Definitions); - long count = rebuild ? checkpoint.Count : 0; + var folding = JointRollupReconstruction.FoldingDefinitions(Meta, options.Rollups.Definitions); + long count = Rebuilding ? checkpoint.Count : 0; var resumed = count; long rows = 0; long matched = 0; @@ -256,7 +185,7 @@ void FoldPartition((Granularity Granularity, string Partition, string Id) source checkpoint.Save(source.Id, lastKey, count); build.Progress($"checkpoint saved; partition={source.Id}; rollup rows scanned={count}; resumed rows={resumed}", count - resumed, "rows"); } - if (rebuild) + if (Rebuilding) { var resumePartition = checkpoint.Partition; var resumeKey = checkpoint.Key; @@ -266,13 +195,13 @@ void FoldPartition((Granularity Granularity, string Partition, string Id) source FoldPartition(source, null, source.Id == resumePartition ? resumeKey : default, save: true); } } - if (repairDays.Count > 0) + if (RepairDays.Count > 0) { // Removes orphaned copies of interrupted writes, then copies the days again. A flush // writes the row before its copy, so a copy deleted here is copied again below. foreach (var (granularity, store) in stores) { - if (!repairDays.TryGetValue(granularity, out var days)) continue; + if (RepairDaysOf(granularity) is not { } days) continue; foreach (var partition in store.ListPartitions()) { using var lease = store.Acquire(partition); @@ -280,8 +209,11 @@ void FoldPartition((Granularity Granularity, string Partition, string Id) source IndexRecoveryScan.Visit(lease.Tree, days, RollupIndexKeyCodec.HeadLength, (key, _) => lease.Tree.ForceDelete(key)); } } - foreach (var source in sources.Where(source => repairDays.ContainsKey(source.Granularity))) - FoldPartition(source, repairDays[source.Granularity], default, save: false); + foreach (var source in sources) + { + if (RepairDaysOf(source.Granularity) is { } days) + FoldPartition(source, days, default, save: false); + } } // Merges the copies into disk segments, as ingestion goes on. foreach (var store in AllStores) store.Compact(); @@ -289,15 +221,19 @@ void FoldPartition((Granularity Granularity, string Partition, string Id) source build.Complete($"indexes={indexes.Count}; physical stores={stores.Count}; source partitions={sourcePartitions}; rollup rows scanned={count}; resumed rows={resumed}; indexed source rows={matched}; index rows written={rows}; historical backfill={historical}", count - resumed, "rows"); } + /// The days of a tier to repair, or null. + long[] RepairDaysOf(Granularity granularity) => + recovery.TryGetValue(granularity, out var journal) ? RepairDays.GetValueOrDefault(journal) : null; + static string PartitionsKey(Granularity granularity) => "rollup-index.partitions." + granularity; - string TierDirectory(Granularity granularity) => Path.Combine(root, granularity.ToString().ToLowerInvariant()); + string TierDirectory(Granularity granularity) => Path.Combine(Root, granularity.ToString().ToLowerInvariant()); PartitionedStore, RollupValue> Open(Granularity granularity) { var store = new PartitionedStore, RollupValue>(TierDirectory(granularity), options.Storage.RollupIndexes.RecordsPerPartition, - directory => factory.OpenRollupIndex(directory, granularity), logger, factory.CreateRollupIndexMaintainer, factory.WriteAheadLogDirectoryOf); - resources.Add(store); + directory => factory.OpenRollupIndex(directory, granularity), Logger, factory.CreateRollupIndexMaintainer, factory.WriteAheadLogDirectoryOf); + Resources.Add(store); return store; } @@ -307,10 +243,9 @@ public void Copy(Granularity granularity, Memory key, RollupValue value) try { CopyCore(granularity, key, value); } catch (Exception exception) { - Volatile.Write(ref invalid, 1); - Interlocked.Increment(ref revision); + Invalidate(); // Primary rollups remain authoritative and must keep accepting deltas. - try { logger.LogError(exception); } catch { } + try { Logger.LogError(exception); } catch { } } } @@ -357,7 +292,7 @@ public string Unbuilt(Granularity granularity, string definitionId, IEnumerable< public string Scan(Granularity granularity, string site, string eventName, string definitionId, long from, long to, IEnumerable filters, Action visit, CancellationToken cancellationToken) { - if (!IsHealthy || !ready || from >= to) return null; + if (!IsHealthy || !IsReady || from >= to) return null; foreach (var filter in DimensionIndexStore.Mandatory(filters)) { if (!DimensionIndexStore.TryEquality(filter, out var dimension, out var values)) continue; @@ -365,7 +300,7 @@ public string Scan(Granularity granularity, string site, string eventName, strin if (index == null) continue; values = values.Distinct(StringComparer.Ordinal).ToArray(); if (values.Length > 64) continue; - var currentRevision = Interlocked.Read(ref revision); + var currentRevision = Revision; Check(currentRevision); foreach (var partition in index.Store.PartitionsInRange(from, to)) { @@ -398,7 +333,7 @@ public string Scan(Granularity granularity, string site, string eventName, strin void Check(long expectedRevision) { - if (!IsHealthy || expectedRevision != Interlocked.Read(ref revision)) + if (!IsHealthy || expectedRevision != Revision) throw new AnalyticsQueryException("Rollup indexes changed during the query. Retry the query."); } @@ -420,29 +355,4 @@ public int ApplyRetention(RetentionCutoffs cutoffs, RetentionClock clock) } return dropped; } - - /// - /// Records that the indexes copy the rollups, when they do: after an orderly shutdown no day - /// needs a repair. An unfinished build keeps its checkpoint and resumes at the next open. - /// - public void MarkClean() - { - if (!IsHealthy || !ready) return; - meta.Update(entries => - { - SaveCoverage(entries); - foreach (var journal in recovery.Values) journal.Clear(entries); - Checkpoint().Clear(entries); - }); - } - - void SaveCoverage(Dictionary entries) - { - entries[CleanKey] = factory.RollupsDurable && factory.RollupIndexesDurable ? "true" : "false"; - entries[StorageFormats.RollupIndexKey] = StorageFormats.Text(StorageFormats.RollupIndex); - entries["rollup-index.config"] = manifest; - foreach (var (granularity, store) in stores) entries[PartitionsKey(granularity)] = string.Join(',', store.ListPartitions()); - } - - public void Dispose() => Disposal.DisposeAll(resources); } diff --git a/src/ZoneTree.Analytics/Storage/RollupStore.cs b/src/ZoneTree.Analytics/Storage/RollupStore.cs index 057defc..d581ada 100644 --- a/src/ZoneTree.Analytics/Storage/RollupStore.cs +++ b/src/ZoneTree.Analytics/Storage/RollupStore.cs @@ -50,7 +50,8 @@ public RollupStore( ownedStores.AddRange(PrimaryStores); } - static int IndexOf(Granularity granularity) => granularity switch + /// The position of a granularity in . + public static int IndexOf(Granularity granularity) => granularity switch { Granularity.Minute => 0, Granularity.Hour => 1, diff --git a/src/ZoneTree.Analytics/Storage/RollupValue.cs b/src/ZoneTree.Analytics/Storage/RollupValue.cs index 9c2aad2..e135595 100644 --- a/src/ZoneTree.Analytics/Storage/RollupValue.cs +++ b/src/ZoneTree.Analytics/Storage/RollupValue.cs @@ -239,17 +239,7 @@ public RollupSection(ReadOnlyMemory bytes, Func, T> d public bool IsStored => !Bytes.IsEmpty; - public T Value - { - get - { - var current = Volatile.Read(ref value); - if (current != null) - return current; - var decoded = decode(Bytes); - return Interlocked.CompareExchange(ref value, decoded, null) ?? decoded; - } - } + public T Value => Once.Get(ref value, this, static x => x.decode(x.Bytes)); } /// @@ -268,17 +258,7 @@ sealed class DetailEntries(RollupSection[] entries) public ValueDetail At(int index) => Entries[index]?.Value; /// Every detail, decoded once. - public ValueDetail[] All - { - get - { - var current = Volatile.Read(ref all); - if (current != null) - return current; - var decoded = Array.ConvertAll(Entries, x => x?.Value); - return Interlocked.CompareExchange(ref all, decoded, null) ?? decoded; - } - } + public ValueDetail[] All => Once.Get(ref all, Entries, static x => Array.ConvertAll(x, entry => entry?.Value)); /// The entries of details created in memory; a detail without data has none. public static DetailEntries Of(ValueDetail[] details) => new(Array.ConvertAll(details, @@ -298,17 +278,7 @@ sealed class DistinctEntries(string[] names, RollupSection[] sketch HyperLogLog[] all; /// Every sketch, decoded once. - public HyperLogLog[] All - { - get - { - var current = Volatile.Read(ref all); - if (current != null) - return current; - var decoded = Array.ConvertAll(Sketches, x => x.Value); - return Interlocked.CompareExchange(ref all, decoded, null) ?? decoded; - } - } + public HyperLogLog[] All => Once.Get(ref all, Sketches, static x => Array.ConvertAll(x, sketch => sketch.Value)); /// The entries of sketches created in memory. public static DistinctEntries Of(string[] names, HyperLogLog[] sketches) => diff --git a/src/ZoneTree.Analytics/Storage/StorageKeyCodec.cs b/src/ZoneTree.Analytics/Storage/StorageKeyCodec.cs index 4afc1ab..392cc14 100644 --- a/src/ZoneTree.Analytics/Storage/StorageKeyCodec.cs +++ b/src/ZoneTree.Analytics/Storage/StorageKeyCodec.cs @@ -62,8 +62,6 @@ public DateTimeOffset GetBucketStart(Granularity granularity, TimeZoneInfo timeZ /// public static class StorageKeyCodec { - static readonly UTF8Encoding StrictUtf8 = new(false, true); - /// Encodes a raw event key in the engine's storage format. public static Memory EncodeRawEvent(RawEventKey key) { @@ -163,7 +161,7 @@ static string ReadUtf8(ReadOnlySpan bytes) { try { - return StrictUtf8.GetString(bytes); + return Text.StrictUtf8.GetString(bytes); } catch (DecoderFallbackException exception) { diff --git a/src/ZoneTree.Analytics/Web/BuiltInUserAgentParser.cs b/src/ZoneTree.Analytics/Web/BuiltInUserAgentParser.cs index 408e7f7..08237a9 100644 --- a/src/ZoneTree.Analytics/Web/BuiltInUserAgentParser.cs +++ b/src/ZoneTree.Analytics/Web/BuiltInUserAgentParser.cs @@ -1,5 +1,3 @@ -using System.Collections.Concurrent; - namespace ZoneTree.Analytics.Web; /// @@ -8,28 +6,11 @@ namespace ZoneTree.Analytics.Web; /// sealed class BuiltInUserAgentParser : IUserAgentParser { - const int MaxCacheSize = 10_000; - - const int MaxUserAgentLength = 1024; - - readonly ConcurrentDictionary cache = new(StringComparer.Ordinal); + readonly UserAgentCache cache = new(ParseCore); public static BuiltInUserAgentParser Instance { get; } = new(); - public UserAgentInfo Parse(string userAgent) - { - if (string.IsNullOrWhiteSpace(userAgent)) - return null; - if (userAgent.Length > MaxUserAgentLength) - userAgent = userAgent[..MaxUserAgentLength]; - if (cache.TryGetValue(userAgent, out var cached)) - return cached; - var info = ParseCore(userAgent); - if (cache.Count >= MaxCacheSize) - cache.Clear(); - cache.TryAdd(userAgent, info); - return info; - } + public UserAgentInfo Parse(string userAgent) => cache.Get(userAgent); static UserAgentInfo ParseCore(string ua) { diff --git a/src/ZoneTree.Analytics/Web/UserAgentBotDetector.cs b/src/ZoneTree.Analytics/Web/UserAgentBotDetector.cs index 9d0304a..8500180 100644 --- a/src/ZoneTree.Analytics/Web/UserAgentBotDetector.cs +++ b/src/ZoneTree.Analytics/Web/UserAgentBotDetector.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Text.RegularExpressions; namespace ZoneTree.Analytics.Web; @@ -14,13 +13,7 @@ namespace ZoneTree.Analytics.Web; /// sealed partial class UserAgentBotDetector : IBotDetector { - const int MaxCacheSize = 10_000; - - const int MaxUserAgentLength = 1024; - - static readonly BotInfo NotABot = new("", ""); - - readonly ConcurrentDictionary cache = new(StringComparer.Ordinal); + readonly UserAgentCache cache = new(DetectCore); public static UserAgentBotDetector Instance { get; } = new(); @@ -158,22 +151,7 @@ static readonly (string Token, string Name, string Category)[] KnownBots = [GeneratedRegex(@"(? MaxUserAgentLength) - userAgent = userAgent[..MaxUserAgentLength]; - if (!cache.TryGetValue(userAgent, out var bot)) - { - bot = DetectCore(userAgent) ?? NotABot; - if (cache.Count >= MaxCacheSize) - cache.Clear(); - cache.TryAdd(userAgent, bot); - } - return ReferenceEquals(bot, NotABot) ? null : bot; - } + public BotInfo Detect(EventContext context) => cache.Get(context.UserAgent); internal static BotInfo DetectCore(string userAgent) { diff --git a/src/ZoneTree.Analytics/Web/UserAgentCache.cs b/src/ZoneTree.Analytics/Web/UserAgentCache.cs new file mode 100644 index 0000000..2bc00cc --- /dev/null +++ b/src/ZoneTree.Analytics/Web/UserAgentCache.cs @@ -0,0 +1,33 @@ +using System.Collections.Concurrent; + +namespace ZoneTree.Analytics.Web; + +/// +/// What is derived from user agents, for up to 10,000 distinct ones: a full cache starts over. +/// User agents are cut to their first 1,024 characters first, which bounds the work and memory of +/// unusually long ones. +/// +sealed class UserAgentCache(Func derive) +{ + const int MaxCount = 10_000; + + const int MaxLength = 1024; + + readonly ConcurrentDictionary cache = new(StringComparer.Ordinal); + + /// What is derived from a user agent; the default value for a missing or blank one. + public T Get(string userAgent) + { + if (string.IsNullOrWhiteSpace(userAgent)) + return default; + if (userAgent.Length > MaxLength) + userAgent = userAgent[..MaxLength]; + if (cache.TryGetValue(userAgent, out var cached)) + return cached; + var derived = derive(userAgent); + if (cache.Count >= MaxCount) + cache.Clear(); + cache.TryAdd(userAgent, derived); + return derived; + } +} diff --git a/yarn.lock b/yarn.lock index 5ece0e7..0741c95 100644 --- a/yarn.lock +++ b/yarn.lock @@ -413,163 +413,196 @@ __metadata: languageName: node linkType: hard -"@purestack/ts-common@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-common@npm:1.1.14" +"@purestack/ts-common@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-common@npm:1.1.15" dependencies: - "@purestack/ts-html": "npm:^1.1.14" - "@purestack/ts-style": "npm:^1.1.14" - "@purestack/ts-util": "npm:^1.1.14" - checksum: 10c0/132802ad798a322a2b29ad3a3b5cbec6c22efe37e636e34656d0fa434de4323f90f2afc99d1996f112885e639cfecc233639850f1007edce60538b9d88506c19 + "@purestack/ts-html": "npm:^1.1.15" + "@purestack/ts-style": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" + checksum: 10c0/aa6239d2ee76293be277053a7c558d425505ef564f5e879691eac22a567731d83d0a41599f090fbb9164bdcbef0e4e381ed26f175854809e21022a9da491604a languageName: node linkType: hard -"@purestack/ts-components@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-components@npm:1.1.14" +"@purestack/ts-components@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-components@npm:1.1.15" dependencies: - "@purestack/ts-common": "npm:^1.1.14" - "@purestack/ts-css": "npm:^1.1.14" - "@purestack/ts-page-scripts": "npm:^1.1.14" - "@purestack/ts-style": "npm:^1.1.14" - "@purestack/ts-svg-icons": "npm:^1.1.14" - "@purestack/ts-util": "npm:^1.1.14" + "@purestack/ts-common": "npm:^1.1.15" + "@purestack/ts-css": "npm:^1.1.15" + "@purestack/ts-page-scripts": "npm:^1.1.15" + "@purestack/ts-style": "npm:^1.1.15" + "@purestack/ts-svg-icons": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" regor: "npm:^1.7.5" - checksum: 10c0/865304a912caafe03e56b9ee2356928cdc83e43f48ea7961daf5c224c56eaf2f5419ed3ada05b1febf673f559dd2ce624280c05ff68b3120322f4cef585a72b7 + checksum: 10c0/b1c0f91654b843dbf3f778514b4f4aee78f61f6cb2cafd4f80d80a234ad4b96b41dbdcf714c12361b9ec3b2b11173c4b4a55be06adb6b05bcae06a9de2e0832a languageName: node linkType: hard -"@purestack/ts-css@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-css@npm:1.1.14" +"@purestack/ts-css@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-css@npm:1.1.15" dependencies: - "@purestack/ts-util": "npm:^1.1.14" + "@purestack/ts-util": "npm:^1.1.15" prettier: "npm:^3.8.1" - checksum: 10c0/4996f71f9758dcd96a060e19a5c6792d258212d6d725cf0733cc4f271780d260ae2c16fb5aca21813bd02954351ab38d64320fa9410ed2ef86f9e1455e2dd48d + checksum: 10c0/7c09845eacb0fbd93660a16a3a8fdb817b61139e6c3a398300c557d69c7f1b62ee7865160ff84d4a13b67f61e1ff5b5f79af8cd4756e512ab96498d1600244b9 languageName: node linkType: hard -"@purestack/ts-html@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-html@npm:1.1.14" +"@purestack/ts-highlight@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-highlight@npm:1.1.15" dependencies: - "@purestack/ts-util": "npm:^1.1.14" + "@purestack/ts-common": "npm:^1.1.15" + highlight.js: "npm:^11.11.1" + shiki: "npm:^3.22.0" + checksum: 10c0/4388a865fe7c2e5b07f4da91e05be8d3b2bede591fffba4b1d522383dc26424302d5d99a2910770bf2ea0e0aa6767284653886647d88f0982a0e4840ab95fc94 + languageName: node + linkType: hard + +"@purestack/ts-html@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-html@npm:1.1.15" + dependencies: + "@purestack/ts-util": "npm:^1.1.15" prettier: "npm:^3.8.1" - checksum: 10c0/5f07904733f477238fedb8ec72d2141efcb1fef62870ff861190d03cf52ee07b6aa060f3dadc0a95e3177c4b28819166fb79464369e0cf80b7815faa7e316194 + checksum: 10c0/509c22aa553c100ab9161516ad8cf048a64174e6d9ceb47d893850dde658f783b92da9647e1180fd5e1b2a871a83f0ca2712783a2ffa424f2803e1d65552063b languageName: node linkType: hard -"@purestack/ts-minidom@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-minidom@npm:1.1.14" - checksum: 10c0/8ba2840474dd173a22d28f93a108cf5d2adbef0fd1cbed263d82ac1785879924ec77d0b0f7d4a0ed1a1c3de3b5f994242c257ddb0e74eb824e6038e340074f2e +"@purestack/ts-minidom@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-minidom@npm:1.1.15" + checksum: 10c0/3ed2780232215dcb08dff5354f1cb26e6fb15aa2b536a055e8a7a0924708fc2983e6c4aee7a566970346248bbc40ce58dc8be148565415ab5039ca8453f89957 languageName: node linkType: hard -"@purestack/ts-page-scripts@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-page-scripts@npm:1.1.14" +"@purestack/ts-page-scripts@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-page-scripts@npm:1.1.15" dependencies: - "@purestack/ts-common": "npm:^1.1.14" - "@purestack/ts-html": "npm:^1.1.14" - checksum: 10c0/c872f719771b74767430705f55b337b1045bf93d0712cb9e80ba9e9d93dd19b5f20cd2d4a8ec70c3f69df042a72e0b5d13709e73ffd4f49d926d2823d0df3330 + "@purestack/ts-common": "npm:^1.1.15" + "@purestack/ts-html": "npm:^1.1.15" + checksum: 10c0/d086b81ac1b73ca4cd2df2077822552b914f46d36ed506767be9305cebebbdf51f92fdf7725d60449a66014d4ead1a2353adcadf91c423855b780d80c1299021 languageName: node linkType: hard -"@purestack/ts-render@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-render@npm:1.1.14" +"@purestack/ts-pagefind@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-pagefind@npm:1.1.15" dependencies: - "@purestack/ts-common": "npm:^1.1.14" - "@purestack/ts-minidom": "npm:^1.1.14" - "@purestack/ts-page-scripts": "npm:^1.1.14" - "@purestack/ts-util": "npm:^1.1.14" + "@purestack/ts-common": "npm:^1.1.15" + logpot: "npm:^1.0.7" + pagefind: "npm:^1.4.0" + checksum: 10c0/189704c83248d8ef41422593970682d3084fb6709aa63bce6cb01f4221ef1b0115ea642f560bbaf2f37282daac6bb651f70a19f4a28fec7f4a5483fc0b2ed850 + languageName: node + linkType: hard + +"@purestack/ts-render@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-render@npm:1.1.15" + dependencies: + "@purestack/ts-common": "npm:^1.1.15" + "@purestack/ts-minidom": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" regor: "npm:^1.7.5" - checksum: 10c0/bc860f38827f3ce4ed0cbfa796ecbb5e9a6e97f4b31ddf24909b1d7bf5af6419bc9ca31e691a386551a0878fc40a4cfc114d28b1e40689d8a1c7ca7a2e82debb + checksum: 10c0/c24f6722ff77d7dd29c51e865b1b1a1e6fea36b5302df7a1f3f17481d8cd98f3a5dc20188bf2c41fbe3e2c71136a267855dc9100457ca529e6e1a1a161d87316 + languageName: node + linkType: hard + +"@purestack/ts-seo@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-seo@npm:1.1.15" + dependencies: + "@purestack/ts-common": "npm:^1.1.15" + "@purestack/ts-html": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" + checksum: 10c0/707c84b87000b89a465baeded777f81d80cd868409f0efce253b77b8806f371f3091c89e5185d6146940b141ce94eae0da9cf22db6cdb95e0e57e80aae293680 languageName: node linkType: hard -"@purestack/ts-ssg@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-ssg@npm:1.1.14" +"@purestack/ts-ssg@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-ssg@npm:1.1.15" dependencies: "@11ty/gray-matter": "npm:^3.0.0" - "@purestack/ts-common": "npm:^1.1.14" - "@purestack/ts-components": "npm:^1.1.14" - "@purestack/ts-html": "npm:^1.1.14" - "@purestack/ts-minidom": "npm:^1.1.14" - "@purestack/ts-page-scripts": "npm:^1.1.14" - "@purestack/ts-render": "npm:^1.1.14" - "@purestack/ts-style": "npm:^1.1.14" - "@purestack/ts-svg-icons": "npm:^1.1.14" - "@purestack/ts-theme": "npm:^1.1.14" - "@purestack/ts-util": "npm:^1.1.14" - "@purestack/ts-util-node": "npm:^1.1.14" + "@purestack/ts-common": "npm:^1.1.15" + "@purestack/ts-components": "npm:^1.1.15" + "@purestack/ts-highlight": "npm:^1.1.15" + "@purestack/ts-html": "npm:^1.1.15" + "@purestack/ts-minidom": "npm:^1.1.15" + "@purestack/ts-page-scripts": "npm:^1.1.15" + "@purestack/ts-pagefind": "npm:^1.1.15" + "@purestack/ts-render": "npm:^1.1.15" + "@purestack/ts-seo": "npm:^1.1.15" + "@purestack/ts-style": "npm:^1.1.15" + "@purestack/ts-theme": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" + "@purestack/ts-util-node": "npm:^1.1.15" esbuild: "npm:^0.28.0" hast-util-from-html: "npm:^2.0.3" hast-util-to-html: "npm:^9.0.5" - highlight.js: "npm:^11.11.1" html-minifier-next: "npm:^4.19.1" logpot: "npm:^1.0.7" mdast-util-to-hast: "npm:^13.2.1" - pagefind: "npm:^1.4.0" regor: "npm:^1.7.5" rehype-raw: "npm:^7.0.0" rehype-stringify: "npm:^10.0.1" remark-gfm: "npm:^4.0.1" remark-parse: "npm:^11.0.0" remark-rehype: "npm:^11.1.2" - shiki: "npm:^3.22.0" unified: "npm:^11.0.5" bin: ts-ssg: dist/cli.mjs - checksum: 10c0/08591fe129303336698d8e7927bc2a29ca983e46d6d6f2bfc879345054d03bf91ad24d88de0d733841760940bbf2010e62b9fe95fa8322362638b787b5fd064d + checksum: 10c0/4602b9c2ca203a19330c742de64167867e0ba830b67ecade14aa44659f842f190e8f4e9b2164ed6e6b4497b7ef02f4e146979f4d11d526fc1bab2a4653bc52c4 languageName: node linkType: hard -"@purestack/ts-style@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-style@npm:1.1.14" +"@purestack/ts-style@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-style@npm:1.1.15" dependencies: - "@purestack/ts-css": "npm:^1.1.14" - "@purestack/ts-util": "npm:^1.1.14" - checksum: 10c0/c7a8ef003938e905ad6f282696c83d65af1c00348aabd9987df95b515a1fa17306d29e11fbc508f397d2e913fb9873be2f8b375606c8e0f1142b419a16f812f0 + "@purestack/ts-css": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" + checksum: 10c0/249ab37f73e4ca398d7471fae03ab3ae5c5fe4e3cba672699650df75f062ea0bf50e4477eb4218611bde3337e407646fa63faadfd40769dd564ef25a7f52a6c0 languageName: node linkType: hard -"@purestack/ts-svg-icons@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-svg-icons@npm:1.1.14" - checksum: 10c0/e8d5c730e8ad93fd570592a2d7c5635144054459b02032ea35254a2dd067ec1350f3cee3eb519d8741d74a808057893c020c2f31658cbc5bfbbd00a265e0b9d6 +"@purestack/ts-svg-icons@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-svg-icons@npm:1.1.15" + checksum: 10c0/f746f320b48b2e3d7698c2dcd3ec2f3a06b0fa6c4083484e692559300a8a41072c310ce2a90e511218d6682f9c284bec46520765d221503a6273be75e7f8b6b9 languageName: node linkType: hard -"@purestack/ts-theme@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-theme@npm:1.1.14" +"@purestack/ts-theme@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-theme@npm:1.1.15" dependencies: - "@purestack/ts-common": "npm:^1.1.14" - "@purestack/ts-html": "npm:^1.1.14" - "@purestack/ts-page-scripts": "npm:^1.1.14" - "@purestack/ts-style": "npm:^1.1.14" - "@purestack/ts-util-node": "npm:^1.1.14" + "@purestack/ts-common": "npm:^1.1.15" + "@purestack/ts-html": "npm:^1.1.15" + "@purestack/ts-page-scripts": "npm:^1.1.15" + "@purestack/ts-style": "npm:^1.1.15" + "@purestack/ts-svg-icons": "npm:^1.1.15" + "@purestack/ts-util-node": "npm:^1.1.15" + "@types/hast": "npm:^3.0.4" highlight.js: "npm:^11.11.1" logpot: "npm:^1.0.7" - checksum: 10c0/925281f830a7f3fd58d79fb544312a2a55e505f4b91473212ee930278224bf7b7ec3fa3ac7769a1e8ae4db00e19e2bbb265af1f140f22afd455c12ae9e20428f + checksum: 10c0/b866afd907b76a6d87e6654a36f4ea6a4efa4c6ac2304114d24ea87551da4a918667bd7952c5b3ca3762ff083a08c53e7ab3355e3fd8d759685874d99b20bc03 languageName: node linkType: hard -"@purestack/ts-util-node@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-util-node@npm:1.1.14" - checksum: 10c0/4e736e5992cf76c433e8b4b8acc63bb862a2d6f5601351a7eec7e4f693b5329d9a655e8fe14097ec548f068701de55a401ff6fb13ed89e771158991d17b5811e +"@purestack/ts-util-node@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-util-node@npm:1.1.15" + checksum: 10c0/d9eb7894513c629036bf64f47360e3dc9d32b329b8a76c78c9c1ecae264892de136ff4226edb348a8c7d510c8970b533f26bdc17965be2cddd29f85af43feaed languageName: node linkType: hard -"@purestack/ts-util@npm:^1.1.14": - version: 1.1.14 - resolution: "@purestack/ts-util@npm:1.1.14" - checksum: 10c0/f7a630ab7da8b2e65638fb96cd594c6fbcc0acb17a957b144312826b6d9a184b650f84573a6faecb4b171b71340f0013f60f151e6694b18ecdc3efc606aa0a90 +"@purestack/ts-util@npm:^1.1.15": + version: 1.1.15 + resolution: "@purestack/ts-util@npm:1.1.15" + checksum: 10c0/1c47d80a31a7b48b8e0922008b1f2bfe6db315693d3f7917197e21d028a3e0848e071038d242dff4065c9a5238dfcd98461cd836e7bae7438b8d3704c1a03645 languageName: node linkType: hard @@ -1028,9 +1061,9 @@ __metadata: version: 0.0.0-use.local resolution: "analytics.zonetree.dev@workspace:docs-site" dependencies: - "@purestack/ts-style": "npm:^1.1.14" - "@purestack/ts-util": "npm:^1.1.14" - purestack: "npm:^1.1.14" + "@purestack/ts-style": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" + purestack: "npm:^1.1.15" regor: "npm:^1.7.5" typescript: "npm:^7.0.2" languageName: unknown @@ -2538,14 +2571,14 @@ __metadata: languageName: node linkType: hard -"purestack@npm:^1.1.14": - version: 1.1.14 - resolution: "purestack@npm:1.1.14" +"purestack@npm:^1.1.15": + version: 1.1.15 + resolution: "purestack@npm:1.1.15" dependencies: - "@purestack/ts-ssg": "npm:^1.1.14" + "@purestack/ts-ssg": "npm:^1.1.15" bin: purestack: dist/cli.mjs - checksum: 10c0/244f87272518a446f17e09cc9bdd6bd099125cc3da1564e5447d71e1ad91d8192927d010620b817d97ceb3cd9c541714817619c66916cb1df32b4cb7e386e74d + checksum: 10c0/04fcaf59ff20bf50065cdc0b0a216a41ca6a0ce2ca51ee9275eb24012fd514957453dcadc403cfd2dca84be7f4bf818f77e3cd2448a97b4f51d86592b923b86a languageName: node linkType: hard @@ -3332,15 +3365,15 @@ __metadata: version: 0.0.0-use.local resolution: "zonetree-analytics-dashboard@workspace:apps/ZoneTree.Analytics.Dashboard/ui" dependencies: - "@purestack/ts-components": "npm:^1.1.14" - "@purestack/ts-css": "npm:^1.1.14" - "@purestack/ts-html": "npm:^1.1.14" - "@purestack/ts-style": "npm:^1.1.14" - "@purestack/ts-svg-icons": "npm:^1.1.14" - "@purestack/ts-util": "npm:^1.1.14" + "@purestack/ts-components": "npm:^1.1.15" + "@purestack/ts-css": "npm:^1.1.15" + "@purestack/ts-html": "npm:^1.1.15" + "@purestack/ts-style": "npm:^1.1.15" + "@purestack/ts-svg-icons": "npm:^1.1.15" + "@purestack/ts-util": "npm:^1.1.15" "@types/node": "npm:^22.0.0" playwright: "npm:^1.56.1" - purestack: "npm:^1.1.14" + purestack: "npm:^1.1.15" regor: "npm:^1.7.5" tsx: "npm:^4.20.6" typescript: "npm:^7.0.2" @@ -3353,6 +3386,7 @@ __metadata: dependencies: "@biomejs/biome": "npm:^2.5.15" "@types/node": "npm:^22.0.0" + playwright: "npm:^1.56.1" tsx: "npm:^4.20.6" typescript: "npm:^7.0.2" vite: "npm:8.3.4" From c9eb865a1586628720bec50ff5464494b9946f5d Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Sat, 17 Oct 2026 12:58:51 +0200 Subject: [PATCH 2/2] Update hero benchmark median. --- docs-site/content/ui/hero.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-site/content/ui/hero.ts b/docs-site/content/ui/hero.ts index 1201561..ea833f0 100644 --- a/docs-site/content/ui/hero.ts +++ b/docs-site/content/ui/hero.ts @@ -72,7 +72,7 @@ export function defineAnalyticsHero() {
5M eventsMeasured dataset - 84 msSix annual reports · median + 33 msSix annual reports · median
Explore the benchmark