Simplifies the storage internals, refuses data directories it cannot load safely, and refreshes the benchmark numbers from Release builds.

README.md

@@ -196,10 +196,10 @@ The repository includes a [documented benchmark][benchmarks] over **5 million ge
 
 | Workload | Median | p95 |
 | --- | ---: | ---: |
-| Complete overview: six reports, up to three requests in flight | **84.0 ms** | 87.0 ms |
-| Country breakdown filtered to GitHub traffic | **13.5 ms** | 14.0 ms |
+| Complete overview: six reports, up to three requests in flight | **33.3 ms** | 35.5 ms |
+| Country breakdown filtered to GitHub traffic | **5.7 ms** | 6.5 ms |
 
-Measured over 30 warm loads per scope on an Intel Core Ultra 7 265KF with 64 GiB RAM, Windows x64, and a .NET 10.0.6 Debug host. Times include loopback HTTP response transfer and JSON parsing; chart rendering is excluded. The dataset has low group cardinality, and filtered reports use configured covering rollup indexes. These figures describe that workload, rather than a latency guarantee.
+Measured over 30 warm loads per scope on an Intel Core Ultra 7 265KF with 64 GiB RAM, Windows x64, and a .NET 10.0.6 Release host. Times include loopback HTTP response transfer and JSON parsing; chart rendering is excluded. The dataset has low group cardinality, and filtered reports use configured covering rollup indexes. These figures describe that workload, rather than a latency guarantee.
 
 Matching rollups answer recurring reports from aggregates. Segments, funnels, and unsupported aggregate combinations can require event scans. The [benchmark report][benchmarks] includes configuration, query plans, individual timings, and methodology; the [benchmarking guide][benchmarking] explains how to measure your workload.
 

apps/ZoneTree.Analytics.Dashboard/server/BenchmarkReportStore.cs

@@ -52,7 +52,7 @@ public SampleEngineLease Acquire()
       if (current == null) throw new BenchmarkDataUnavailableException();
       var path = Path.Combine(root, current.Directory);
       ValidateOwned(path);
-      engine ??= new AnalyticsEngine(DashboardAnalyticsOptions.CreateBenchmarkReport(path, current.TimeZone, current.Result.Percentiles, logger));
+      engine ??= new AnalyticsEngine(DashboardAnalyticsOptions.CreateBenchmarkReport(path, current.TimeZone, current.Result.Percentiles, current.Result.Indexes, logger));
       if (readers++ == 0) drained = new(TaskCreationOptions.RunContinuationsAsynchronously);
       return new(engine, () => { lock (sync) { if (--readers == 0) drained.TrySetResult(); } });
     }

apps/ZoneTree.Analytics.Dashboard/server/DashboardAnalyticsOptions.cs

@@ -39,20 +39,26 @@ public static AnalyticsOptions CreatePerformanceSamples(string directory, TimeZo
     return options;
   }
 
+  // Without indexes, ingestion writes only raw events, sessions and rollups.
   public static AnalyticsOptions CreateBenchmark(string directory, TimeZoneInfo timeZone,
-      int workers, bool percentiles, ILogger logger = null)
+      int workers, bool percentiles, bool indexes, ILogger logger = null)
   {
     var options = Create(directory, BenchmarkSite, timeZone, logger);
     SetWorkers(options, workers);
     options.Ingestion.QueueFullBehavior = QueueFullBehavior.Wait;
     if (percentiles) options.Rollups.ValueAggregates["load_time"] = ValueAggregates.Percentiles;
+    if (!indexes)
+    {
+      options.Storage.DimensionIndexes.Dimensions.Clear();
+      options.Storage.RollupIndexes.Definitions.Clear();
+    }
     return options;
   }
 
-  // Reopening uses the saved dataset time zone and percentile setting, with one worker.
+  // Reopening uses the saved dataset time zone, percentile and index settings, with one worker.
   public static AnalyticsOptions CreateBenchmarkReport(string directory, string timeZoneId,
-      bool percentiles, ILogger logger = null) =>
-      CreateBenchmark(directory, TimeZoneInfo.FindSystemTimeZoneById(timeZoneId), 1, percentiles, logger);
+      bool percentiles, bool indexes, ILogger logger = null) =>
+      CreateBenchmark(directory, TimeZoneInfo.FindSystemTimeZoneById(timeZoneId), 1, percentiles, indexes, logger);
 
   // Generated ingestion runs as many rollup writers as workers, so the rows that every worker
   // buffers are merged and written in parallel at the same scale.

apps/ZoneTree.Analytics.Dashboard/server/Program.cs

@@ -39,9 +39,10 @@
     builder.Configuration["Analytics:DataDirectory"] ?? DashboardAnalyticsOptions.DefaultDataDirectory, timeZone));
 
 var app = builder.Build();
+// Open the engine now: data it cannot load fails startup instead of every request.
+var engine = app.Services.GetRequiredService<AnalyticsEngine>();
 if (builder.Configuration.GetValue<bool>("Analytics:SeedDemo"))
 {
-  var engine = app.Services.GetRequiredService<AnalyticsEngine>();
   if (engine.GetEventNames(DateRange.LastDays(60)).Count == 0)
     await SampleTraffic.SeedAsync(engine);
 }

apps/ZoneTree.Analytics.Dashboard/server/SampleBenchmark.cs

@@ -4,13 +4,14 @@
 using Microsoft.Extensions.Logging;
 using ZoneTree.Analytics;
 
-sealed record SampleBenchmarkRequest(int Events = 5_000_000, int Days = 365, int Workers = 20, bool Percentiles = false);
+sealed record SampleBenchmarkRequest(int Events = 5_000_000, int Days = 365, int Workers = 20, bool Percentiles = false,
+    bool Indexes = true);
 sealed record SampleBenchmarkResult(int Events, int Days, int Workers, bool Percentiles,
     double PreparationMs, double SubmissionMs, double IngestionMs, double EventsPerSecond,
     long Ingested, long Errors, long Dropped, long Rejected, string Build, string GarbageCollection, string TieredCompilation,
-    string FromDate = "", string ToDate = "");
+    string FromDate = "", string ToDate = "", bool Indexes = true);
 sealed record SampleBenchmarkStatus(string State = "idle", int Requested = 0, int Prepared = 0, int Submitted = 0,
-    int Workers = 20, int Days = 365, bool Percentiles = false, string Error = null, SampleBenchmarkResult Result = null,
+    int Workers = 20, int Days = 365, bool Percentiles = false, bool Indexes = true, string Error = null, SampleBenchmarkResult Result = null,
     long Ingested = 0, double ElapsedMs = 0, double EventsPerSecond = 0);
 sealed record SampleBenchmarkProgress(string State, int Prepared = 0, int Submitted = 0,
     long Ingested = 0, double ElapsedMs = 0, double EventsPerSecond = 0);
@@ -59,7 +60,7 @@ public async Task<SampleBenchmarkResult> RunAsync(SampleBenchmarkRequest request
   async Task<SampleBenchmarkResult> MeasureAsync(string directory, AnalyticsEvent[] events, SampleBenchmarkRequest request, DateTimeOffset end,
       double preparationMs, Action<SampleBenchmarkProgress> progress, CancellationToken cancellationToken, CancellationToken applicationStopping)
   {
-    var options = DashboardAnalyticsOptions.CreateBenchmark(directory, timeZone, request.Workers, request.Percentiles, logger);
+    var options = DashboardAnalyticsOptions.CreateBenchmark(directory, timeZone, request.Workers, request.Percentiles, request.Indexes, logger);
     await using var engine = new AnalyticsEngine(options);
     progress(new("running", Prepared: events.Length));
     var stopwatch = Stopwatch.StartNew();
@@ -107,7 +108,7 @@ async Task<SampleBenchmarkResult> MeasureAsync(string directory, AnalyticsEvent[
           statistics.EventsProcessed, statistics.Errors, statistics.EventsDropped, statistics.EventsRejected,
           build, GCSettings.IsServerGC ? "Server" : "Workstation", tiered == "0" ? "Disabled" : tiered == "1" ? "Enabled" : "Default",
           TimeZoneInfo.ConvertTime(end.AddDays(-request.Days), timeZone).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
-          TimeZoneInfo.ConvertTime(end, timeZone).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
+          TimeZoneInfo.ConvertTime(end, timeZone).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), request.Indexes);
     }
     catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
     {

apps/ZoneTree.Analytics.Dashboard/server/SampleDataJobs.cs

@@ -67,7 +67,7 @@ public bool Benchmark(SampleBenchmarkRequest request)
       var token = cancellation.Token;
       activeOperation = "benchmark";
       benchmarkStatus = new("preparing", request.Events, Workers: request.Workers, Days: request.Days,
-          Percentiles: request.Percentiles, Result: benchmarkStatus.Result);
+          Percentiles: request.Percentiles, Indexes: request.Indexes, Result: benchmarkStatus.Result);
       operation = Task.Run(async () =>
       {
         try

apps/ZoneTree.Analytics.Dashboard/ui/README.md

@@ -44,7 +44,7 @@ Edit `ConfigureCommon` for retention, storage, ingestion, queries, or rollups sh
 traffic, performance samples, new benchmarks and reopened benchmark reports. Raw events are
 retained for three years in all datasets. `ConfigurePrimary`, `CreatePerformanceSamples`,
 `CreateBenchmark` and `CreateBenchmarkReport` contain the per-engine settings; sample and
-benchmark worker counts come from the dashboard, and the benchmark percentile setting and
+benchmark worker counts come from the dashboard, and the benchmark percentile and index settings and
 reporting time zone are preserved when reopening a completed dataset. Each engine receives
 independent options and collections. Restart the host to apply edited engine settings.
 
@@ -56,7 +56,7 @@ Select **View sample reports**, or choose **Performance samples** in a report's
 
 **Delete sample data** closes the sample engine and removes its events, rollups and metadata, then permits a fresh dataset. Primary traffic and `Analytics:DataDirectory` are preserved. Sample controls and endpoints are available only in Development mode, including when `Dashboard:Watch=false`; production hosts do not expose them.
 
-**Benchmark ingestion** prepares the same realistic traffic in memory before timing, then feeds it to a fresh engine with the default per-worker limits. **Include load-time percentile aggregation** controls only percentile rollups; event values and traffic shape are retained. Progress polls lightweight engine counters every 250 ms and shows ingested events, queued events, elapsed time and average throughput so far. The progress bar follows processed events, then shows a separate saving-aggregates stage while the final flush finishes. The result separates preparation, submission, and ingestion-through-flush time. Throughput uses successfully processed events divided by the submission-through-flush interval, with preparation, store creation, dataset publication, disposal and cleanup excluded.
+**Benchmark ingestion** prepares the same realistic traffic in memory before timing, then feeds it to a fresh engine with the default per-worker limits. **Include load-time percentile aggregation** controls only percentile rollups; event values and traffic shape are retained. Clear **Maintain dimension and rollup indexes** to measure ingestion without the dashboard's secondary indexes; that run's reports read rollups and raw events directly. Progress polls lightweight engine counters every 250 ms and shows ingested events, queued events, elapsed time and average throughput so far. The progress bar follows processed events, then shows a separate saving-aggregates stage while the final flush finishes. The result separates preparation, submission, and ingestion-through-flush time. Throughput uses successfully processed events divided by the submission-through-flush interval, with preparation, store creation, dataset publication, disposal and cleanup excluded.
 
 Completed sample generations and benchmarks compact their stores before reporting completion,
 moving remaining in-memory records into disk segments and draining their WALs. The dashboard
@@ -72,7 +72,7 @@ $env:DOTNET_TieredCompilation = '0'
 dotnet run -c Release --project apps/ZoneTree.Analytics.Dashboard/server -- --Dashboard:Watch=false
 ```
 
-The dashboard server exposes `GET /_analytics/sample-data/status` and JSON `POST` requests to `/generate` (`{"events":1000000,"days":60,"workers":8}`), `/benchmark` (`{"events":5000000,"days":90,"workers":20,"percentiles":false}`), `/cancel` (`{}`), `/clear` (`{}`), and `/clear-benchmark` (`{}`). Generation workers default to 1; benchmark defaults are described above. Status includes the active operation, generation progress, benchmark preparation/ingestion progress, and the previous completed benchmark result. Its config response advertises both report sites and their endpoints; other sites continue using the configured reporting API.
+The dashboard server exposes `GET /_analytics/sample-data/status` and JSON `POST` requests to `/generate` (`{"events":1000000,"days":60,"workers":8}`), `/benchmark` (`{"events":5000000,"days":90,"workers":20,"percentiles":false,"indexes":true}`), `/cancel` (`{}`), `/clear` (`{}`), and `/clear-benchmark` (`{}`). Generation workers default to 1; benchmark defaults are described above. Status includes the active operation, generation progress, benchmark preparation/ingestion progress, and the previous completed benchmark result. Its config response advertises both report sites and their endpoints; other sites continue using the configured reporting API.
 
 ## Build and host
 

apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/benchmark.ts

@@ -17,6 +17,7 @@ export interface SampleBenchmarkPanel {
   days: SRef<number | string>
   workers: SRef<number | string>
   percentiles: SRef<boolean>
+  indexes: SRef<boolean>
   benchmark: ComputedRef<SampleBenchmarkStatus | null>
   run: (event: Event) => Promise<void>
   seconds: (milliseconds: number) => string
@@ -68,6 +69,11 @@ const template = html`<Panel class="form-panel sample-benchmark-panel">
       label="Include load-time percentile aggregation"
       :checked="percentiles"
       :disabled="jobs.busy"/>
+    <FormCheck
+      id="benchmark-indexes"
+      label="Maintain dimension and rollup indexes"
+      :checked="indexes"
+      :disabled="jobs.busy"/>
     <div
       class="sample-job-status"
       r-if="benchmark && benchmark.state !== 'idle'"
@@ -169,6 +175,7 @@ const template = html`<Panel class="form-panel sample-benchmark-panel">
       {{ dashboard.number(benchmark.result.events) }} events · {{ benchmark.result.days }} days ·
       {{ benchmark.result.workers }} workers · Percentiles
       {{ benchmark.result.percentiles ? 'on' : 'off' }}
+      · Indexes {{ benchmark.result.indexes ? 'on' : 'off' }}
     </p>
     <dl class="sample-ingestion-metrics">
       <div>
@@ -221,12 +228,14 @@ export function defineSampleBenchmarkPanel() {
       const days = sref<number | string>(365)
       const workers = sref<number | string>(20)
       const percentiles = sref(false)
+      const indexes = sref(true)
       return {
         ...head.props,
         count,
         days,
         workers,
         percentiles,
+        indexes,
         benchmark: computed(() => jobs.status()?.benchmark ?? null),
         viewUrl: computed(() => {
           const model = unref(head.props.dashboard)
@@ -267,6 +276,7 @@ export function defineSampleBenchmarkPanel() {
             days: span,
             workers: concurrency,
             percentiles: percentiles(),
+            indexes: indexes(),
           })
         },
       }

apps/ZoneTree.Analytics.Dashboard/ui/content/sample-data/jobs.ts

@@ -13,6 +13,7 @@ export interface SampleBenchmarkResult {
   days: number
   workers: number
   percentiles: boolean
+  indexes: boolean
   preparationMs: number
   submissionMs: number
   ingestionMs: number
@@ -47,6 +48,7 @@ export interface SampleBenchmarkStatus {
   workers: number
   days: number
   percentiles: boolean
+  indexes: boolean
   error: string | null
   result: SampleBenchmarkResult | null
   ingested: number

apps/ZoneTree.Analytics.Dashboard/ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zonetree-analytics-dashboard",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "private": true,
   "type": "module",
   "scripts": {
@@ -10,13 +10,13 @@
     "publish": "purestack publish --content ./content"
   },
   "dependencies": {
-    "@purestack/ts-components": "^1.1.14",
-    "@purestack/ts-css": "^1.1.14",
-    "@purestack/ts-html": "^1.1.14",
-    "@purestack/ts-style": "^1.1.14",
-    "@purestack/ts-svg-icons": "^1.1.14",
-    "@purestack/ts-util": "^1.1.14",
-    "purestack": "^1.1.14",
+    "@purestack/ts-components": "^1.1.15",
+    "@purestack/ts-css": "^1.1.15",
+    "@purestack/ts-html": "^1.1.15",
+    "@purestack/ts-style": "^1.1.15",
+    "@purestack/ts-svg-icons": "^1.1.15",
+    "@purestack/ts-util": "^1.1.15",
+    "purestack": "^1.1.15",
     "regor": "^1.7.5"
   },
   "devDependencies": {

apps/ZoneTree.Analytics.Dashboard/ui/tests/sample-data-browser.ts

@@ -14,6 +14,7 @@ export async function checkSampleDataTools(page: Page, base: string) {
     days: 7,
     workers: 2,
     percentiles: false,
+    indexes: true,
     preparationMs: 100,
     submissionMs: 200,
     ingestionMs: 500,
@@ -48,6 +49,7 @@ export async function checkSampleDataTools(page: Page, base: string) {
       workers: 1,
       days: 30,
       percentiles: false,
+      indexes: true,
       error: null,
       result,
       ingested: 0,
@@ -93,6 +95,7 @@ export async function checkSampleDataTools(page: Page, base: string) {
           workers: 2,
           days: 7,
           percentiles: true,
+          indexes: false,
         },
       }
     } else if (endpoint === 'cancel') {
@@ -131,6 +134,13 @@ export async function checkSampleDataTools(page: Page, base: string) {
   await page
     .getByText('Include load-time percentile aggregation', { exact: true })
     .click()
+  assert.equal(
+    await page.getByLabel('Maintain dimension and rollup indexes').isChecked(),
+    true,
+  )
+  await page
+    .getByText('Maintain dimension and rollup indexes', { exact: true })
+    .click()
   assert.equal(
     await page
       .getByLabel('Include load-time percentile aggregation')
@@ -140,7 +150,7 @@ export async function checkSampleDataTools(page: Page, base: string) {
   await page.getByRole('button', { name: 'Run benchmark', exact: true }).click()
   await page.getByText('Preparing 0 / 17 events', { exact: true }).waitFor()
   assert.deepEqual(submissions, [
-    { events: 17, days: 7, workers: 2, percentiles: true },
+    { events: 17, days: 7, workers: 2, percentiles: true, indexes: false },
   ])
   assert.equal(
     await page
@@ -196,7 +206,7 @@ export async function checkSampleDataTools(page: Page, base: string) {
     benchmark: {
       ...status.benchmark,
       state: 'completed',
-      result: { ...result, percentiles: true },
+      result: { ...result, percentiles: true, indexes: false },
     },
   }
   await page.getByText('Benchmark completed', { exact: true }).waitFor()
@@ -213,9 +223,12 @@ export async function checkSampleDataTools(page: Page, base: string) {
     true,
   )
   await page
-    .getByText('17 events · 7 days · 2 workers · Percentiles on', {
-      exact: true,
-    })
+    .getByText(
+      '17 events · 7 days · 2 workers · Percentiles on · Indexes off',
+      {
+        exact: true,
+      },
+    )
     .waitFor()
   assert.equal(
     await page
@@ -255,6 +268,6 @@ export async function checkSampleDataTools(page: Page, base: string) {
     fullPage: true,
   })
   console.log(
-    'Sample tools checks passed: benchmark inputs, percentile toggle, stages, shared busy state, cancellation, previous results and mobile layout (mock responses).',
+    'Sample tools checks passed: benchmark inputs, percentile and index toggles, stages, shared busy state, cancellation, previous results and mobile layout (mock responses).',
   )
 }