diff --git a/benchmark/_benchmark_progress.js b/benchmark/_benchmark_progress.js index 6c925f34e68..117e8660902 100644 --- a/benchmark/_benchmark_progress.js +++ b/benchmark/_benchmark_progress.js @@ -25,9 +25,10 @@ function getTime(diff) { // A run is an item in the job queue: { binary, filename, iter } // A config is an item in the subqueue: { binary, filename, iter, configs } class BenchmarkProgress { - constructor(queue, benchmarks) { + constructor(queue, benchmarks, options = {}) { this.queue = queue; // Scheduled runs. this.benchmarks = benchmarks; // Filenames of scheduled benchmarks. + this.analyze = !!options.analyze; // stdout is not piped, but unused. this.completedRuns = 0; // Number of completed runs. this.scheduledRuns = queue.length; // Number of scheduled runs. // Time when starting to run benchmarks. @@ -107,7 +108,10 @@ class BenchmarkProgress { } updateProgress() { - if (!process.stderr.isTTY || process.stdout.isTTY) { + // Progress renders on stderr when stdout is piped (not a TTY). + // In --analyze mode, stdout is the terminal but is unused during + // the run, so treat it the same as piped. + if (!process.stderr.isTTY || (process.stdout.isTTY && !this.analyze)) { return; } readline.clearLine(process.stderr); diff --git a/benchmark/compare.js b/benchmark/compare.js index ad3084db390..6aaaee7a919 100644 --- a/benchmark/compare.js +++ b/benchmark/compare.js @@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] ... Run each benchmark in the directory many times using two different node versions. More than one directory can be specified. The output is formatted as csv, which can be processed using for - example 'compare.R'. + example 'compare.R'. Use --analyze to perform statistical analysis + directly without R. --new ./new-node-binary new node binary (required) --old ./old-node-binary old node binary (required) @@ -24,13 +25,21 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] ... repeated) --set variable=value set benchmark variable (can be repeated) --no-progress don't show benchmark progress indicator + --analyze perform statistical analysis after benchmarks + complete (Welch's t-test, effect size) instead + of printing csv output + --scale 1000 rate-to-integer multiplier for histogram + precision when using --analyze (default: 1000) + --max-regression N exit with code 1 if any statistically + significant regression exceeds N% (implies + --analyze) Examples: --set CPUSET=0 Runs benchmarks on CPU core 0. --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2. Note: The CPUSET format should match the specifications of the 'taskset' command -`, { arrayArgs: ['set', 'filter', 'exclude'], boolArgs: ['no-progress'] }); +`, { arrayArgs: ['set', 'filter', 'exclude'], boolArgs: ['no-progress', 'analyze'] }); if (!cli.optional.new || !cli.optional.old) { cli.abort(cli.usage); @@ -38,6 +47,11 @@ if (!cli.optional.new || !cli.optional.old) { const binaries = ['old', 'new']; const runs = cli.optional.runs ? parseInt(cli.optional.runs, 10) : 30; +const maxRegression = cli.optional['max-regression'] ? + parseFloat(cli.optional['max-regression']) : + 0; +const analyze = !!cli.optional.analyze || maxRegression > 0; +const scale = cli.optional.scale ? parseInt(cli.optional.scale, 10) : 1000; const benchmarks = cli.benchmarks(); if (benchmarks.length === 0) { @@ -46,6 +60,9 @@ if (benchmarks.length === 0) { return; } +// When --analyze is set, collect results for statistical analysis. +const results = analyze ? new Map() : null; + // Create queue from the benchmarks list such both node versions are tested // `runs` amount of times each. // Note: BenchmarkProgress relies on this order to estimate @@ -61,15 +78,17 @@ for (const filename of benchmarks) { } // queue.length = binary.length * runs * benchmarks.length -// Print csv header -console.log('"binary","filename","configuration","rate","time"'); +// Print csv header (unless analyzing inline). +if (!analyze) { + console.log('"binary","filename","configuration","rate","time"'); +} const kStartOfQueue = 0; const showProgress = !cli.optional['no-progress']; let progress; if (showProgress) { - progress = new BenchmarkProgress(queue, benchmarks); + progress = new BenchmarkProgress(queue, benchmarks, { analyze }); progress.startQueue(kStartOfQueue); } @@ -99,11 +118,20 @@ if (showProgress) { conf += ` ${key}=${inspect(data.conf[key])}`; } conf = conf.slice(1); - // Escape quotes (") for correct csv formatting - conf = conf.replace(/"/g, '""'); - console.log(`"${job.binary}","${job.filename}","${conf}",` + - `${data.rate},${data.time}`); + if (analyze) { + // Collect results for post-run analysis. + const name = `${job.filename} ${conf}`; + if (!results.has(name)) { + results.set(name, { old: [], new: [] }); + } + results.get(name)[job.binary].push(data.rate); + } else { + // Escape quotes (") for correct csv formatting + conf = conf.replace(/"/g, '""'); + console.log(`"${job.binary}","${job.filename}","${conf}",` + + `${data.rate},${data.time}`); + } if (showProgress) { // One item in the subqueue has been completed. progress.completeConfig(data); @@ -125,6 +153,199 @@ if (showProgress) { // If there are more benchmarks execute the next if (i + 1 < queue.length) { recursive(i + 1); + } else if (analyze) { + printAnalysis(results, scale, maxRegression); } }); })(kStartOfQueue); + +function printAnalysis(results, scale, maxRegression) { + const { createHistogram } = require('node:perf_hooks'); + + // Build per-benchmark histograms and run statistical tests. + const rows = []; + let maxNameLen = 0; + + let skipped = 0; + + for (const [name, { old: oldRates, new: newRates }] of results) { + if (oldRates.length < 2 || newRates.length < 2) { + skipped++; + continue; + } + + const hOld = createHistogram({ figures: 3 }); + const hNew = createHistogram({ figures: 3 }); + + for (const r of oldRates) hOld.record(Math.max(1, Math.round(r * scale))); + for (const r of newRates) hNew.record(Math.max(1, Math.round(r * scale))); + + const oldMean = oldRates.reduce((a, b) => a + b, 0) / oldRates.length; + const newMean = newRates.reduce((a, b) => a + b, 0) / newRates.length; + const improvement = ((newMean - oldMean) / oldMean) * 100; + + // Query the three confidence levels. The p-value and t-statistic + // are the same regardless of the confidence level, so we extract + // them from the first result. + const w95 = hOld.welchTest(hNew, { confidence: 0.95 }); + const w99 = hOld.welchTest(hNew, { confidence: 0.99 }); + const w999 = hOld.welchTest(hNew, { confidence: 0.999 }); + + // Significance stars matching compare.R convention. + let stars = ''; + if (w95.pValue < 0.001) stars = '***'; + else if (w95.pValue < 0.01) stars = ' **'; + else if (w95.pValue < 0.05) stars = ' *'; + + // Confidence intervals expressed as percentage of the old mean. + const ciPct = (w) => { + const half = + (w.confidenceInterval.upper - w.confidenceInterval.lower) / 2; + return (half / (oldMean * scale)) * 100; + }; + + rows.push({ + name, + stars, + improvement, + ci95: ciPct(w95), + ci99: ciPct(w99), + ci999: ciPct(w999), + pValue: w95.pValue, + }); + + if (name.length > maxNameLen) maxNameLen = name.length; + } + + // Print header. + const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length)); + const rpad = (s, n) => ' '.repeat(Math.max(0, n - s.length)) + s; + + console.log(`${pad('', maxNameLen)} confidence` + + ` improvement accuracy (*) (**) (***)`); + + for (const row of rows) { + const imp = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)} %`; + console.log( + `${pad(row.name, maxNameLen)} ${pad(row.stars, 10)}` + + ` ${rpad(imp, 11)}` + + ` ±${row.ci95.toFixed(2)}%` + + ` ±${row.ci99.toFixed(2)}%` + + ` ±${row.ci999.toFixed(2)}%`, + ); + } + + if (skipped > 0) { + console.log(''); + console.log( + `Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` + + ` skipped because Welch's t-test requires at least 2 samples per` + + ` binary. Use --runs 2 or higher.`, + ); + } + + // --- Bar chart visualization --- + printChart(rows, maxNameLen); + + console.log(''); + console.log( + `Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n` + + `Use --scale to adjust precision if needed.\n`, + ); + console.log( + `Be aware that when doing many comparisons the risk of a false-positive\n` + + `result increases. In this case, there are ${rows.length} comparisons, ` + + `you can thus\nexpect the following amount of false-positive results:\n` + + ` ${(rows.length * 0.05).toFixed(2)} false positives, when considering ` + + `a 5% risk acceptance (*, **, ***),\n` + + ` ${(rows.length * 0.01).toFixed(2)} false positives, when considering ` + + `a 1% risk acceptance (**, ***),\n` + + ` ${(rows.length * 0.001).toFixed(2)} false positives, when considering ` + + `a 0.1% risk acceptance (***)`, + ); + + // Gate: exit with error if any significant regression exceeds the limit. + if (maxRegression > 0) { + const failures = rows.filter( + (r) => r.stars.trim() !== '' && r.improvement < -maxRegression, + ); + if (failures.length > 0) { + console.log(''); + console.log( + `FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` + + ` showed a statistically significant regression exceeding` + + ` ${maxRegression}%:`, + ); + for (const f of failures) { + console.log(` ${f.name} ${f.improvement.toFixed(2)}%`); + } + process.exitCode = 1; + } + } +} + +function printChart(rows, maxNameLen) { + if (rows.length === 0) return; + + // Determine the chart scale from the data. The bar region covers + // the range [-maxAbs, +maxAbs] so the zero line sits in the center. + const barWidth = 40; + const halfWidth = barWidth / 2; + let maxAbs = 0; + for (const row of rows) { + const extent = Math.abs(row.improvement) + row.ci95; + if (extent > maxAbs) maxAbs = extent; + } + if (maxAbs === 0) maxAbs = 1; + + const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length)); + + // Scale axis labels. + const axisLeft = `-${maxAbs.toFixed(1)}%`; + const axisRight = `+${maxAbs.toFixed(1)}%`; + const axisCenter = '0%'; + + // Print axis header. + const labelPad = maxNameLen + 5; + const leftLabel = ' '.repeat(labelPad) + + axisLeft + + ' '.repeat(Math.max(0, halfWidth - axisLeft.length - Math.floor(axisCenter.length / 2))) + + axisCenter + + ' '.repeat(Math.max(0, halfWidth - Math.ceil(axisCenter.length / 2) - axisRight.length)) + + axisRight; + console.log(''); + console.log(leftLabel); + + for (const row of rows) { + const imp = row.improvement; + const ci = row.ci95; + + // Position of the improvement value in the bar region [0, barWidth]. + const center = halfWidth; + const impPos = center + (imp / maxAbs) * halfWidth; + + // CI extent in bar positions. + const ciLeft = center + ((imp - ci) / maxAbs) * halfWidth; + const ciRight = center + ((imp + ci) / maxAbs) * halfWidth; + + // Build the bar character by character. + const chars = []; + for (let x = 0; x < barWidth; x++) { + const pos = x + 0.5; // Center of this character cell. + if (x === Math.floor(center)) { + chars.push('|'); + } else if ((imp >= 0 && pos > center && pos <= impPos) || + (imp < 0 && pos < center && pos >= impPos)) { + chars.push(row.stars ? '\u2588' : '\u2593'); // solid or dark shade + } else if (pos >= ciLeft && pos <= ciRight) { + chars.push('\u2591'); // Light shade for CI region + } else { + chars.push(' '); + } + } + + const label = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)}%`; + const sig = row.stars.trim(); + console.log(`${pad(row.name, maxNameLen)} ${chars.join('')} ${label} ${sig}`); + } +} diff --git a/doc/api/perf_hooks.md b/doc/api/perf_hooks.md index eb2076eb792..2bf1b247538 100644 --- a/doc/api/perf_hooks.md +++ b/doc/api/perf_hooks.md @@ -1625,10 +1625,48 @@ added: **Default:** `Number.MAX_SAFE_INTEGER`. * `figures` {number} The number of accuracy digits. Must be a number between `1` and `5`. **Default:** `3`. + * `halfLife` {number} The EWMA half-life in number of samples. When set to + a value greater than 0, the histogram tracks an exponentially weighted + moving average and standard deviation, accessible via + `histogram.ewmaMean` and `histogram.ewmaStddev`. After `halfLife` + recordings, a value's influence has decayed to 50%. **Default:** `0` + (disabled). + * `threshold` {number} An SLO threshold value. When set together with + `halfLife`, the histogram tracks a smoothed error rate for values + exceeding this threshold, accessible via `histogram.ewmaErrorRate` and + `histogram.burnRate()`. **Default:** `0` (disabled). * Returns: {RecordableHistogram} Returns a {RecordableHistogram}. +## `perf_hooks.importHistogram(data)` + + + +* `data` {Uint8Array} A CBOR-encoded histogram previously produced by + [`histogram.export()`][]. +* Returns: {RecordableHistogram} + +Reconstructs a histogram from a CBOR-encoded `Uint8Array`. The returned +histogram is a full {RecordableHistogram} with all bucket data, configuration, +and EWMA state restored. New values can be recorded into it. + +```js +const { createHistogram, importHistogram } = require('node:perf_hooks'); + +const h = createHistogram(); +for (let i = 1; i <= 1000; i++) h.record(i); + +// Serialize and reconstruct +const data = h.export(); +const h2 = importHistogram(data); + +console.log(h2.count); // 1000 +console.log(h2.percentile(99)); // Same as h.percentile(99) +``` + ## `perf_hooks.eventLoopUtilization([utilization1[, utilization2]])` +### `histogram.burnRate(sloTarget)` + + + +* `sloTarget` {number} The SLO target as a fraction between 0 and 1 + (exclusive). For example, `0.999` for a 99.9% SLO. +* Returns: {number} + +Returns the SLO burn rate: `ewmaErrorRate / (1 - sloTarget)`. A burn rate +of 1 means the error budget will be exactly exhausted over the SLO window. +A burn rate greater than 1 means it is being consumed faster than allowed. +Requires the histogram to have been created with both `halfLife` and +`threshold` options. + +```js +const { createHistogram } = require('node:perf_hooks'); + +// Track latency with a 200ms SLO threshold, half-life of 100 samples +const h = createHistogram({ halfLife: 100, threshold: 200_000_000 }); + +// ... record latency values ... + +// Check burn rate against a 99.9% SLO +const rate = h.burnRate(0.999); +if (rate > 1) { + console.log(`SLO burn rate: ${rate.toFixed(2)}x — error budget depleting`); +} +``` + ### `histogram.count` + +* `other` {Histogram} The histogram to compare against. +* Returns: {number} A value between -1.0 and 1.0. + +Computes [Cliff's delta][], a non-parametric effect size measure. Returns +the probability that a random value from this histogram exceeds a random +value from `other`, minus the reverse probability. A value of 1 means every +value in this histogram exceeds every value in `other`; -1 means the +opposite; 0 means no tendency in either direction. + +### `histogram.cohensD(other)` + + + +* `other` {Histogram} The histogram to compare against. +* Returns: {number} The effect size. + +Computes [Cohen's d][] effect size, the standardized difference between the +means of this histogram and `other`, using the pooled standard deviation. +Positive values indicate this histogram has a higher mean. By convention, +|d| < 0.2 is a small effect, 0.5 is medium, and 0.8 or greater is large. +Both histograms must have at least 2 recorded values; otherwise returns 0. + ### `histogram.countAt(value)` + +* Returns: {Uint8Array} + +Serializes the histogram to a [CBOR][]-encoded (RFC 8949) `Uint8Array` +suitable for transmission or persistent storage. The encoding uses a +delta-encoded sparse representation of the bucket counts, so the output size +scales with the number of distinct recorded values rather than the total +bucket count. + +The output includes all histogram configuration, bucket data, and EWMA +state (when enabled). It can be reconstructed into a new histogram using +[`perf_hooks.importHistogram()`][]. + +The CBOR payload is a map with integer keys: + +| Key | Type | Field | +| --- | ------- | --------------------------------------------- | +| 0 | uint | Format version (currently 1) | +| 1 | uint | Lowest discernible value | +| 2 | uint | Highest trackable value | +| 3 | uint | Significant figures | +| 4 | uint | Total count | +| 5 | uint | Min value | +| 6 | uint | Max value | +| 7 | uint | Normalizing index offset | +| 8 | float64 | Conversion ratio | +| 9 | uint | Counts array length | +| 10 | array | Delta-encoded sparse counts `[delta, c, ...]` | +| 11 | map | EWMA state (omitted when disabled) | + +Any standard CBOR decoder can parse the output. + +### `histogram.ewmaMean` + + + +* Type: {number} + +The exponentially weighted moving average of recorded values. Only active +when the histogram was created with a `halfLife` option greater than 0. +Returns `0` when EWMA is disabled or no values have been recorded. + +### `histogram.ewmaStddev` + + + +* Type: {number} + +The exponentially weighted moving standard deviation. Only active when the +histogram was created with a `halfLife` option greater than 0. Returns `0` +when EWMA is disabled or no values have been recorded. + +### `histogram.ewmaErrorRate` + + + +* Type: {number} + +The EWMA-smoothed probability of a recorded value exceeding the configured +`threshold`. Only active when the histogram was created with both `halfLife` +and `threshold` options. Returns `0` when not enabled or no values have been +recorded. + ### `histogram.ksTest(other)` + +* `other` {Histogram} The histogram to compare against. +* Returns: {Object} + * `uStatistic` {number} The Mann-Whitney U statistic. + * `zScore` {number} The z-score (normal approximation). + * `pValue` {number} Two-tailed p-value. + +Performs a [Mann-Whitney U test][] comparing whether this histogram tends to +produce larger or smaller values than `other`. Unlike `welchTest()`, this is a +non-parametric test that makes no assumptions about the shape of the +distributions. Uses the normal approximation with tie correction for the +p-value. + ### `histogram.max` + +* `percentile` {number} A percentile value in the range (0, 100]. +* `options` {Object} + * `confidence` {number} The confidence level for the interval, between + 0 and 1 (exclusive). **Default:** `0.95`. +* Returns: {Object} + * `value` {number} The point estimate (same as `histogram.percentile()`). + * `lower` {number} The lower bound of the confidence interval. + * `upper` {number} The upper bound of the confidence interval. + +Returns a confidence interval for the given percentile using the exact +binomial method. With fewer samples, the interval will be wider, reflecting +the greater uncertainty in the percentile estimate. Requires at least 2 +recorded values; with fewer than 2, `lower` and `upper` will equal `value`. + +```js +const { createHistogram } = require('node:perf_hooks'); + +const h = createHistogram(); +for (let i = 0; i < 1000; i++) { + h.record(Math.floor(Math.random() * 100)); +} + +const ci = h.percentileCI(99); +console.log(ci.value); // The p99 point estimate +console.log(ci.lower); // The lower bound (95% confidence) +console.log(ci.upper); // The upper bound (95% confidence) +``` + ### `histogram.percentiles` + +* `other` {Histogram} The histogram to compare against. +* `options` {Object} + * `confidence` {number} Confidence level for the interval, between 0 and 1. + **Default:** `0.95`. +* Returns: {Object} + * `tStatistic` {number} The Welch t-statistic. + * `degreesOfFreedom` {number} Welch-Satterthwaite degrees of freedom. + * `pValue` {number} Two-tailed p-value. + * `confidenceInterval` {Object} + * `lower` {number} Lower bound of the confidence interval on the + difference of means. + * `upper` {number} Upper bound. + +Performs [Welch's t-test][] comparing the means of this histogram and `other`. +The p-value indicates the probability of observing a difference at least this +extreme under the null hypothesis that the two distributions have the same +mean. Both histograms must have at least 2 recorded values; otherwise the +result has `pValue` 1 and `tStatistic` 0. + ## Class: `ELDHistogram extends Histogram` A `Histogram` that records event loop delay, returned by @@ -2290,6 +2540,32 @@ const violating = latency.ccdf(500_000_000); console.log(`${(violating * 100).toFixed(1)}% of requests violating SLO`); ``` +### SLO burn rate monitoring + +```js +const { createHistogram } = require('node:perf_hooks'); + +// Track latency with EWMA (half-life 100 samples) and a 200ms SLO threshold +const latency = createHistogram({ + halfLife: 100, + threshold: 200_000_000, // 200ms in nanoseconds +}); + +// Record request latencies... + +// Smoothed error rate: probability of exceeding the threshold +console.log(`Error rate: ${(latency.ewmaErrorRate * 100).toFixed(2)}%`); + +// Burn rate against a 99.9% SLO +// >1 means the error budget is depleting faster than allowed +const rate = latency.burnRate(0.999); +console.log(`Burn rate: ${rate.toFixed(2)}x`); + +// EWMA mean and stddev track the smoothed latency +console.log(`EWMA latency: ${latency.ewmaMean.toFixed(0)}ns`); +console.log(`EWMA stddev: ${latency.ewmaStddev.toFixed(0)}ns`); +``` + ### Regression detection with KS test ```js @@ -2341,6 +2617,46 @@ newSnapshot.subtract(snapshot); console.log('Recent p99:', newSnapshot.percentile(99)); ``` +### Benchmark comparison with Welch's t-test + +```js +const { createHistogram } = require('node:perf_hooks'); + +const baseline = createHistogram(); +const candidate = createHistogram(); + +// Record operation rates from the old and new builds... + +const result = baseline.welchTest(candidate); +const improvement = ((candidate.mean - baseline.mean) / baseline.mean * 100); + +console.log(`Improvement: ${improvement.toFixed(2)}%`); +console.log(`p-value: ${result.pValue.toFixed(6)}`); +console.log(`95% CI: [${result.confidenceInterval.lower.toFixed(2)}, ` + + `${result.confidenceInterval.upper.toFixed(2)}]`); + +if (result.pValue < 0.05) { + const d = baseline.cohensD(candidate); + console.log(`Statistically significant (Cohen's d = ${d.toFixed(4)})`); +} +``` + +### Effect size with Cliff's delta + +```js +const { createHistogram } = require('node:perf_hooks'); + +const before = createHistogram(); +const after = createHistogram(); + +// Record latencies before and after a change... + +const delta = before.cliffsD(after); +// A delta > 0: before tends to produce larger values (improvement) +// A delta < 0: after tends to produce larger values (regression) +console.log(`Cliff's delta: ${delta.toFixed(4)}`); +``` + ## Examples ### Measuring the duration of async operations @@ -2595,17 +2911,24 @@ dns.promises.resolve('localhost'); ``` [Async Hooks]: async_hooks.md +[CBOR]: https://www.rfc-editor.org/rfc/rfc8949 +[Cliff's delta]: https://en.wikipedia.org/wiki/Effect_size#Cliff's_delta +[Cohen's d]: https://en.wikipedia.org/wiki/Effect_size#Cohen's_d [Fetch Response Body Info]: https://fetch.spec.whatwg.org/#response-body-info [Fetch Timing Info]: https://fetch.spec.whatwg.org/#fetch-timing-info [High Resolution Time]: https://www.w3.org/TR/hr-time-2 +[Mann-Whitney U test]: https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test [Performance Timeline]: https://w3c.github.io/performance-timeline/ [Resource Timing]: https://www.w3.org/TR/resource-timing-2/ [User Timing]: https://www.w3.org/TR/user-timing/ [Web Performance APIs]: https://w3c.github.io/perf-timing-primer/ +[Welch's t-test]: https://en.wikipedia.org/wiki/Welch%27s_t-test [Worker threads]: worker_threads.md#worker-threads [`'exit'`]: process.md#event-exit [`child_process.spawnSync()`]: child_process.md#child_processspawnsynccommand-args-options +[`histogram.export()`]: #histogramexport [`perf_hooks.eventLoopUtilization()`]: #perf_hookseventlooputilizationutilization1-utilization2 +[`perf_hooks.importHistogram()`]: #perf_hooksimporthistogramdata [`perf_hooks.monitorEventLoopDelay()`]: #perf_hooksmonitoreventloopdelayoptions [`perf_hooks.timerify()`]: #perf_hookstimerifyfn-options [`process.hrtime()`]: process.md#processhrtimetime diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index 23132d9eb0f..a31e0e82aab 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -14,6 +14,8 @@ * [Specifying CPU Cores for Benchmarks with run.js](#specifying-cpu-cores-for-benchmarks-with-runjs) * [Filtering benchmarks](#filtering-benchmarks) * [Comparing Node.js versions](#comparing-nodejs-versions) + * [Using `--analyze` (no external tools needed)](#using---analyze-no-external-tools-needed) + * [Using R scripts or node-benchmark-compare](#using-r-scripts-or-node-benchmark-compare) * [Comparing parameters](#comparing-parameters) * [Running benchmarks on the CI](#running-benchmarks-on-the-ci) * [Creating a benchmark](#creating-a-benchmark) @@ -73,18 +75,27 @@ node benchmark/http2/simple.js benchmarker=h2load ### Benchmark analysis requirements -To analyze the results statistically, you can use either the -[node-benchmark-compare][] tool or the R script `benchmark/compare.R`. +To analyze the results statistically, there are three options: -[node-benchmark-compare][] is a Node.js script that can be installed with -`npm install -g node-benchmark-compare`. +* **`--analyze` flag** (built-in, no dependencies): Pass `--analyze` to + `benchmark/compare.js` to perform Welch's t-test directly after the + benchmarks complete. This uses the histogram API's statistical testing + methods and requires no external tools. +* **R scripts** (`benchmark/compare.R`, `benchmark/bar.R`): Perform the same + Welch's t-test analysis as `--analyze`, with the additional ability to + generate plots. Requires R with the `ggplot2` and `plyr` packages. +* **[node-benchmark-compare][]** (legacy): A Node.js script that can be + installed with `npm install -g node-benchmark-compare`. It reads the CSV + output of `benchmark/compare.js`. Predates the built-in `--analyze` flag + and is no longer necessary for most workflows. -To draw comparison plots when analyzing the results, `R` must be installed. -Use one of the available package managers or download it from -. +For most use cases, `--analyze` is the simplest option since it requires +nothing beyond Node.js itself. -The R packages `ggplot2` and `plyr` are also used and can be installed using -the R REPL. +To install R for plot generation, use one of the available package managers or +download it from . + +The R packages `ggplot2` and `plyr` can be installed using the R REPL. ```console $ R @@ -403,16 +414,38 @@ module, you can use the `--filter` option:_ repeated) --set variable=value set benchmark variable (can be repeated) --no-progress don't show benchmark progress indicator + --analyze perform statistical analysis inline (no R needed) + --scale 1000 rate multiplier for --analyze precision + --max-regression N exit with code 1 if any significant regression + exceeds N% (implies --analyze) +``` + +#### Using `--analyze` (no external tools needed) + +The simplest way to get statistical results is to pass `--analyze`: + +```bash +node benchmark/compare.js --old ./node-main --new ./node-pr-5134 --analyze string_decoder +``` - Examples: - --set CPUSET=0 Runs benchmarks on CPU core 0. - --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2. +This runs the benchmarks and prints the analysis directly: - Note: The CPUSET format should match the specifications of the 'taskset' command +```console + confidence improvement accuracy (*) (**) (***) +string_decoder/string-decoder.js n=2500000 chunkLen=16 inLen=128 encoding='ascii' *** -3.76 % ±1.36% ±1.82% ±2.40% +string_decoder/string-decoder.js n=2500000 chunkLen=16 inLen=128 encoding='utf8' ** -0.81 % ±0.53% ±0.71% ±0.93% +... ``` -For analyzing the benchmark results, use [node-benchmark-compare][] or the R -scripts: +The `--analyze` mode uses the histogram API's `welchTest()` method to perform +the same Welch's t-test that the R script uses. Benchmark rates are scaled to +integers for the histogram (controlled by `--scale`, default 1000). With the +default settings, results are identical to the R script at two decimal places. + +#### Using R scripts or node-benchmark-compare + +Alternatively, save the CSV output and analyze it separately using +[node-benchmark-compare][] or the R scripts: * `benchmark/compare.R` * `benchmark/bar.R` @@ -428,6 +461,10 @@ $ node-benchmark-compare compare-pr-5134.csv # or cat compare-pr-5134.csv | Rscr ... ``` +The R approach is still useful when you need to generate plots (box plots via +`compare.R --plot`, scatter plots via `scatter.R --plot`) or when you want to +analyze previously saved CSV files. + In the output, _improvement_ is the relative improvement of the new version, hopefully this is positive. _confidence_ tells if there is enough statistical evidence to validate the _improvement_. If there is enough evidence diff --git a/lib/internal/histogram.js b/lib/internal/histogram.js index c16c894dd14..99c2c1d87da 100644 --- a/lib/internal/histogram.js +++ b/lib/internal/histogram.js @@ -38,6 +38,10 @@ const { validateObject, } = require('internal/validators'); +const { + isUint8Array, +} = require('internal/util/types'); + const kDestroy = Symbol('kDestroy'); const kHandle = Symbol('kHandle'); const kRecordable = Symbol('kRecordable'); @@ -213,6 +217,79 @@ class Histogram { return this[kHandle]?.exceedsBigInt(); } + /** + * Serializes the histogram to a CBOR-encoded Uint8Array suitable for + * transmission or storage. The data can be reconstructed into a new + * histogram using `importHistogram()`. + * @returns {Uint8Array} + */ + export() { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + return this[kHandle]?.export(); + } + + /** + * Returns the exponentially weighted moving average of recorded values. + * Only active when the histogram was created with a `halfLife` option. + * Returns 0 when EWMA is not enabled or no values have been recorded. + * @readonly + * @type {number} + */ + get ewmaMean() { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + return this[kHandle]?.ewmaMean(); + } + + /** + * Returns the exponentially weighted moving standard deviation. + * Only active when the histogram was created with a `halfLife` option. + * Returns 0 when EWMA is not enabled or no values have been recorded. + * @readonly + * @type {number} + */ + get ewmaStddev() { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + return this[kHandle]?.ewmaStddev(); + } + + /** + * Returns the EWMA-smoothed error rate: the probability of a recorded + * value exceeding the configured `threshold`. Only active when the + * histogram was created with both `halfLife` and `threshold` options. + * Returns 0 when not enabled or no values have been recorded. + * @readonly + * @type {number} + */ + get ewmaErrorRate() { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + return this[kHandle]?.ewmaErrorRate(); + } + + /** + * Returns the SLO burn rate: how fast the error budget is being consumed. + * A burn rate of 1 means the budget will be exactly exhausted over the + * SLO window. A burn rate of 10 means it is being consumed 10x faster. + * Requires `halfLife` and `threshold` to be configured. + * @param {number} sloTarget - The SLO target as a fraction (e.g. 0.999 + * for 99.9%). + * @returns {number} + */ + burnRate(sloTarget) { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + validateNumber(sloTarget, 'sloTarget'); + if (NumberIsNaN(sloTarget) || sloTarget <= 0 || sloTarget >= 1) + throw new ERR_OUT_OF_RANGE('sloTarget', '> 0 && < 1', sloTarget); + const errorRate = this[kHandle]?.ewmaErrorRate(); + if (errorRate === undefined) return undefined; + const errorBudget = 1 - sloTarget; + return errorRate / errorBudget; + } + /** * Returns the Kolmogorov-Smirnov test statistic comparing this * histogram's distribution to another's. Returns a value between @@ -228,6 +305,95 @@ class Histogram { return this[kHandle]?.ksTest(other[kHandle]); } + /** + * Performs Welch's t-test comparing this histogram to another. + * Returns an object with the t-statistic, degrees of freedom, + * two-tailed p-value, and confidence interval on the difference + * of means. + * @param {Histogram} other + * @param {{ confidence?: number }} [options] + * @returns {{ tStatistic: number, degreesOfFreedom: number, + * pValue: number, + * confidenceInterval: { lower: number, upper: number } }} + */ + welchTest(other, options = kEmptyObject) { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + if (!isHistogram(other)) + throw new ERR_INVALID_ARG_TYPE('other', 'Histogram', other); + validateObject(options, 'options'); + const { confidence = 0.95 } = options; + validateNumber(confidence, 'options.confidence'); + if (NumberIsNaN(confidence) || confidence <= 0 || confidence >= 1) + throw new ERR_OUT_OF_RANGE('options.confidence', + '> 0 && < 1', confidence); + const result = this[kHandle]?.welchTest(other[kHandle], confidence); + if (result === undefined) return undefined; + return { + __proto__: null, + tStatistic: result[0], + degreesOfFreedom: result[1], + pValue: result[2], + confidenceInterval: { + __proto__: null, + lower: result[3], + upper: result[4], + }, + }; + } + + /** + * Performs a Mann-Whitney U test comparing this histogram to + * another. Returns an object with the U statistic, z-score, + * and two-tailed p-value (normal approximation). + * @param {Histogram} other + * @returns {{ uStatistic: number, zScore: number, pValue: number }} + */ + mannWhitneyTest(other) { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + if (!isHistogram(other)) + throw new ERR_INVALID_ARG_TYPE('other', 'Histogram', other); + const result = this[kHandle]?.mannWhitneyTest(other[kHandle]); + if (result === undefined) return undefined; + return { + __proto__: null, + uStatistic: result[0], + zScore: result[1], + pValue: result[2], + }; + } + + /** + * Computes Cohen's d effect size comparing this histogram to + * another. Uses the pooled standard deviation. Positive values + * indicate this histogram has a higher mean. + * @param {Histogram} other + * @returns {number} + */ + cohensD(other) { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + if (!isHistogram(other)) + throw new ERR_INVALID_ARG_TYPE('other', 'Histogram', other); + return this[kHandle]?.cohensD(other[kHandle]); + } + + /** + * Computes Cliff's delta comparing this histogram to another. + * Returns a value between -1 and 1. Positive values indicate + * this histogram tends to produce larger values. + * @param {Histogram} other + * @returns {number} + */ + cliffsD(other) { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + if (!isHistogram(other)) + throw new ERR_INVALID_ARG_TYPE('other', 'Histogram', other); + return this[kHandle]?.cliffsD(other[kHandle]); + } + /** * Returns the excess kurtosis of the recorded values, a measure of * the heaviness of the distribution's tails. A positive value indicates @@ -326,6 +492,36 @@ class Histogram { return this[kHandle]?.percentileBigInt(percentile); } + /** + * Returns a confidence interval for the given percentile using the + * exact binomial method. The result contains the point estimate and + * the lower/upper bounds of the interval. + * @param {number} percentile + * @param {{ confidence?: number }} [options] + * @returns {{ value: number, lower: number, upper: number }} + */ + percentileCI(percentile, options = kEmptyObject) { + if (!isHistogram(this)) + throw new ERR_INVALID_THIS('Histogram'); + validateNumber(percentile, 'percentile'); + if (NumberIsNaN(percentile) || percentile <= 0 || percentile > 100) + throw new ERR_OUT_OF_RANGE('percentile', '> 0 && <= 100', percentile); + validateObject(options, 'options'); + const { confidence = 0.95 } = options; + validateNumber(confidence, 'options.confidence'); + if (NumberIsNaN(confidence) || confidence <= 0 || confidence >= 1) + throw new ERR_OUT_OF_RANGE('options.confidence', + '> 0 && < 1', confidence); + const result = this[kHandle]?.percentileCI(percentile, confidence); + if (result === undefined) return undefined; + return { + __proto__: null, + value: result[0], + lower: result[1], + upper: result[2], + }; + } + /** * @readonly * @type {Map} @@ -397,7 +593,7 @@ class Histogram { } toJSON() { - return { + const json = { count: this.count, min: this.min, max: this.max, @@ -406,8 +602,12 @@ class Histogram { stddev: this.stddev, skewness: this.skewness, kurtosis: this.kurtosis, + ewmaMean: this.ewmaMean, + ewmaStddev: this.ewmaStddev, + ewmaErrorRate: this.ewmaErrorRate, percentiles: ObjectFromEntries(MapPrototypeEntries(this.percentiles)), }; + return json; } } @@ -538,7 +738,9 @@ function createRecordableHistogram(handle) { * @param {{ * lowest? : number, * highest? : number, - * figures? : number + * figures? : number, + * halfLife? : number, + * threshold? : number * }} [options] * @returns {RecordableHistogram} */ @@ -548,6 +750,8 @@ function createHistogram(options = kEmptyObject) { lowest = 1, highest = NumberMAX_SAFE_INTEGER, figures = 3, + halfLife = 0, + threshold = 0, } = options; if (typeof lowest !== 'bigint') validateInteger(lowest, 'options.lowest', 1, NumberMAX_SAFE_INTEGER); @@ -558,7 +762,26 @@ function createHistogram(options = kEmptyObject) { throw new ERR_INVALID_ARG_VALUE.RangeError('options.highest', highest); } validateInteger(figures, 'options.figures', 1, 5); - return createRecordableHistogram(new _Histogram(lowest, highest, figures)); + validateNumber(halfLife, 'options.halfLife'); + if (halfLife < 0) + throw new ERR_OUT_OF_RANGE('options.halfLife', '>= 0', halfLife); + validateNumber(threshold, 'options.threshold'); + if (threshold < 0) + throw new ERR_OUT_OF_RANGE('options.threshold', '>= 0', threshold); + return createRecordableHistogram( + new _Histogram(lowest, highest, figures, halfLife, threshold)); +} + +/** + * Reconstructs a histogram from a CBOR-encoded Uint8Array previously + * produced by `histogram.export()`. + * @param {Uint8Array} data + * @returns {RecordableHistogram} + */ +function importHistogram(data) { + if (!isUint8Array(data)) + throw new ERR_INVALID_ARG_TYPE('data', 'Uint8Array', data); + return createRecordableHistogram(_Histogram.import(data)); } module.exports = { @@ -571,4 +794,5 @@ module.exports = { kHandle, kSkipThrow, createHistogram, + importHistogram, }; diff --git a/lib/perf_hooks.js b/lib/perf_hooks.js index 18e979630e0..cc158e5c762 100644 --- a/lib/perf_hooks.js +++ b/lib/perf_hooks.js @@ -25,6 +25,7 @@ const { const { createHistogram, + importHistogram, } = require('internal/histogram'); const monitorEventLoopDelay = require('internal/perf/event_loop_delay'); @@ -43,6 +44,7 @@ module.exports = { eventLoopUtilization, timerify, createHistogram, + importHistogram, performance, }; diff --git a/src/histogram-inl.h b/src/histogram-inl.h index 7c3545f53aa..eea0e89bef1 100644 --- a/src/histogram-inl.h +++ b/src/histogram-inl.h @@ -9,11 +9,39 @@ namespace node { +void Histogram::UpdateEwma(double value) { + // Called inside a write lock. No-op when EWMA is disabled. + if (ewma_alpha_ <= 0) return; + if (!ewma_initialized_) { + ewma_mean_ = value; + ewma_variance_ = 0; + ewma_initialized_ = true; + if (threshold_ > 0) { + ewma_error_rate_ = (value > static_cast(threshold_)) ? 1.0 : 0.0; + } + return; + } + double diff = value - ewma_mean_; + ewma_mean_ += ewma_alpha_ * diff; + ewma_variance_ = + (1.0 - ewma_alpha_) * (ewma_variance_ + ewma_alpha_ * diff * diff); + + // Binary EWMA for SLO error rate: feed 1 if over threshold, 0 otherwise. + if (threshold_ > 0) { + double exceeded = (value > static_cast(threshold_)) ? 1.0 : 0.0; + ewma_error_rate_ += ewma_alpha_ * (exceeded - ewma_error_rate_); + } +} + void Histogram::Reset() { RwLock::ScopedWriteLock lock(mutex_); hdr_reset(histogram_.get()); exceeds_ = 0; prev_ = 0; + ewma_mean_ = 0; + ewma_variance_ = 0; + ewma_error_rate_ = 0; + ewma_initialized_ = false; } double Histogram::Add(const Histogram& other) { @@ -74,6 +102,21 @@ double Histogram::Stddev() const { return hdr_stddev(histogram_.get()); } +double Histogram::EwmaMean() const { + RwLock::ScopedReadLock lock(mutex_); + return ewma_initialized_ ? ewma_mean_ : 0; +} + +double Histogram::EwmaStddev() const { + RwLock::ScopedReadLock lock(mutex_); + return ewma_initialized_ ? std::sqrt(ewma_variance_) : 0; +} + +double Histogram::EwmaErrorRate() const { + RwLock::ScopedReadLock lock(mutex_); + return ewma_initialized_ ? ewma_error_rate_ : 0; +} + int64_t Histogram::Percentile(double percentile) const { RwLock::ScopedReadLock lock(mutex_); CHECK_GT(percentile, 0); @@ -101,14 +144,20 @@ bool Histogram::RecordCorrected(int64_t value, int64_t expected_interval) { RwLock::ScopedWriteLock lock(mutex_); bool recorded = hdr_record_corrected_value(histogram_.get(), value, expected_interval); - if (!recorded) exceeds_++; + if (!recorded) + exceeds_++; + else + UpdateEwma(static_cast(value)); return recorded; } bool Histogram::Record(int64_t value) { RwLock::ScopedWriteLock lock(mutex_); bool recorded = hdr_record_value(histogram_.get(), value); - if (!recorded) exceeds_++; + if (!recorded) + exceeds_++; + else + UpdateEwma(static_cast(value)); return recorded; } @@ -119,7 +168,10 @@ uint64_t Histogram::RecordDelta() { if (prev_ > 0) { CHECK_GE(time, prev_); delta = time - prev_; - if (!hdr_record_value(histogram_.get(), delta)) exceeds_++; + if (!hdr_record_value(histogram_.get(), delta)) + exceeds_++; + else + UpdateEwma(static_cast(delta)); } prev_ = time; return delta; diff --git a/src/histogram.cc b/src/histogram.cc index 3aa451685e8..1a852dcf3f8 100644 --- a/src/histogram.cc +++ b/src/histogram.cc @@ -6,14 +6,19 @@ #include "node_errors.h" #include "node_external_reference.h" #include "util.h" +#include "v8-typed-array.h" +#include +#include #include namespace node { +using v8::Array; using v8::BigInt; using v8::CFunction; using v8::Context; +using v8::Float64Array; using v8::FunctionCallbackInfo; using v8::FunctionTemplate; using v8::Integer; @@ -41,13 +46,22 @@ void StopHandleHistogram(Local receiver) { histogram->OnStop(); } -Histogram::Histogram(const Options& options) { +Histogram::Histogram(HistogramPointer histogram, const Options& options) + : histogram_(std::move(histogram)) { + // alpha = 1 - 2^(-1/halfLife). With halfLife <= 0, EWMA is disabled. + if (options.half_life > 0) { + ewma_alpha_ = 1.0 - std::exp(-std::log(2.0) / options.half_life); + } + threshold_ = options.threshold; +} + +std::shared_ptr Histogram::Create(const Options& options) { hdr_histogram* histogram; - CHECK_EQ(0, hdr_init(options.lowest, - options.highest, - options.figures, - &histogram)); - histogram_.reset(histogram); + if (hdr_init(options.lowest, options.highest, options.figures, &histogram) != + 0) { + return {}; + } + return std::make_shared(HistogramPointer(histogram), options); } void Histogram::MemoryInfo(MemoryTracker* tracker) const { @@ -217,8 +231,825 @@ void Histogram::PercentilesAt(const double* percentiles, hdr_value_at_percentiles(histogram_.get(), percentiles, values, length); } +namespace { +// Continued fraction evaluation for the regularized incomplete beta +// function using Lentz's modified method. Reference: Numerical Recipes +// in C, 2nd edition, section 6.4. +static double BetaContinuedFraction(double a, double b, double x) { + constexpr double FPMIN = 1e-30; + constexpr int MAXIT = 200; + constexpr double EPS = 3e-12; + + double qab = a + b; + double qap = a + 1.0; + double qam = a - 1.0; + double c = 1.0; + double d = 1.0 - qab * x / qap; + if (std::fabs(d) < FPMIN) d = FPMIN; + d = 1.0 / d; + double h = d; + + for (int m = 1; m <= MAXIT; m++) { + int m2 = 2 * m; + // Even step. + double aa = m * (b - m) * x / ((qam + m2) * (a + m2)); + d = 1.0 + aa * d; + if (std::fabs(d) < FPMIN) d = FPMIN; + c = 1.0 + aa / c; + if (std::fabs(c) < FPMIN) c = FPMIN; + d = 1.0 / d; + h *= d * c; + // Odd step. + aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)); + d = 1.0 + aa * d; + if (std::fabs(d) < FPMIN) d = FPMIN; + c = 1.0 + aa / c; + if (std::fabs(c) < FPMIN) c = FPMIN; + d = 1.0 / d; + double del = d * c; + h *= del; + if (std::fabs(del - 1.0) <= EPS) break; + } + return h; +} + +// Regularized incomplete beta function I_x(a, b). +// Returns the probability that a Beta(a,b) random variable is <= x. +static double RegularizedIncompleteBeta(double a, double b, double x) { + if (x <= 0.0) return 0.0; + if (x >= 1.0) return 1.0; + + double ln_front = std::lgamma(a + b) - std::lgamma(a) - std::lgamma(b) + + a * std::log(x) + b * std::log(1.0 - x); + double bt = std::exp(ln_front); + + // Use the symmetry relation to ensure the continued fraction + // converges in the region where it is most accurate. + if (x < (a + 1.0) / (a + b + 2.0)) { + return bt * BetaContinuedFraction(a, b, x) / a; + } + return 1.0 - bt * BetaContinuedFraction(b, a, 1.0 - x) / b; +} + +// Standard normal CDF: Phi(x) = P(Z <= x). +static double NormalCdf(double x) { + return 0.5 * std::erfc(-x * std::numbers::sqrt2 / 2.0); +} + +// Student's t-distribution CDF: P(T <= t) for df degrees of freedom. +static double StudentTCdf(double t, double df) { + double x = df / (df + t * t); + double ibeta = RegularizedIncompleteBeta(df / 2.0, 0.5, x); + if (t >= 0.0) { + return 1.0 - 0.5 * ibeta; + } + return 0.5 * ibeta; +} + +// Student's t-distribution quantile (inverse CDF) using bisection. +// Returns the value t such that P(T <= t) = p. +static double StudentTQuantile(double p, double df) { + if (p <= 0.0) return -std::numeric_limits::infinity(); + if (p >= 1.0) return std::numeric_limits::infinity(); + if (p == 0.5) return 0.0; + + // Bisection search. The range [-1e6, 1e6] is sufficient for any + // practical confidence level and degrees of freedom. + double lo = -1e6; + double hi = 1e6; + for (int i = 0; i < 100; i++) { + double mid = (lo + hi) / 2.0; + if (StudentTCdf(mid, df) < p) { + lo = mid; + } else { + hi = mid; + } + } + return (lo + hi) / 2.0; +} + +// Binomial CDF: P(X <= k) for X ~ Binomial(n, p). +// Uses the identity P(X <= k) = I_{1-p}(n-k, k+1). +static double BinomialCdf(int64_t k, int64_t n, double p) { + if (k < 0) return 0.0; + if (k >= n) return 1.0; + return RegularizedIncompleteBeta( + static_cast(n - k), static_cast(k + 1), 1.0 - p); +} + +// ----------------------------------------------------------------------- +// Minimal CBOR encoder/decoder (RFC 8949) -- just enough types for +// histogram export/import: unsigned int, float64, array, and map. +// ----------------------------------------------------------------------- + +// CBOR major types (upper 3 bits of the initial byte). +constexpr uint8_t kCborUint = 0 << 5; // Major 0: unsigned integer +constexpr uint8_t kCborArray = 4 << 5; // Major 4: array +constexpr uint8_t kCborMap = 5 << 5; // Major 5: map +constexpr uint8_t kCborFloat64 = 0xfb; // Major 7, additional 27 + +static void CborWriteUint(std::vector& out, + uint8_t major, + uint64_t val) { + if (val <= 23) { + out.push_back(major | static_cast(val)); + } else if (val <= 0xff) { + out.push_back(major | 24); + out.push_back(static_cast(val)); + } else if (val <= 0xffff) { + out.push_back(major | 25); + out.push_back(static_cast(val >> 8)); + out.push_back(static_cast(val)); + } else if (val <= 0xffffffff) { + out.push_back(major | 26); + out.push_back(static_cast(val >> 24)); + out.push_back(static_cast(val >> 16)); + out.push_back(static_cast(val >> 8)); + out.push_back(static_cast(val)); + } else { + out.push_back(major | 27); + out.push_back(static_cast(val >> 56)); + out.push_back(static_cast(val >> 48)); + out.push_back(static_cast(val >> 40)); + out.push_back(static_cast(val >> 32)); + out.push_back(static_cast(val >> 24)); + out.push_back(static_cast(val >> 16)); + out.push_back(static_cast(val >> 8)); + out.push_back(static_cast(val)); + } +} + +static void CborWriteFloat64(std::vector& out, double val) { + out.push_back(kCborFloat64); + uint64_t bits; + memcpy(&bits, &val, sizeof(bits)); + // Network byte order (big-endian). + out.push_back(static_cast(bits >> 56)); + out.push_back(static_cast(bits >> 48)); + out.push_back(static_cast(bits >> 40)); + out.push_back(static_cast(bits >> 32)); + out.push_back(static_cast(bits >> 24)); + out.push_back(static_cast(bits >> 16)); + out.push_back(static_cast(bits >> 8)); + out.push_back(static_cast(bits)); +} + +static bool CborReadUint(const uint8_t*& p, const uint8_t* end, uint64_t* val) { + if (p >= end) return false; + uint8_t info = *p++ & 0x1f; + if (info <= 23) { + *val = info; + } else if (info == 24) { + if (p + 1 > end) return false; + *val = p[0]; + p += 1; + } else if (info == 25) { + if (p + 2 > end) return false; + *val = (static_cast(p[0]) << 8) | p[1]; + p += 2; + } else if (info == 26) { + if (p + 4 > end) return false; + *val = (static_cast(p[0]) << 24) | + (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | p[3]; + p += 4; + } else if (info == 27) { + if (p + 8 > end) return false; + *val = (static_cast(p[0]) << 56) | + (static_cast(p[1]) << 48) | + (static_cast(p[2]) << 40) | + (static_cast(p[3]) << 32) | + (static_cast(p[4]) << 24) | + (static_cast(p[5]) << 16) | + (static_cast(p[6]) << 8) | p[7]; + p += 8; + } else { + return false; // Indefinite length or reserved -- not supported. + } + return true; +} + +static bool CborReadFloat64(const uint8_t*& p, + const uint8_t* end, + double* val) { + if (p >= end || *p != kCborFloat64) return false; + p++; + if (p + 8 > end) return false; + uint64_t bits = (static_cast(p[0]) << 56) | + (static_cast(p[1]) << 48) | + (static_cast(p[2]) << 40) | + (static_cast(p[3]) << 32) | + (static_cast(p[4]) << 24) | + (static_cast(p[5]) << 16) | + (static_cast(p[6]) << 8) | p[7]; + p += 8; + memcpy(val, &bits, sizeof(*val)); + return true; +} + +// Read a value that may be either a uint or float64. +static bool CborReadNumber(const uint8_t*& p, const uint8_t* end, double* val) { + if (p >= end) return false; + if (*p == kCborFloat64) return CborReadFloat64(p, end, val); + uint64_t u; + if (!CborReadUint(p, end, &u)) return false; + *val = static_cast(u); + return true; +} + +// Histogram export format version. +constexpr uint64_t kExportVersion = 1; + +// Integer keys for the top-level CBOR map. +constexpr uint64_t kKeyVersion = 0; +constexpr uint64_t kKeyLowest = 1; +constexpr uint64_t kKeyHighest = 2; +constexpr uint64_t kKeyFigures = 3; +constexpr uint64_t kKeyTotalCount = 4; +constexpr uint64_t kKeyMin = 5; +constexpr uint64_t kKeyMax = 6; +constexpr uint64_t kKeyNormOffset = 7; +constexpr uint64_t kKeyConvRatio = 8; +constexpr uint64_t kKeyCountsLen = 9; +constexpr uint64_t kKeyCounts = 10; +constexpr uint64_t kKeyEwma = 11; + +// Integer keys for the EWMA sub-map. +constexpr uint64_t kEwmaAlpha = 0; +constexpr uint64_t kEwmaMean = 1; +constexpr uint64_t kEwmaVariance = 2; +constexpr uint64_t kEwmaErrorRate = 3; +constexpr uint64_t kEwmaThreshold = 4; +} // namespace + +Histogram::WelchTestResult Histogram::WelchTest(const Histogram& other, + double confidence) const { + auto do_welch = [&]() -> WelchTestResult { + int64_t n1 = histogram_->total_count; + int64_t n2 = other.histogram_->total_count; + if (n1 < 2 || n2 < 2) return {0, 0, 1, 0, 0}; + + double mean1 = hdr_mean(histogram_.get()); + double mean2 = hdr_mean(other.histogram_.get()); + double sd1 = hdr_stddev(histogram_.get()); + double sd2 = hdr_stddev(other.histogram_.get()); + + // HdrHistogram computes population stddev (divides by N). + // Welch's t-test requires sample variance (divides by N-1). + double var1 = + sd1 * sd1 * static_cast(n1) / static_cast(n1 - 1); + double var2 = + sd2 * sd2 * static_cast(n2) / static_cast(n2 - 1); + + double se1 = var1 / static_cast(n1); + double se2 = var2 / static_cast(n2); + double se_sum = se1 + se2; + if (se_sum == 0.0) return {0, 0, 1, 0, 0}; + + double t = (mean1 - mean2) / std::sqrt(se_sum); + + // Welch-Satterthwaite degrees of freedom. + double df = (se_sum * se_sum) / (se1 * se1 / static_cast(n1 - 1) + + se2 * se2 / static_cast(n2 - 1)); + + // Two-tailed p-value. + double p = 2.0 * StudentTCdf(-std::fabs(t), df); + + // Confidence interval on the difference of means. + double alpha = 1.0 - confidence; + double t_crit = StudentTQuantile(1.0 - alpha / 2.0, df); + double margin = t_crit * std::sqrt(se_sum); + double diff = mean1 - mean2; + + return {t, df, p, diff - margin, diff + margin}; + }; + + if (this == &other) return {0, 0, 1, 0, 0}; + + if (this < &other) { + RwLock::ScopedReadLock lock1(mutex_); + RwLock::ScopedReadLock lock2(other.mutex_); + return do_welch(); + } + + RwLock::ScopedReadLock lock1(other.mutex_); + RwLock::ScopedReadLock lock2(mutex_); + return do_welch(); +} + +Histogram::MannWhitneyResult Histogram::MannWhitneyTest( + const Histogram& other) const { + auto do_mw = [&]() -> MannWhitneyResult { + int64_t n1 = histogram_->total_count; + int64_t n2 = other.histogram_->total_count; + if (n1 == 0 || n2 == 0) return {0, 0, 1}; + + // Walk the counts arrays to compute the U statistic. + // At each bucket index, values from histogram 1 at index i "beat" + // all values from histogram 2 at indices < i (concordant pairs). + int32_t len = + std::max(histogram_->counts_len, other.histogram_->counts_len); + + // Forward pass: count concordant pairs (h1 values > h2 values). + int64_t cum2 = 0; + double concordant = 0.0; + double tied = 0.0; + for (int32_t i = 0; i < len; i++) { + int64_t c1 = (i < histogram_->counts_len) ? histogram_->counts[i] : 0; + int64_t c2 = + (i < other.histogram_->counts_len) ? other.histogram_->counts[i] : 0; + concordant += static_cast(c1) * static_cast(cum2); + tied += static_cast(c1) * static_cast(c2); + cum2 += c2; + } + + // U statistic for sample 1: concordant + half of ties. + double u = concordant + 0.5 * tied; + double dn1 = static_cast(n1); + double dn2 = static_cast(n2); + double mu = dn1 * dn2 / 2.0; + + // Tie correction for the variance. + // sigma^2 = n1*n2/12 * (N+1 - sum(t_k^3 - t_k) / (N*(N-1))) + // where t_k is the number of observations tied at rank k. + double n_total = dn1 + dn2; + double tie_correction = 0.0; + for (int32_t i = 0; i < len; i++) { + int64_t c1 = (i < histogram_->counts_len) ? histogram_->counts[i] : 0; + int64_t c2 = + (i < other.histogram_->counts_len) ? other.histogram_->counts[i] : 0; + double tk = static_cast(c1 + c2); + if (tk > 1) { + tie_correction += tk * tk * tk - tk; + } + } + + double sigma_sq = + (dn1 * dn2 / 12.0) * + (n_total + 1.0 - tie_correction / (n_total * (n_total - 1.0))); + if (sigma_sq <= 0.0) return {u, 0, 1}; + + // Continuity-corrected z-score. + double z = (u - mu) / std::sqrt(sigma_sq); + // Two-tailed p-value using normal approximation. + double p = 2.0 * NormalCdf(-std::fabs(z)); + + return {u, z, p}; + }; + + if (this == &other) return {0, 0, 1}; + + if (this < &other) { + RwLock::ScopedReadLock lock1(mutex_); + RwLock::ScopedReadLock lock2(other.mutex_); + return do_mw(); + } + + RwLock::ScopedReadLock lock1(other.mutex_); + RwLock::ScopedReadLock lock2(mutex_); + return do_mw(); +} + +double Histogram::CohensD(const Histogram& other) const { + auto do_cohens = [&]() -> double { + int64_t n1 = histogram_->total_count; + int64_t n2 = other.histogram_->total_count; + if (n1 < 2 || n2 < 2) return 0.0; + + double mean1 = hdr_mean(histogram_.get()); + double mean2 = hdr_mean(other.histogram_.get()); + double sd1 = hdr_stddev(histogram_.get()); + double sd2 = hdr_stddev(other.histogram_.get()); + + // Convert population variance to sample variance (Bessel's correction). + double var1 = + sd1 * sd1 * static_cast(n1) / static_cast(n1 - 1); + double var2 = + sd2 * sd2 * static_cast(n2) / static_cast(n2 - 1); + + double pooled_sd = std::sqrt((static_cast(n1 - 1) * var1 + + static_cast(n2 - 1) * var2) / + static_cast(n1 + n2 - 2)); + if (pooled_sd == 0.0) return 0.0; + + return (mean1 - mean2) / pooled_sd; + }; + + if (this == &other) return 0.0; + + if (this < &other) { + RwLock::ScopedReadLock lock1(mutex_); + RwLock::ScopedReadLock lock2(other.mutex_); + return do_cohens(); + } + + RwLock::ScopedReadLock lock1(other.mutex_); + RwLock::ScopedReadLock lock2(mutex_); + return do_cohens(); +} + +double Histogram::CliffsD(const Histogram& other) const { + auto do_cliffs = [&]() -> double { + int64_t n1 = histogram_->total_count; + int64_t n2 = other.histogram_->total_count; + if (n1 == 0 || n2 == 0) return 0.0; + + int32_t len = + std::max(histogram_->counts_len, other.histogram_->counts_len); + + // Forward pass: count pairs where h1 value > h2 value (concordant). + int64_t cum2 = 0; + double concordant = 0.0; + double tied = 0.0; + for (int32_t i = 0; i < len; i++) { + int64_t c1 = (i < histogram_->counts_len) ? histogram_->counts[i] : 0; + int64_t c2 = + (i < other.histogram_->counts_len) ? other.histogram_->counts[i] : 0; + concordant += static_cast(c1) * static_cast(cum2); + tied += static_cast(c1) * static_cast(c2); + cum2 += c2; + } + + double discordant = + static_cast(n1) * static_cast(n2) - concordant - tied; + + return (concordant - discordant) / + (static_cast(n1) * static_cast(n2)); + }; + + if (this == &other) return 0.0; + + if (this < &other) { + RwLock::ScopedReadLock lock1(mutex_); + RwLock::ScopedReadLock lock2(other.mutex_); + return do_cliffs(); + } + + RwLock::ScopedReadLock lock1(other.mutex_); + RwLock::ScopedReadLock lock2(mutex_); + return do_cliffs(); +} + +Histogram::PercentileCIResult Histogram::PercentileCI(double percentile, + double confidence) const { + RwLock::ScopedReadLock lock(mutex_); + + int64_t value = hdr_value_at_percentile(histogram_.get(), percentile); + int64_t n = histogram_->total_count; + + if (n < 2) { + return {value, value, value}; + } + + double p = percentile / 100.0; + double alpha = 1.0 - confidence; + + // Lower rank: largest j such that BinomialCdf(j-1, n, p) <= alpha/2. + // Binary search over [0, n]. + int64_t lo = 0; + int64_t hi = n; + while (lo < hi) { + int64_t mid = lo + (hi - lo + 1) / 2; + if (BinomialCdf(mid - 1, n, p) <= alpha / 2.0) { + lo = mid; + } else { + hi = mid - 1; + } + } + double lower_pct = static_cast(lo) / static_cast(n) * 100.0; + + // Upper rank: smallest k such that BinomialCdf(k-1, n, p) >= 1 - alpha/2. + lo = 0; + hi = n; + while (lo < hi) { + int64_t mid = lo + (hi - lo) / 2; + if (BinomialCdf(mid - 1, n, p) >= 1.0 - alpha / 2.0) { + hi = mid; + } else { + lo = mid + 1; + } + } + double upper_pct = static_cast(lo) / static_cast(n) * 100.0; + + int64_t lower_val = hdr_value_at_percentile(histogram_.get(), lower_pct); + int64_t upper_val = hdr_value_at_percentile(histogram_.get(), upper_pct); + + return {value, lower_val, upper_val}; +} + +// Serialize the histogram to a CBOR (RFC 8949) byte sequence. There is no +// standard for serializing histograms, so this format is specific to this +// implementation. But, it has been designed to be compact, extensible, and +// portable across languages and platforms. Should be easily parseable in +// other languages with or without a CBOR library. +// +// The delta-encoded sparse counts is a space-saving optimization in the +// common case. +// +// Layout: a CBOR map with integer keys: +// 0 -> uint format version (currently 1) +// 1 -> uint lowest discernible value +// 2 -> uint highest trackable value +// 3 -> uint significant figures +// 4 -> uint total count +// 5 -> uint min value +// 6 -> uint max value +// 7 -> uint normalizing index offset +// 8 -> float64 conversion ratio +// 9 -> uint counts array length +// 10 -> array delta-encoded sparse counts as flat [delta, count, ...] +// (first delta is the absolute index) +// 11 -> map EWMA state (omitted when alpha = 0): +// 0 -> float64 alpha +// 1 -> float64 mean +// 2 -> float64 variance +// 3 -> float64 error rate +// 4 -> uint threshold +std::vector Histogram::Export() const { + RwLock::ScopedReadLock lock(mutex_); + + // Count non-zero buckets for the sparse encoding. + int32_t non_zero = 0; + for (int32_t i = 0; i < histogram_->counts_len; i++) { + if (histogram_->counts[i] != 0) non_zero++; + } + + bool has_ewma = ewma_alpha_ > 0; + uint64_t map_size = has_ewma ? 12 : 11; + + std::vector out; + out.reserve(64 + non_zero * 10); + + // Top-level map. + CborWriteUint(out, kCborMap, map_size); + + // 0: version + CborWriteUint(out, kCborUint, kKeyVersion); + CborWriteUint(out, kCborUint, kExportVersion); + // 1: lowest + CborWriteUint(out, kCborUint, kKeyLowest); + CborWriteUint(out, + kCborUint, + static_cast(histogram_->lowest_discernible_value)); + // 2: highest + CborWriteUint(out, kCborUint, kKeyHighest); + CborWriteUint(out, + kCborUint, + static_cast(histogram_->highest_trackable_value)); + // 3: figures + CborWriteUint(out, kCborUint, kKeyFigures); + CborWriteUint( + out, kCborUint, static_cast(histogram_->significant_figures)); + // 4: total_count + CborWriteUint(out, kCborUint, kKeyTotalCount); + CborWriteUint(out, kCborUint, static_cast(histogram_->total_count)); + // 5: min + CborWriteUint(out, kCborUint, kKeyMin); + CborWriteUint(out, kCborUint, static_cast(histogram_->min_value)); + // 6: max + CborWriteUint(out, kCborUint, kKeyMax); + CborWriteUint(out, kCborUint, static_cast(histogram_->max_value)); + // 7: normalizing_index_offset + CborWriteUint(out, kCborUint, kKeyNormOffset); + CborWriteUint(out, + kCborUint, + static_cast(histogram_->normalizing_index_offset)); + // 8: conversion_ratio + CborWriteUint(out, kCborUint, kKeyConvRatio); + CborWriteFloat64(out, histogram_->conversion_ratio); + // 9: counts_len + CborWriteUint(out, kCborUint, kKeyCountsLen); + CborWriteUint(out, kCborUint, static_cast(histogram_->counts_len)); + // 10: sparse counts -- array of [delta, count, ...] pairs. + // Indices are delta-encoded: the first value is the absolute index, + // each subsequent value is the difference from the previous index. + CborWriteUint(out, kCborUint, kKeyCounts); + CborWriteUint(out, kCborArray, static_cast(non_zero) * 2); + int32_t prev_idx = 0; + for (int32_t i = 0; i < histogram_->counts_len; i++) { + if (histogram_->counts[i] != 0) { + CborWriteUint(out, kCborUint, static_cast(i - prev_idx)); + CborWriteUint( + out, kCborUint, static_cast(histogram_->counts[i])); + prev_idx = i; + } + } + + // 11: EWMA state (optional) + if (has_ewma) { + CborWriteUint(out, kCborUint, kKeyEwma); + CborWriteUint(out, kCborMap, 5); + CborWriteUint(out, kCborUint, kEwmaAlpha); + CborWriteFloat64(out, ewma_alpha_); + CborWriteUint(out, kCborUint, kEwmaMean); + CborWriteFloat64(out, ewma_mean_); + CborWriteUint(out, kCborUint, kEwmaVariance); + CborWriteFloat64(out, ewma_variance_); + CborWriteUint(out, kCborUint, kEwmaErrorRate); + CborWriteFloat64(out, ewma_error_rate_); + CborWriteUint(out, kCborUint, kEwmaThreshold); + CborWriteUint(out, kCborUint, static_cast(threshold_)); + } + + return out; +} + +std::shared_ptr Histogram::Import(const uint8_t* data, size_t len) { + const uint8_t* p = data; + const uint8_t* end = data + len; + + // Read top-level map header. + if (p >= end || (*p >> 5) != 5) return nullptr; // Must be a map. + uint64_t map_size; + if (!CborReadUint(p, end, &map_size)) return nullptr; + + int64_t lowest = 1; + int64_t highest = std::numeric_limits::max(); + int figures = 3; + int64_t total_count = 0; + int64_t min_value = std::numeric_limits::max(); + int64_t max_value = 0; + int32_t norm_offset = 0; + double conv_ratio = 1.0; + int32_t counts_len = 0; + uint64_t version = 0; + + // Sparse counts storage. + std::vector> sparse_counts; + + // EWMA state. + double ewma_alpha = 0; + double ewma_mean = 0; + double ewma_variance = 0; + double ewma_error_rate = 0; + int64_t threshold = 0; + + for (uint64_t i = 0; i < map_size; i++) { + // Read key (unsigned int). + uint64_t key; + if (!CborReadUint(p, end, &key)) return nullptr; + + switch (key) { + case kKeyVersion: + if (!CborReadUint(p, end, &version)) return nullptr; + if (version != kExportVersion) return nullptr; + break; + case kKeyLowest: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + lowest = static_cast(v); + break; + } + case kKeyHighest: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + highest = static_cast(v); + break; + } + case kKeyFigures: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + figures = static_cast(v); + break; + } + case kKeyTotalCount: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + total_count = static_cast(v); + break; + } + case kKeyMin: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + min_value = static_cast(v); + break; + } + case kKeyMax: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + max_value = static_cast(v); + break; + } + case kKeyNormOffset: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + norm_offset = static_cast(v); + break; + } + case kKeyConvRatio: + if (!CborReadNumber(p, end, &conv_ratio)) return nullptr; + break; + case kKeyCountsLen: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + counts_len = static_cast(v); + break; + } + case kKeyCounts: { + // Array of flat [delta, count, ...] pairs. Indices are + // delta-encoded: accumulate to recover absolute indices. + if (p >= end || (*p >> 5) != 4) return nullptr; + uint64_t arr_len; + if (!CborReadUint(p, end, &arr_len)) return nullptr; + if (arr_len % 2 != 0) return nullptr; + // Each element needs at least 1 byte of CBOR encoding, so + // arr_len can't exceed the remaining buffer. Without this + // check, a crafted buffer claiming arr_len=2^60 would cause + // reserve() to OOM-crash before the loop catches the error. + if (arr_len > static_cast(end - p)) return nullptr; + sparse_counts.reserve(static_cast(arr_len / 2)); + int32_t acc_idx = 0; + for (uint64_t j = 0; j < arr_len; j += 2) { + uint64_t delta, cnt; + if (!CborReadUint(p, end, &delta)) return nullptr; + if (!CborReadUint(p, end, &cnt)) return nullptr; + acc_idx += static_cast(delta); + sparse_counts.emplace_back(acc_idx, static_cast(cnt)); + } + break; + } + case kKeyEwma: { + // Sub-map for EWMA state. + if (p >= end || (*p >> 5) != 5) return nullptr; + uint64_t sub_size; + if (!CborReadUint(p, end, &sub_size)) return nullptr; + for (uint64_t j = 0; j < sub_size; j++) { + uint64_t sub_key; + if (!CborReadUint(p, end, &sub_key)) return nullptr; + switch (sub_key) { + case kEwmaAlpha: + if (!CborReadNumber(p, end, &ewma_alpha)) return nullptr; + break; + case kEwmaMean: + if (!CborReadNumber(p, end, &ewma_mean)) return nullptr; + break; + case kEwmaVariance: + if (!CborReadNumber(p, end, &ewma_variance)) return nullptr; + break; + case kEwmaErrorRate: + if (!CborReadNumber(p, end, &ewma_error_rate)) return nullptr; + break; + case kEwmaThreshold: { + uint64_t v; + if (!CborReadUint(p, end, &v)) return nullptr; + threshold = static_cast(v); + break; + } + default: + return nullptr; // Unknown EWMA key. + } + } + break; + } + default: + return nullptr; // Unknown key. + } + } + + // Reconstruct the histogram. + Options opts; + opts.lowest = lowest; + opts.highest = highest; + opts.figures = figures; + // Compute half_life from alpha: alpha = 1 - 2^(-1/halfLife) + // => halfLife = -1 / log2(1 - alpha) + if (ewma_alpha > 0 && ewma_alpha < 1) { + opts.half_life = -1.0 / std::log2(1.0 - ewma_alpha); + } + opts.threshold = threshold; + + auto histogram = Histogram::Create(opts); + if (!histogram) return nullptr; + + // Validate counts_len matches what the options produce. + if (histogram->histogram_->counts_len != counts_len) return nullptr; + + // Restore counts directly. + for (const auto& [idx, cnt] : sparse_counts) { + if (idx < 0 || idx >= counts_len) return nullptr; + histogram->histogram_->counts[idx] = cnt; + } + histogram->histogram_->total_count = total_count; + histogram->histogram_->min_value = min_value; + histogram->histogram_->max_value = max_value; + histogram->histogram_->normalizing_index_offset = norm_offset; + histogram->histogram_->conversion_ratio = conv_ratio; + + // Restore EWMA state. + if (ewma_alpha > 0) { + histogram->ewma_mean_ = ewma_mean; + histogram->ewma_variance_ = ewma_variance; + histogram->ewma_error_rate_ = ewma_error_rate; + histogram->ewma_initialized_ = true; + } + + return histogram; +} + HistogramImpl::HistogramImpl(const Histogram::Options& options) - : histogram_(new Histogram(options)) {} + : histogram_(Histogram::Create(options)) { + CHECK(histogram_); +} HistogramImpl::HistogramImpl(std::shared_ptr histogram) : histogram_(std::move(histogram)) {} @@ -247,6 +1078,12 @@ CFunction HistogramImpl::fast_get_cdf_( CFunction::Make(&HistogramImpl::FastGetCdf)); CFunction HistogramImpl::fast_get_count_at_( CFunction::Make(&HistogramImpl::FastGetCountAt)); +CFunction HistogramImpl::fast_get_ewma_mean_( + CFunction::Make(&HistogramImpl::FastGetEwmaMean)); +CFunction HistogramImpl::fast_get_ewma_stddev_( + CFunction::Make(&HistogramImpl::FastGetEwmaStddev)); +CFunction HistogramImpl::fast_get_ewma_error_rate_( + CFunction::Make(&HistogramImpl::FastGetEwmaErrorRate)); CFunction HistogramBase::fast_record_( CFunction::Make(&HistogramBase::FastRecord)); CFunction HistogramBase::fast_record_delta_( @@ -296,6 +1133,22 @@ void HistogramImpl::AddMethods(Isolate* isolate, Local tmpl) { SetProtoMethodNoSideEffect(isolate, tmpl, "percentilesAt", GetPercentilesAt); SetProtoMethodNoSideEffect(isolate, tmpl, "linearBuckets", GetLinearBuckets); SetProtoMethodNoSideEffect(isolate, tmpl, "logBuckets", GetLogBuckets); + SetProtoMethodNoSideEffect(isolate, tmpl, "welchTest", GetWelchTest); + SetProtoMethodNoSideEffect( + isolate, tmpl, "mannWhitneyTest", GetMannWhitneyTest); + SetProtoMethodNoSideEffect(isolate, tmpl, "cohensD", GetCohensD); + SetProtoMethodNoSideEffect(isolate, tmpl, "cliffsD", GetCliffsD); + SetProtoMethodNoSideEffect(isolate, tmpl, "percentileCI", GetPercentileCI); + SetFastMethodNoSideEffect( + isolate, instance, "ewmaMean", GetEwmaMean, &fast_get_ewma_mean_); + SetFastMethodNoSideEffect( + isolate, instance, "ewmaStddev", GetEwmaStddev, &fast_get_ewma_stddev_); + SetFastMethodNoSideEffect(isolate, + instance, + "ewmaErrorRate", + GetEwmaErrorRate, + &fast_get_ewma_error_rate_); + SetProtoMethodNoSideEffect(isolate, tmpl, "export", DoExport); SetFastMethod(isolate, instance, "reset", DoReset, &fast_reset_); } @@ -334,6 +1187,18 @@ void HistogramImpl::RegisterExternalReferences( registry->Register(GetPercentilesAt); registry->Register(GetLinearBuckets); registry->Register(GetLogBuckets); + registry->Register(GetWelchTest); + registry->Register(GetMannWhitneyTest); + registry->Register(GetCohensD); + registry->Register(GetCliffsD); + registry->Register(GetPercentileCI); + registry->Register(GetEwmaMean); + registry->Register(GetEwmaStddev); + registry->Register(GetEwmaErrorRate); + registry->Register(DoExport); + registry->Register(fast_get_ewma_mean_); + registry->Register(fast_get_ewma_stddev_); + registry->Register(fast_get_ewma_error_rate_); registry->Register(fast_get_skewness_); registry->Register(fast_get_kurtosis_); registry->Register(fast_get_cdf_); @@ -509,9 +1374,20 @@ void HistogramBase::New(const FunctionCallbackInfo& args) { } int32_t figures = args[2].As()->Value(); - new HistogramBase(env, args.This(), Histogram::Options { - lowest, highest, figures - }); + double half_life = 0; + if (args.Length() > 3 && args[3]->IsNumber()) { + half_life = args[3].As()->Value(); + } + int64_t threshold = 0; + if (args.Length() > 4 && args[4]->IsNumber()) { + threshold = static_cast(args[4].As()->Value()); + } + auto histogram = Histogram::Create( + Histogram::Options{lowest, highest, figures, half_life, threshold}); + if (!histogram) { + return THROW_ERR_INVALID_ARG_VALUE(env, "Invalid histogram options"); + } + new HistogramBase(env, args.This(), std::move(histogram)); } Local HistogramBase::GetConstructorTemplate( @@ -546,16 +1422,17 @@ void HistogramBase::RegisterExternalReferences( registry->Register(RecordCorrected); registry->Register(fast_record_); registry->Register(fast_record_delta_); + registry->Register(HistogramImpl::DoImport); HistogramImpl::RegisterExternalReferences(registry); } void HistogramBase::Initialize(IsolateData* isolate_data, Local target) { - SetConstructorFunction(isolate_data->isolate(), - target, - "Histogram", - GetConstructorTemplate(isolate_data), - SetConstructorFunctionFlag::NONE); + Isolate* isolate = isolate_data->isolate(); + Local tmpl = GetConstructorTemplate(isolate_data); + SetMethodNoSideEffect(isolate, tmpl, "import", HistogramImpl::DoImport); + SetConstructorFunction( + isolate, target, "Histogram", tmpl, SetConstructorFunctionFlag::NONE); } BaseObjectPtr HistogramBase::HistogramTransferData::Deserialize( @@ -1018,13 +1895,149 @@ void HistogramImpl::GetKsTest(const FunctionCallbackInfo& args) { args.GetReturnValue().Set((*histogram)->KsTest(*(other->histogram()))); } +void HistogramImpl::GetWelchTest(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + HistogramImpl* other = HistogramImpl::FromJSObject(args[0]); + CHECK(args[1]->IsNumber()); + double confidence = args[1].As()->Value(); + + auto result = (*histogram)->WelchTest(*(other->histogram()), confidence); + + Local values[] = {Number::New(isolate, result.t_statistic), + Number::New(isolate, result.degrees_of_freedom), + Number::New(isolate, result.p_value), + Number::New(isolate, result.ci_lower), + Number::New(isolate, result.ci_upper)}; + Local arr = Array::New(isolate, &values[0], arraysize(values)); + args.GetReturnValue().Set(arr); +} + +void HistogramImpl::GetMannWhitneyTest( + const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + HistogramImpl* other = HistogramImpl::FromJSObject(args[0]); + + auto result = (*histogram)->MannWhitneyTest(*(other->histogram())); + + Local values[] = {Number::New(isolate, result.u_statistic), + Number::New(isolate, result.z_score), + Number::New(isolate, result.p_value)}; + Local arr = Array::New(isolate, &values[0], arraysize(values)); + args.GetReturnValue().Set(arr); +} + +void HistogramImpl::GetCohensD(const FunctionCallbackInfo& args) { + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + HistogramImpl* other = HistogramImpl::FromJSObject(args[0]); + args.GetReturnValue().Set((*histogram)->CohensD(*(other->histogram()))); +} + +void HistogramImpl::GetCliffsD(const FunctionCallbackInfo& args) { + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + HistogramImpl* other = HistogramImpl::FromJSObject(args[0]); + args.GetReturnValue().Set((*histogram)->CliffsD(*(other->histogram()))); +} + +void HistogramImpl::GetPercentileCI(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + CHECK(args[0]->IsNumber()); + CHECK(args[1]->IsNumber()); + double percentile = args[0].As()->Value(); + double confidence = args[1].As()->Value(); + + auto result = (*histogram)->PercentileCI(percentile, confidence); + + Local values[] = { + Number::New(isolate, static_cast(result.value)), + Number::New(isolate, static_cast(result.lower)), + Number::New(isolate, static_cast(result.upper))}; + Local arr = Array::New(isolate, &values[0], arraysize(values)); + args.GetReturnValue().Set(arr); +} + +void HistogramImpl::GetEwmaMean(const FunctionCallbackInfo& args) { + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + args.GetReturnValue().Set((*histogram)->EwmaMean()); +} + +double HistogramImpl::FastGetEwmaMean(Local receiver) { + TRACK_V8_FAST_API_CALL("histogram.ewmaMean"); + HistogramImpl* histogram = HistogramImpl::FromJSObject(receiver); + return (*histogram)->EwmaMean(); +} + +void HistogramImpl::GetEwmaStddev(const FunctionCallbackInfo& args) { + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + args.GetReturnValue().Set((*histogram)->EwmaStddev()); +} + +double HistogramImpl::FastGetEwmaStddev(Local receiver) { + TRACK_V8_FAST_API_CALL("histogram.ewmaStddev"); + HistogramImpl* histogram = HistogramImpl::FromJSObject(receiver); + return (*histogram)->EwmaStddev(); +} + +void HistogramImpl::GetEwmaErrorRate(const FunctionCallbackInfo& args) { + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + args.GetReturnValue().Set((*histogram)->EwmaErrorRate()); +} + +double HistogramImpl::FastGetEwmaErrorRate(Local receiver) { + TRACK_V8_FAST_API_CALL("histogram.ewmaErrorRate"); + HistogramImpl* histogram = HistogramImpl::FromJSObject(receiver); + return (*histogram)->EwmaErrorRate(); +} + +void HistogramImpl::DoExport(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); + std::vector data = (*histogram)->Export(); + + auto store = v8::ArrayBuffer::NewBackingStore(env->isolate(), data.size()); + memcpy(store->Data(), data.data(), data.size()); + auto buf = v8::ArrayBuffer::New(env->isolate(), std::move(store)); + auto arr = v8::Uint8Array::New(buf, 0, data.size()); + args.GetReturnValue().Set(arr); +} + +void HistogramImpl::DoImport(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + if (!args[0]->IsUint8Array()) { + THROW_ERR_INVALID_ARG_TYPE(env, "data must be a Uint8Array"); + return; + } + Local input = args[0].As(); + auto backing = input->Buffer()->GetBackingStore(); + const uint8_t* data = + static_cast(backing->Data()) + input->ByteOffset(); + size_t len = input->ByteLength(); + + auto histogram = Histogram::Import(data, len); + if (!histogram) { + THROW_ERR_INVALID_ARG_VALUE(env, "Invalid histogram export data"); + return; + } + + // Create a new HistogramBase wrapping the imported histogram. + Local tmpl = + HistogramBase::GetConstructorTemplate(env->isolate_data()); + Local obj; + if (!tmpl->InstanceTemplate()->NewInstance(env->context()).ToLocal(&obj)) + return; + new HistogramBase(env, obj, std::move(histogram)); + args.GetReturnValue().Set(obj); +} + void HistogramImpl::GetPercentilesAt(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); HistogramImpl* histogram = HistogramImpl::FromJSObject(args.This()); CHECK(args[0]->IsMap()); Local map = args[0].As(); CHECK(args[1]->IsFloat64Array()); - Local input = args[1].As(); + Local input = args[1].As(); size_t length = input->Length(); auto backing = input->Buffer()->GetBackingStore(); double* percentiles = reinterpret_cast( diff --git a/src/histogram.h b/src/histogram.h index 5fbffa2a487..05d6003c25c 100644 --- a/src/histogram.h +++ b/src/histogram.h @@ -12,6 +12,7 @@ #include "v8.h" #include +#include namespace node { @@ -30,9 +31,18 @@ class Histogram : public MemoryRetainer { int64_t lowest = 1; int64_t highest = std::numeric_limits::max(); int figures = kDefaultHistogramFigures; + double half_life = 0; // EWMA half-life in number of samples (0 = off) + int64_t threshold = 0; // SLO threshold (0 = off). When set with + // half_life, tracks EWMA error rate for values + // exceeding this threshold. }; - explicit Histogram(const Options& options); + using HistogramPointer = DeleteFnPtr; + + // Factory method that returns nullptr on hdr_init failure. + static std::shared_ptr Create(const Options& options); + + Histogram(HistogramPointer histogram, const Options& options); virtual ~Histogram() = default; inline bool Record(int64_t value); @@ -41,6 +51,9 @@ class Histogram : public MemoryRetainer { inline int64_t Max() const; inline double Mean() const; inline double Stddev() const; + inline double EwmaMean() const; + inline double EwmaStddev() const; + inline double EwmaErrorRate() const; inline int64_t Percentile(double percentile) const; inline size_t Exceeds() const; inline size_t Count() const; @@ -67,6 +80,39 @@ class Histogram : public MemoryRetainer { int64_t* values, size_t length) const; + // Statistical hypothesis testing + struct WelchTestResult { + double t_statistic; + double degrees_of_freedom; + double p_value; + double ci_lower; + double ci_upper; + }; + + struct MannWhitneyResult { + double u_statistic; + double z_score; + double p_value; + }; + + struct PercentileCIResult { + int64_t value; + int64_t lower; + int64_t upper; + }; + + WelchTestResult WelchTest(const Histogram& other, + double confidence = 0.95) const; + MannWhitneyResult MannWhitneyTest(const Histogram& other) const; + double CohensD(const Histogram& other) const; + double CliffsD(const Histogram& other) const; + PercentileCIResult PercentileCI(double percentile, + double confidence = 0.95) const; + + // CBOR-encoded export/import for histogram exchange. + std::vector Export() const; + static std::shared_ptr Import(const uint8_t* data, size_t len); + inline bool RecordCorrected(int64_t value, int64_t expected_interval); template @@ -82,10 +128,22 @@ class Histogram : public MemoryRetainer { SET_SELF_SIZE(Histogram) private: - using HistogramPointer = DeleteFnPtr; + inline void UpdateEwma(double value); + HistogramPointer histogram_; uint64_t prev_ = 0; size_t exceeds_ = 0; + + // EWMA state (active when ewma_alpha_ > 0) + double ewma_alpha_ = 0; + double ewma_mean_ = 0; + double ewma_variance_ = 0; + bool ewma_initialized_ = false; + + // SLO error rate EWMA (active when threshold_ > 0 and ewma_alpha_ > 0) + int64_t threshold_ = 0; + double ewma_error_rate_ = 0; + RwLock mutex_; }; @@ -131,6 +189,17 @@ class HistogramImpl { static void GetPercentilesAt(const v8::FunctionCallbackInfo& args); static void GetLinearBuckets(const v8::FunctionCallbackInfo& args); static void GetLogBuckets(const v8::FunctionCallbackInfo& args); + static void GetWelchTest(const v8::FunctionCallbackInfo& args); + static void GetMannWhitneyTest( + const v8::FunctionCallbackInfo& args); + static void GetCohensD(const v8::FunctionCallbackInfo& args); + static void GetCliffsD(const v8::FunctionCallbackInfo& args); + static void GetPercentileCI(const v8::FunctionCallbackInfo& args); + static void GetEwmaMean(const v8::FunctionCallbackInfo& args); + static void GetEwmaStddev(const v8::FunctionCallbackInfo& args); + static void GetEwmaErrorRate(const v8::FunctionCallbackInfo& args); + static void DoExport(const v8::FunctionCallbackInfo& args); + static void DoImport(const v8::FunctionCallbackInfo& args); static void FastReset(v8::Local receiver); static double FastGetCount(v8::Local receiver); @@ -146,6 +215,9 @@ class HistogramImpl { static double FastGetCdf(v8::Local receiver, const int64_t value); static double FastGetCountAt(v8::Local receiver, const int64_t value); + static double FastGetEwmaMean(v8::Local receiver); + static double FastGetEwmaStddev(v8::Local receiver); + static double FastGetEwmaErrorRate(v8::Local receiver); static void AddMethods(v8::Isolate* isolate, v8::Local tmpl); @@ -169,6 +241,9 @@ class HistogramImpl { static v8::CFunction fast_get_kurtosis_; static v8::CFunction fast_get_cdf_; static v8::CFunction fast_get_count_at_; + static v8::CFunction fast_get_ewma_mean_; + static v8::CFunction fast_get_ewma_stddev_; + static v8::CFunction fast_get_ewma_error_rate_; }; class HistogramBase final : public BaseObject, public HistogramImpl { diff --git a/test/parallel/test-perf-hooks-histogram-stats.js b/test/parallel/test-perf-hooks-histogram-stats.js new file mode 100644 index 00000000000..61a3894924f --- /dev/null +++ b/test/parallel/test-perf-hooks-histogram-stats.js @@ -0,0 +1,697 @@ +// Flags: --expose-internals --no-warnings +'use strict'; + +require('../common'); +const assert = require('assert'); +const { createHistogram, importHistogram } = require('perf_hooks'); + +// --------------------------------------------------------------------------- +// welchTest(other) — Welch's t-test +// --------------------------------------------------------------------------- +{ + const h1 = createHistogram(); + const h2 = createHistogram(); + + // Both empty → p-value 1 (no evidence of difference) + const empty = h1.welchTest(h2); + assert.strictEqual(empty.pValue, 1); + assert.strictEqual(empty.tStatistic, 0); + + // Identical distributions → high p-value (not significant) + for (let i = 0; i < 100; i++) { + h1.record(50 + Math.ceil(Math.random() * 10)); + h2.record(50 + Math.ceil(Math.random() * 10)); + } + const identical = h1.welchTest(h2); + assert.strictEqual(typeof identical.tStatistic, 'number'); + assert.strictEqual(typeof identical.degreesOfFreedom, 'number'); + assert.strictEqual(typeof identical.pValue, 'number'); + assert.ok(identical.pValue >= 0 && identical.pValue <= 1); + assert.ok(identical.degreesOfFreedom > 0); + assert.strictEqual(typeof identical.confidenceInterval.lower, 'number'); + assert.strictEqual(typeof identical.confidenceInterval.upper, 'number'); + assert.ok(identical.confidenceInterval.lower <= + identical.confidenceInterval.upper); + + // Very different distributions → low p-value (significant) + const hLow = createHistogram(); + const hHigh = createHistogram(); + for (let i = 0; i < 200; i++) hLow.record(10 + Math.ceil(Math.random() * 5)); + for (let i = 0; i < 200; i++) { + hHigh.record(1000 + Math.ceil(Math.random() * 5)); + } + const different = hLow.welchTest(hHigh); + assert.ok(different.pValue < 0.001, + `Expected p < 0.001, got ${different.pValue}`); + assert.ok(different.tStatistic < 0, 'hLow mean < hHigh mean → negative t'); + + // Confidence interval should not contain 0 when significant + assert.ok(different.confidenceInterval.upper < 0 || + different.confidenceInterval.lower > 0); + + // Same histogram → p-value 1 + const self = hLow.welchTest(hLow); + assert.strictEqual(self.pValue, 1); + + // Custom confidence level + const ci90 = hLow.welchTest(hHigh, { confidence: 0.90 }); + const ci99 = hLow.welchTest(hHigh, { confidence: 0.99 }); + // 99% CI should be wider than 90% CI + const width90 = ci90.confidenceInterval.upper - + ci90.confidenceInterval.lower; + const width99 = ci99.confidenceInterval.upper - + ci99.confidenceInterval.lower; + assert.ok(width99 > width90); + + // Validation + assert.throws(() => h1.welchTest('not a histogram'), + { code: 'ERR_INVALID_ARG_TYPE' }); + assert.throws(() => h1.welchTest(h2, { confidence: 0 }), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h1.welchTest(h2, { confidence: 1 }), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h1.welchTest(h2, { confidence: 'high' }), + { code: 'ERR_INVALID_ARG_TYPE' }); +} + +// --------------------------------------------------------------------------- +// mannWhitneyTest(other) — Mann-Whitney U test +// --------------------------------------------------------------------------- +{ + const h1 = createHistogram(); + const h2 = createHistogram(); + + // Both empty → p-value 1 + const empty = h1.mannWhitneyTest(h2); + assert.strictEqual(empty.pValue, 1); + assert.strictEqual(empty.uStatistic, 0); + assert.strictEqual(empty.zScore, 0); + + // Very different distributions → significant + const hLow = createHistogram(); + const hHigh = createHistogram(); + for (let i = 0; i < 100; i++) hLow.record(1 + Math.ceil(Math.random() * 5)); + for (let i = 0; i < 100; i++) { + hHigh.record(1000 + Math.ceil(Math.random() * 5)); + } + const result = hLow.mannWhitneyTest(hHigh); + assert.strictEqual(typeof result.uStatistic, 'number'); + assert.strictEqual(typeof result.zScore, 'number'); + assert.strictEqual(typeof result.pValue, 'number'); + assert.ok(result.pValue < 0.001, + `Expected p < 0.001, got ${result.pValue}`); + + // Same histogram → p-value 1 + const self = hLow.mannWhitneyTest(hLow); + assert.strictEqual(self.pValue, 1); + + // Identical data → high p-value + const a = createHistogram(); + const b = createHistogram(); + for (let i = 1; i <= 50; i++) { a.record(i); b.record(i); } + const same = a.mannWhitneyTest(b); + assert.ok(same.pValue > 0.05, + `Expected p > 0.05, got ${same.pValue}`); + + // Validation + assert.throws(() => h1.mannWhitneyTest('not a histogram'), + { code: 'ERR_INVALID_ARG_TYPE' }); +} + +// --------------------------------------------------------------------------- +// cohensD(other) — Cohen's d effect size +// --------------------------------------------------------------------------- +{ + const h1 = createHistogram(); + const h2 = createHistogram(); + + // Both empty → 0 + assert.strictEqual(h1.cohensD(h2), 0); + + // Same histogram → 0 + for (let i = 1; i <= 100; i++) h1.record(i); + assert.strictEqual(h1.cohensD(h1), 0); + + // Identical distributions → near 0 + const a = createHistogram(); + const b = createHistogram(); + for (let i = 0; i < 100; i++) { + const v = 50 + Math.ceil(Math.random() * 10); + a.record(v); + b.record(v); + } + assert.ok(Math.abs(a.cohensD(b)) < 0.5); + + // Very different distributions → large |d| + const hLow = createHistogram(); + const hHigh = createHistogram(); + for (let i = 0; i < 200; i++) hLow.record(8 + Math.ceil(Math.random() * 5)); + for (let i = 0; i < 200; i++) { + hHigh.record(998 + Math.ceil(Math.random() * 5)); + } + const d = hLow.cohensD(hHigh); + assert.ok(Math.abs(d) > 1.0, + `Expected |d| > 1, got ${d}`); + // hLow has lower mean → d should be negative + assert.ok(d < 0); + + // Antisymmetry: d(a,b) = -d(b,a) + const dReverse = hHigh.cohensD(hLow); + assert.ok(Math.abs(d + dReverse) < 1e-10); + + // Uniform variance → 0 + const u1 = createHistogram(); + const u2 = createHistogram(); + for (let i = 0; i < 100; i++) u1.record(5); + for (let i = 0; i < 100; i++) u2.record(5); + assert.strictEqual(u1.cohensD(u2), 0); + + // Validation + assert.throws(() => h1.cohensD('not a histogram'), + { code: 'ERR_INVALID_ARG_TYPE' }); +} + +// --------------------------------------------------------------------------- +// cliffsD(other) — Cliff's delta +// --------------------------------------------------------------------------- +{ + const h1 = createHistogram(); + const h2 = createHistogram(); + + // Both empty → 0 + assert.strictEqual(h1.cliffsD(h2), 0); + + // Same histogram → 0 + for (let i = 1; i <= 100; i++) h1.record(i); + assert.strictEqual(h1.cliffsD(h1), 0); + + // All values in h1 > all values in h2 → delta = 1 + const hHigh = createHistogram(); + const hLow = createHistogram(); + for (let i = 0; i < 100; i++) hHigh.record(1000); + for (let i = 0; i < 100; i++) hLow.record(1); + assert.strictEqual(hHigh.cliffsD(hLow), 1); + + // All values in h1 < all values in h2 → delta = -1 + assert.strictEqual(hLow.cliffsD(hHigh), -1); + + // Antisymmetry: d(a,b) = -d(b,a) + const a = createHistogram(); + const b = createHistogram(); + for (let i = 0; i < 50; i++) a.record(1 + Math.ceil(Math.random() * 100)); + for (let i = 0; i < 50; i++) { + b.record(50 + Math.ceil(Math.random() * 100)); + } + const dAB = a.cliffsD(b); + const dBA = b.cliffsD(a); + assert.ok(Math.abs(dAB + dBA) < 1e-10); + + // Range check: -1 <= delta <= 1 + assert.ok(dAB >= -1 && dAB <= 1); + + // Identical data → 0 + const x = createHistogram(); + const y = createHistogram(); + for (let i = 1; i <= 50; i++) { x.record(i); y.record(i); } + assert.strictEqual(x.cliffsD(y), 0); + + // Validation + assert.throws(() => h1.cliffsD('not a histogram'), + { code: 'ERR_INVALID_ARG_TYPE' }); +} + +// --------------------------------------------------------------------------- +// percentileCI(percentile[, options]) — percentile confidence intervals +// --------------------------------------------------------------------------- +{ + const h = createHistogram(); + + // With < 2 samples, lower/upper equal value + h.record(50); + const one = h.percentileCI(99); + assert.strictEqual(one.lower, one.value); + assert.strictEqual(one.upper, one.value); + + // Fill with enough data for a meaningful CI + for (let i = 1; i <= 1000; i++) h.record(i); + const ci = h.percentileCI(50); + assert.strictEqual(typeof ci.value, 'number'); + assert.strictEqual(typeof ci.lower, 'number'); + assert.strictEqual(typeof ci.upper, 'number'); + assert.ok(ci.lower <= ci.value, `lower ${ci.lower} <= value ${ci.value}`); + assert.ok(ci.upper >= ci.value, `upper ${ci.upper} >= value ${ci.value}`); + + // 99% CI should be wider than 90% CI + const ci90 = h.percentileCI(50, { confidence: 0.90 }); + const ci99 = h.percentileCI(50, { confidence: 0.99 }); + assert.ok((ci99.upper - ci99.lower) >= (ci90.upper - ci90.lower), + '99% CI should be at least as wide as 90% CI'); + + // Extreme percentile: p99 CI + const ci99p = h.percentileCI(99); + assert.ok(ci99p.lower <= ci99p.value); + assert.ok(ci99p.upper >= ci99p.value); + + // Constant values → CI collapses to a single value + const constant = createHistogram(); + for (let i = 0; i < 100; i++) constant.record(42); + const constCI = constant.percentileCI(50); + assert.strictEqual(constCI.lower, constCI.value); + assert.strictEqual(constCI.upper, constCI.value); + + // More samples → narrower CI + const small = createHistogram(); + const large = createHistogram(); + for (let i = 1; i <= 50; i++) { small.record(i); large.record(i); } + for (let i = 1; i <= 950; i++) large.record(i % 50 + 1); + const ciSmall = small.percentileCI(50); + const ciLarge = large.percentileCI(50); + assert.ok((ciSmall.upper - ciSmall.lower) >= (ciLarge.upper - ciLarge.lower), + 'CI should narrow with more samples'); + + // Validation + assert.throws(() => h.percentileCI(0), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h.percentileCI(101), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h.percentileCI('fifty'), + { code: 'ERR_INVALID_ARG_TYPE' }); + assert.throws(() => h.percentileCI(50, { confidence: 0 }), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h.percentileCI(50, { confidence: 1 }), + { code: 'ERR_OUT_OF_RANGE' }); +} + +// --------------------------------------------------------------------------- +// EWMA — exponentially weighted moving average +// --------------------------------------------------------------------------- +{ + // Without halfLife, EWMA is disabled (returns 0) + const noEwma = createHistogram(); + for (let i = 1; i <= 100; i++) noEwma.record(i); + assert.strictEqual(noEwma.ewmaMean, 0); + assert.strictEqual(noEwma.ewmaStddev, 0); + + // With halfLife, EWMA tracks the smoothed mean + const h = createHistogram({ halfLife: 10 }); + assert.strictEqual(h.ewmaMean, 0); + assert.strictEqual(h.ewmaStddev, 0); + + // First record initializes the mean + h.record(100); + assert.strictEqual(h.ewmaMean, 100); + assert.strictEqual(h.ewmaStddev, 0); + + // Record the same value repeatedly — mean should stay stable + for (let i = 0; i < 50; i++) h.record(100); + assert.ok(Math.abs(h.ewmaMean - 100) < 1, + `Expected ewmaMean near 100, got ${h.ewmaMean}`); + assert.ok(h.ewmaStddev < 1, + `Expected near-zero stddev for constant input, got ${h.ewmaStddev}`); + + // Shift to a new value — mean should move towards it + const meanBefore = h.ewmaMean; + for (let i = 0; i < 100; i++) h.record(200); + assert.ok(h.ewmaMean > meanBefore, + 'EWMA mean should increase when recording larger values'); + assert.ok(Math.abs(h.ewmaMean - 200) < 5, + `Expected ewmaMean near 200, got ${h.ewmaMean}`); + + // Stddev should be small after converging + for (let i = 0; i < 100; i++) h.record(200); + assert.ok(h.ewmaStddev < 5, + `Expected small stddev after convergence, got ${h.ewmaStddev}`); + + // Reset clears EWMA state + h.reset(); + assert.strictEqual(h.ewmaMean, 0); + assert.strictEqual(h.ewmaStddev, 0); + + // Shorter halfLife reacts faster + const fast = createHistogram({ halfLife: 2 }); + const slow = createHistogram({ halfLife: 100 }); + for (let i = 0; i < 20; i++) { fast.record(100); slow.record(100); } + for (let i = 0; i < 20; i++) { fast.record(200); slow.record(200); } + // Fast should be closer to 200 than slow + assert.ok(fast.ewmaMean > slow.ewmaMean, + `fast.ewmaMean (${fast.ewmaMean}) should be > ` + + `slow.ewmaMean (${slow.ewmaMean})`); + + // toJSON includes separate EWMA fields + const j = createHistogram({ halfLife: 10, threshold: 50 }); + j.record(50); + j.record(60); + const json = j.toJSON(); + // mean/stddev are always the histogram (non-EWMA) values + assert.strictEqual(json.mean, j.mean); + assert.strictEqual(json.stddev, j.stddev); + // EWMA fields are present and match getter values + assert.strictEqual(json.ewmaMean, j.ewmaMean); + assert.strictEqual(json.ewmaStddev, j.ewmaStddev); + assert.strictEqual(json.ewmaErrorRate, j.ewmaErrorRate); + assert.ok(json.ewmaMean > 0); + assert.ok(json.ewmaErrorRate > 0); + + // toJSON still includes EWMA fields when EWMA is not enabled (all zero) + const noEwmaJson = createHistogram(); + noEwmaJson.record(50); + noEwmaJson.record(60); + const json2 = noEwmaJson.toJSON(); + assert.strictEqual(json2.mean, noEwmaJson.mean); + assert.strictEqual(json2.stddev, noEwmaJson.stddev); + assert.strictEqual(json2.ewmaMean, 0); + assert.strictEqual(json2.ewmaStddev, 0); + assert.strictEqual(json2.ewmaErrorRate, 0); + + // Validation + assert.throws(() => createHistogram({ halfLife: -1 }), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => createHistogram({ halfLife: 'ten' }), + { code: 'ERR_INVALID_ARG_TYPE' }); +} + +// --------------------------------------------------------------------------- +// ewmaErrorRate / burnRate — SLO error rate tracking +// --------------------------------------------------------------------------- +{ + // Without threshold, error rate is 0 + const noThreshold = createHistogram({ halfLife: 10 }); + for (let i = 0; i < 50; i++) noThreshold.record(100); + assert.strictEqual(noThreshold.ewmaErrorRate, 0); + + // Without halfLife, error rate is 0 even with threshold + const noHalfLife = createHistogram({ threshold: 50 }); + for (let i = 0; i < 50; i++) noHalfLife.record(100); + assert.strictEqual(noHalfLife.ewmaErrorRate, 0); + + // All values below threshold → error rate converges to 0 + const allGood = createHistogram({ halfLife: 10, threshold: 200 }); + for (let i = 0; i < 100; i++) allGood.record(100); + assert.ok(allGood.ewmaErrorRate < 0.01, + `Expected near-zero error rate, got ${allGood.ewmaErrorRate}`); + + // All values above threshold → error rate converges to 1 + const allBad = createHistogram({ halfLife: 10, threshold: 50 }); + for (let i = 0; i < 100; i++) allBad.record(100); + assert.ok(allBad.ewmaErrorRate > 0.99, + `Expected near-1 error rate, got ${allBad.ewmaErrorRate}`); + + // Mixed: ~50% above threshold + const mixed = createHistogram({ halfLife: 50, threshold: 50 }); + for (let i = 0; i < 500; i++) { + mixed.record(i % 2 === 0 ? 100 : 10); // Alternating above/below + } + assert.ok(mixed.ewmaErrorRate > 0.3 && mixed.ewmaErrorRate < 0.7, + `Expected ~0.5 error rate, got ${mixed.ewmaErrorRate}`); + + // burnRate calculation + const h = createHistogram({ halfLife: 10, threshold: 50 }); + for (let i = 0; i < 100; i++) h.record(100); // All exceed + // Error rate ~1.0, SLO target 0.999 → budget 0.001 → burn rate ~1000 + const rate = h.burnRate(0.999); + assert.ok(rate > 500, + `Expected high burn rate, got ${rate}`); + + // When error rate is 0, burn rate is 0 + const perfect = createHistogram({ halfLife: 10, threshold: 200 }); + for (let i = 0; i < 100; i++) perfect.record(100); + assert.ok(perfect.burnRate(0.999) < 1, + `Expected low burn rate, got ${perfect.burnRate(0.999)}`); + + // Reset clears error rate + h.reset(); + assert.strictEqual(h.ewmaErrorRate, 0); + assert.strictEqual(h.burnRate(0.999), 0); + + // burnRate validation + assert.throws(() => h.burnRate(0), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h.burnRate(1), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h.burnRate(NaN), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => h.burnRate('high'), + { code: 'ERR_INVALID_ARG_TYPE' }); + + // createHistogram threshold validation + assert.throws(() => createHistogram({ threshold: -1 }), + { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => createHistogram({ threshold: 'high' }), + { code: 'ERR_INVALID_ARG_TYPE' }); +} + +// --------------------------------------------------------------------------- +// export() / importHistogram() — CBOR round-trip +// --------------------------------------------------------------------------- +{ + // Basic round-trip + const h = createHistogram(); + for (let i = 1; i <= 1000; i++) h.record(i); + + const buf = h.export(); + assert.ok(buf instanceof Uint8Array, 'export should return Uint8Array'); + assert.ok(buf.length > 0, 'export should not be empty'); + + const h2 = importHistogram(buf); + assert.strictEqual(h2.count, h.count); + assert.strictEqual(h2.min, h.min); + assert.strictEqual(h2.max, h.max); + assert.strictEqual(h2.mean, h.mean); + assert.strictEqual(h2.stddev, h.stddev); + assert.strictEqual(h2.percentile(50), h.percentile(50)); + assert.strictEqual(h2.percentile(99), h.percentile(99)); + assert.strictEqual(h2.percentile(99.9), h.percentile(99.9)); + + // Imported histogram is recordable + h2.record(9999); + assert.strictEqual(h2.count, h.count + 1); + + // Round-trip with EWMA and threshold + const h3 = createHistogram({ halfLife: 10, threshold: 500 }); + for (let i = 1; i <= 200; i++) h3.record(i); + const buf3 = h3.export(); + const h4 = importHistogram(buf3); + assert.strictEqual(h4.count, h3.count); + assert.strictEqual(h4.ewmaMean, h3.ewmaMean); + assert.strictEqual(h4.ewmaStddev, h3.ewmaStddev); + assert.strictEqual(h4.ewmaErrorRate, h3.ewmaErrorRate); + + // Empty histogram round-trip + const empty = createHistogram(); + const emptyBuf = empty.export(); + const empty2 = importHistogram(emptyBuf); + assert.strictEqual(empty2.count, 0); + assert.strictEqual(empty2.min, 9223372036854776000); // INT64_MAX as double + + // Sparse: only a few distinct values + const sparse = createHistogram(); + sparse.record(1); + sparse.record(1000000); + const sparseBuf = sparse.export(); + const sparse2 = importHistogram(sparseBuf); + assert.strictEqual(sparse2.count, 2); + assert.strictEqual(sparse2.percentile(1), sparse.percentile(1)); + assert.strictEqual(sparse2.percentile(100), sparse.percentile(100)); + + // Size scales with distinct values, not total bucket count + assert.ok(sparseBuf.length < 200, + `Sparse export should be small, got ${sparseBuf.length}`); + + // Validation — type and format + assert.throws(() => importHistogram('not a uint8array'), + { code: 'ERR_INVALID_ARG_TYPE' }); + assert.throws(() => importHistogram(new Uint8Array(0)), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => importHistogram(new Uint8Array([0xff, 0xff])), + { code: 'ERR_INVALID_ARG_VALUE' }); + + // --- hdr_init failures (invalid histogram options) --- + + // lowest=0 violates lowest>=1. + assert.throws(() => importHistogram(new Uint8Array([0xa1, 0x01, 0x00])), + { code: 'ERR_INVALID_ARG_VALUE' }); + // figures=0 violates figures>=1. + assert.throws(() => importHistogram(new Uint8Array([0xa1, 0x03, 0x00])), + { code: 'ERR_INVALID_ARG_VALUE' }); + // figures=6 violates figures<=5. + assert.throws(() => importHistogram(new Uint8Array([0xa1, 0x03, 0x06])), + { code: 'ERR_INVALID_ARG_VALUE' }); + // lowest=100, highest=100: lowest*2 > highest. + assert.throws(() => importHistogram(new Uint8Array([ + 0xa2, // map(2) + 0x01, 0x18, 100, // 1 (lowest) = 100 + 0x02, 0x18, 100, // 2 (highest) = 100 + ])), { code: 'ERR_INVALID_ARG_VALUE' }); + // Large values that trigger hdr_init internal overflow + // (unit_magnitude + sub_bucket_half_count_magnitude > 61). + assert.throws(() => importHistogram(new Uint8Array([ + 0xa3, // map(3) + 0x01, 0x1b, 0, 0, 0x20, 0, 0, 0, 0, 0, // 1 (lowest) = 2**45 + 0x02, 0x1b, 0, 0, 0x40, 0, 0, 0, 0, 0, // 2 (highest) = 2**46 + 0x03, 0x05, // 3 (figures) = 5 + ])), { code: 'ERR_INVALID_ARG_VALUE' }); + + // --- Structural CBOR validation --- + + // Wrong version number (version=99). + assert.throws(() => importHistogram(new Uint8Array([ + 0xa1, // map(1) + 0x00, 0x18, 99, // 0 (version) = 99 + ])), { code: 'ERR_INVALID_ARG_VALUE' }); + // Unknown top-level key (key=255). + assert.throws(() => importHistogram(new Uint8Array([ + 0xa1, // map(1) + 0x18, 0xff, 0x00, // 255 (unknown) = 0 + ])), { code: 'ERR_INVALID_ARG_VALUE' }); + // Counts array with odd length (must be even: delta/count pairs). + assert.throws(() => importHistogram(new Uint8Array([ + 0xa1, // map(1) + 0x0a, 0x81, 0x01, // 10 (counts) = [1] + ])), { code: 'ERR_INVALID_ARG_VALUE' }); + + // --- Post-construction validation --- + + // counts_len mismatch: valid options but declared counts_len + // doesn't match what hdr_init actually produces. + assert.throws(() => importHistogram(new Uint8Array([ + 0xa2, // map(2) + 0x09, 0x01, // 9 (countsLen) = 1 + 0x0a, 0x80, // 10 (counts) = [] + ])), { code: 'ERR_INVALID_ARG_VALUE' }); + // Oversized counts array: declared length exceeds remaining buffer. + // Without bounds checking, reserve() would OOM-crash. + assert.throws(() => importHistogram(new Uint8Array([ + 0xa1, // map(1) + 0x0a, 0x9b, // 10 (counts) = array( + 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2**52 elements) + ])), { code: 'ERR_INVALID_ARG_VALUE' }); + // Sparse count index out of bounds: lowest=1, highest=100, figures=1 + // produces counts_len=64. Index 100 exceeds it. + assert.throws(() => importHistogram(new Uint8Array([ + 0xa4, // map(4) + 0x02, 0x18, 0x64, // 2 (highest) = 100 + 0x03, 0x01, // 3 (figures) = 1 + 0x09, 0x18, 0x40, // 9 (countsLen) = 64 + 0x0a, 0x82, 0x18, 0x64, 0x01, // 10 (counts) = [100, 1] + ])), { code: 'ERR_INVALID_ARG_VALUE' }); +} + +// --------------------------------------------------------------------------- +// ERR_INVALID_THIS for all new methods on wrong receiver +// --------------------------------------------------------------------------- +{ + const { Histogram } = require('internal/histogram'); + const h = createHistogram(); + for (let i = 1; i <= 10; i++) h.record(i); + const wrongThis = {}; + + const methods = [ + ['welchTest', [h]], + ['mannWhitneyTest', [h]], + ['cohensD', [h]], + ['cliffsD', [h]], + ['percentileCI', [50]], + ['burnRate', [0.999]], + ]; + + for (const [method, args] of methods) { + assert.throws( + () => Histogram.prototype[method].call(wrongThis, ...args), + { code: 'ERR_INVALID_THIS' }, + `${method} should throw ERR_INVALID_THIS`, + ); + } + + // Getter properties + const getters = ['ewmaMean', 'ewmaStddev', 'ewmaErrorRate']; + for (const getter of getters) { + const desc = Object.getOwnPropertyDescriptor(Histogram.prototype, getter); + assert.throws( + () => desc.get.call(wrongThis), + { code: 'ERR_INVALID_THIS' }, + `${getter} should throw ERR_INVALID_THIS`, + ); + } +} + +// --------------------------------------------------------------------------- +// Undefined return when kHandle is missing native methods +// --------------------------------------------------------------------------- +{ + const { + Histogram, + kHandle, + kSkipThrow, + } = require('internal/histogram'); + const h = createHistogram(); + for (let i = 1; i <= 10; i++) h.record(i); + + // Create a histogram instance with a null handle. This passes + // isHistogram() (null !== undefined) but the optional chaining + // (this[kHandle]?.method()) short-circuits to undefined. + const stub = new Histogram(kSkipThrow); + stub[kHandle] = null; + + assert.strictEqual(stub.welchTest(h), undefined); + assert.strictEqual(stub.mannWhitneyTest(h), undefined); + assert.strictEqual(stub.percentileCI(50), undefined); + assert.strictEqual(stub.burnRate(0.999), undefined); +} + +// --------------------------------------------------------------------------- +// Fast API path coverage for EWMA getters +// --------------------------------------------------------------------------- +{ + const h = createHistogram({ halfLife: 10, threshold: 50 }); + for (let i = 1; i <= 100; i++) h.record(i); + + // Call in a tight loop to trigger V8 fast-path optimization. + function readEwma(histogram, iterations) { + let mean = 0; + let stddev = 0; + let errorRate = 0; + for (let i = 0; i < iterations; i++) { + mean = histogram.ewmaMean; + stddev = histogram.ewmaStddev; + errorRate = histogram.ewmaErrorRate; + } + return { mean, stddev, errorRate }; + } + + const result = readEwma(h, 1e4); + assert.strictEqual(typeof result.mean, 'number'); + assert.ok(result.mean > 0); + assert.strictEqual(typeof result.stddev, 'number'); + assert.ok(result.stddev > 0); + assert.strictEqual(typeof result.errorRate, 'number'); + assert.ok(result.errorRate > 0); +} + +// --------------------------------------------------------------------------- +// Cross-consistency: when welchTest is significant, cohensD should +// indicate a non-trivial effect, and cliffsD should agree on direction. +// --------------------------------------------------------------------------- +{ + const baseline = createHistogram(); + const regressed = createHistogram(); + for (let i = 0; i < 500; i++) { + baseline.record(10 + Math.ceil(Math.random() * 20)); + } + for (let i = 0; i < 500; i++) { + regressed.record(50 + Math.ceil(Math.random() * 20)); + } + + const welch = baseline.welchTest(regressed); + const d = baseline.cohensD(regressed); + const cliff = baseline.cliffsD(regressed); + + // Should be highly significant + assert.ok(welch.pValue < 0.001); + // Cohen's d should indicate a large effect (|d| > 0.8) + assert.ok(Math.abs(d) > 0.8); + // Cliff's delta should indicate baseline < regressed + assert.ok(cliff < -0.5); + // All three agree on the direction + assert.ok(d < 0); // Baseline mean < regressed mean + assert.ok(welch.tStatistic < 0); +}