From e52129e1dbf4e5ba2da60b7769c79eab12f4d592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20Ekin=20Erdo=C4=9Fdu?= <65937473+ilker-ekin@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:10:41 +0300 Subject: [PATCH] feat: export k6 latency percentiles as JSON summaries (#36) Benchmark runs could only be compared by scraping k6 console output. Both k6 scenarios now report p50/p90/p95/p99 via a shared summaryTrendStats setting and use a shared handleSummary that prints a short console summary and writes k6-summary--.json with the scenario, run parameters, latency, throughput, HTTP failure and check rates, threshold results and raw metric values. Existing requests, checks and thresholds are unchanged. The benchmark report documents how to compare runs and interpret results per environment. Co-Authored-By: Claude Opus 5.5 --- docs/benchmark-and-validation.md | 13 ++++++ load-tests/baseline.js | 5 +++ load-tests/distributed.js | 7 ++++ load-tests/lib/summary.js | 71 ++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 load-tests/lib/summary.js diff --git a/docs/benchmark-and-validation.md b/docs/benchmark-and-validation.md index 5d3b924..a7df4a4 100644 --- a/docs/benchmark-and-validation.md +++ b/docs/benchmark-and-validation.md @@ -744,6 +744,19 @@ Current validated performance: These results are environment-specific and should be treated as benchmark observations, not universal capacity guarantees. +## Benchmark summary artifacts + +Both k6 scripts use `load-tests/lib/summary.js`. Each run prints a short console summary and writes `k6-summary--.json` to the k6 working directory, containing the scenario, the run parameters, latency p50/p90/p95/p99, throughput, HTTP failure rate, check pass rate, each threshold with its pass/fail result, and the raw values of every k6 metric. + +Run the scripts from a file path (not `k6 run -`) so the shared module resolves, for example `k6 run load-tests/baseline.js`. Two runs are compared by diffing their JSON files. + +When comparing runs: + +- Only compare runs of the same scenario and parameters on the same environment; absolute latency and throughput depend on the host, Docker resources and network path. +- The HTTP failure rate counts HTTP errors only. `HTTP 200 + REJECTED` is successful enforcement (see section 14), so use `traffic_decision_allowed` / `traffic_decision_rejected` in the distributed results for the admission split. +- The distributed scenario exhausts the global capacity pool, so start each comparable run from the same Redis state. +- p99 and max are sensitive to warm-up and short runs; prefer p50–p95 for run-to-run comparison. + --- # 19. Validation Checklist diff --git a/load-tests/baseline.js b/load-tests/baseline.js index 52b3aa6..59718c3 100644 --- a/load-tests/baseline.js +++ b/load-tests/baseline.js @@ -1,5 +1,6 @@ import http from 'k6/http'; import {check} from 'k6'; +import {createSummaryHandler, SUMMARY_TREND_STATS} from './lib/summary.js'; export const options = { vus: 10, duration: '30s', @@ -7,10 +8,14 @@ export const options = { thresholds: { http_req_failed: ['rate<0.05'], http_req_duration: ['p(95)<500'], }, + + summaryTrendStats: SUMMARY_TREND_STATS, }; const BASE_URL = __ENV.BASE_URL || 'http://host.docker.internal:8081'; +export const handleSummary = createSummaryHandler('baseline', {...options, baseUrl: BASE_URL}); + export default function () { const payload = JSON.stringify({ requestId: `k6-${__VU}-${__ITER}-${Date.now()}`, subject: { diff --git a/load-tests/distributed.js b/load-tests/distributed.js index a9dbc02..f9f7a7c 100644 --- a/load-tests/distributed.js +++ b/load-tests/distributed.js @@ -2,6 +2,7 @@ import http from 'k6/http'; import { check } from 'k6'; import { Counter, Rate } from 'k6/metrics'; +import { createSummaryHandler, SUMMARY_TREND_STATS } from './lib/summary.js'; export const options = { vus: 1, @@ -10,6 +11,7 @@ export const options = { http_req_failed: ['rate<0.001'], http_req_duration: ['p(95)<500'], }, + summaryTrendStats: SUMMARY_TREND_STATS, }; const GATEWAY_URLS = ( @@ -28,6 +30,11 @@ const gatewayRequests = GATEWAY_URLS.map( (_, index) => new Counter(`gateway_${index + 1}_requests`) ); +export const handleSummary = createSummaryHandler('distributed', { + ...options, + gatewayUrls: GATEWAY_URLS, +}); + const allowedDecisions = new Rate('traffic_decision_allowed'); const rejectedDecisions = new Rate('traffic_decision_rejected'); const unknownDecisions = new Rate('traffic_decision_unknown'); diff --git a/load-tests/lib/summary.js b/load-tests/lib/summary.js new file mode 100644 index 0000000..b45f449 --- /dev/null +++ b/load-tests/lib/summary.js @@ -0,0 +1,71 @@ +// Shared k6 summary: latency percentiles, a short console summary and a JSON artifact per run. + +export const SUMMARY_TREND_STATS = ['avg', 'min', 'p(50)', 'p(90)', 'p(95)', 'p(99)', 'max']; + +export function createSummaryHandler(scenario, parameters) { + return function handleSummary(data) { + const report = buildReport(scenario, parameters, data); + const timestamp = report.generatedAt.replace(/[:.]/g, '-'); + + return { + stdout: renderConsoleSummary(report), + [`k6-summary-${scenario}-${timestamp}.json`]: JSON.stringify(report, null, 2), + }; + }; +} + +function buildReport(scenario, parameters, data) { + const value = (name, stat) => { + const metric = data.metrics[name]; + return metric && metric.values[stat] !== undefined ? metric.values[stat] : null; + }; + + const thresholds = []; + const metrics = {}; + + Object.keys(data.metrics).sort().forEach((name) => { + const metric = data.metrics[name]; + metrics[name] = metric.values; + + Object.keys(metric.thresholds || {}).forEach((expression) => { + thresholds.push({ metric: name, expression, ok: metric.thresholds[expression].ok }); + }); + }); + + return { + scenario, + generatedAt: new Date().toISOString(), + parameters, + latencyMs: { + avg: value('http_req_duration', 'avg'), + p50: value('http_req_duration', 'p(50)'), + p90: value('http_req_duration', 'p(90)'), + p95: value('http_req_duration', 'p(95)'), + p99: value('http_req_duration', 'p(99)'), + max: value('http_req_duration', 'max'), + }, + requests: value('http_reqs', 'count'), + requestsPerSecond: value('http_reqs', 'rate'), + httpFailureRate: value('http_req_failed', 'rate'), + checksPassRate: value('checks', 'rate'), + thresholds, + metrics, + }; +} + +function renderConsoleSummary(report) { + const ms = (stat) => (report.latencyMs[stat] === null ? 'n/a' : report.latencyMs[stat].toFixed(3)); + const percent = (rate) => (rate === null ? 'n/a' : `${(rate * 100).toFixed(2)}%`); + + const lines = [ + '', + `Scenario: ${report.scenario}`, + `Requests: ${report.requests} Throughput: ${(report.requestsPerSecond || 0).toFixed(2)} req/s`, + `Latency (ms): avg=${ms('avg')} p50=${ms('p50')} p90=${ms('p90')} p95=${ms('p95')} p99=${ms('p99')} max=${ms('max')}`, + `HTTP failures: ${percent(report.httpFailureRate)} Checks passed: ${percent(report.checksPassRate)}`, + ...report.thresholds.map((t) => `${t.ok ? 'PASS' : 'FAIL'} ${t.metric}: ${t.expression}`), + '', + ]; + + return lines.join('\n'); +}