diff --git a/docs/benchmark-and-validation.md b/docs/benchmark-and-validation.md index 5d3b924..a7df4a4 100644 --- a/docs/benchmark-and-validation.md +++ b/docs/benchmark-and-validation.md @@ -744,6 +744,19 @@ Current validated performance: These results are environment-specific and should be treated as benchmark observations, not universal capacity guarantees. +## Benchmark summary artifacts + +Both k6 scripts use `load-tests/lib/summary.js`. Each run prints a short console summary and writes `k6-summary--.json` to the k6 working directory, containing the scenario, the run parameters, latency p50/p90/p95/p99, throughput, HTTP failure rate, check pass rate, each threshold with its pass/fail result, and the raw values of every k6 metric. + +Run the scripts from a file path (not `k6 run -`) so the shared module resolves, for example `k6 run load-tests/baseline.js`. Two runs are compared by diffing their JSON files. + +When comparing runs: + +- Only compare runs of the same scenario and parameters on the same environment; absolute latency and throughput depend on the host, Docker resources and network path. +- The HTTP failure rate counts HTTP errors only. `HTTP 200 + REJECTED` is successful enforcement (see section 14), so use `traffic_decision_allowed` / `traffic_decision_rejected` in the distributed results for the admission split. +- The distributed scenario exhausts the global capacity pool, so start each comparable run from the same Redis state. +- p99 and max are sensitive to warm-up and short runs; prefer p50–p95 for run-to-run comparison. + --- # 19. Validation Checklist diff --git a/load-tests/baseline.js b/load-tests/baseline.js index 52b3aa6..59718c3 100644 --- a/load-tests/baseline.js +++ b/load-tests/baseline.js @@ -1,5 +1,6 @@ import http from 'k6/http'; import {check} from 'k6'; +import {createSummaryHandler, SUMMARY_TREND_STATS} from './lib/summary.js'; export const options = { vus: 10, duration: '30s', @@ -7,10 +8,14 @@ export const options = { thresholds: { http_req_failed: ['rate<0.05'], http_req_duration: ['p(95)<500'], }, + + summaryTrendStats: SUMMARY_TREND_STATS, }; const BASE_URL = __ENV.BASE_URL || 'http://host.docker.internal:8081'; +export const handleSummary = createSummaryHandler('baseline', {...options, baseUrl: BASE_URL}); + export default function () { const payload = JSON.stringify({ requestId: `k6-${__VU}-${__ITER}-${Date.now()}`, subject: { diff --git a/load-tests/distributed.js b/load-tests/distributed.js index a9dbc02..f9f7a7c 100644 --- a/load-tests/distributed.js +++ b/load-tests/distributed.js @@ -2,6 +2,7 @@ import http from 'k6/http'; import { check } from 'k6'; import { Counter, Rate } from 'k6/metrics'; +import { createSummaryHandler, SUMMARY_TREND_STATS } from './lib/summary.js'; export const options = { vus: 1, @@ -10,6 +11,7 @@ export const options = { http_req_failed: ['rate<0.001'], http_req_duration: ['p(95)<500'], }, + summaryTrendStats: SUMMARY_TREND_STATS, }; const GATEWAY_URLS = ( @@ -28,6 +30,11 @@ const gatewayRequests = GATEWAY_URLS.map( (_, index) => new Counter(`gateway_${index + 1}_requests`) ); +export const handleSummary = createSummaryHandler('distributed', { + ...options, + gatewayUrls: GATEWAY_URLS, +}); + const allowedDecisions = new Rate('traffic_decision_allowed'); const rejectedDecisions = new Rate('traffic_decision_rejected'); const unknownDecisions = new Rate('traffic_decision_unknown'); diff --git a/load-tests/lib/summary.js b/load-tests/lib/summary.js new file mode 100644 index 0000000..b45f449 --- /dev/null +++ b/load-tests/lib/summary.js @@ -0,0 +1,71 @@ +// Shared k6 summary: latency percentiles, a short console summary and a JSON artifact per run. + +export const SUMMARY_TREND_STATS = ['avg', 'min', 'p(50)', 'p(90)', 'p(95)', 'p(99)', 'max']; + +export function createSummaryHandler(scenario, parameters) { + return function handleSummary(data) { + const report = buildReport(scenario, parameters, data); + const timestamp = report.generatedAt.replace(/[:.]/g, '-'); + + return { + stdout: renderConsoleSummary(report), + [`k6-summary-${scenario}-${timestamp}.json`]: JSON.stringify(report, null, 2), + }; + }; +} + +function buildReport(scenario, parameters, data) { + const value = (name, stat) => { + const metric = data.metrics[name]; + return metric && metric.values[stat] !== undefined ? metric.values[stat] : null; + }; + + const thresholds = []; + const metrics = {}; + + Object.keys(data.metrics).sort().forEach((name) => { + const metric = data.metrics[name]; + metrics[name] = metric.values; + + Object.keys(metric.thresholds || {}).forEach((expression) => { + thresholds.push({ metric: name, expression, ok: metric.thresholds[expression].ok }); + }); + }); + + return { + scenario, + generatedAt: new Date().toISOString(), + parameters, + latencyMs: { + avg: value('http_req_duration', 'avg'), + p50: value('http_req_duration', 'p(50)'), + p90: value('http_req_duration', 'p(90)'), + p95: value('http_req_duration', 'p(95)'), + p99: value('http_req_duration', 'p(99)'), + max: value('http_req_duration', 'max'), + }, + requests: value('http_reqs', 'count'), + requestsPerSecond: value('http_reqs', 'rate'), + httpFailureRate: value('http_req_failed', 'rate'), + checksPassRate: value('checks', 'rate'), + thresholds, + metrics, + }; +} + +function renderConsoleSummary(report) { + const ms = (stat) => (report.latencyMs[stat] === null ? 'n/a' : report.latencyMs[stat].toFixed(3)); + const percent = (rate) => (rate === null ? 'n/a' : `${(rate * 100).toFixed(2)}%`); + + const lines = [ + '', + `Scenario: ${report.scenario}`, + `Requests: ${report.requests} Throughput: ${(report.requestsPerSecond || 0).toFixed(2)} req/s`, + `Latency (ms): avg=${ms('avg')} p50=${ms('p50')} p90=${ms('p90')} p95=${ms('p95')} p99=${ms('p99')} max=${ms('max')}`, + `HTTP failures: ${percent(report.httpFailureRate)} Checks passed: ${percent(report.checksPassRate)}`, + ...report.thresholds.map((t) => `${t.ok ? 'PASS' : 'FAIL'} ${t.metric}: ${t.expression}`), + '', + ]; + + return lines.join('\n'); +}