From 52cce83c6a5ba20221346dc6339899e4bb207bfa Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 11:28:20 +0800 Subject: [PATCH 1/8] fix(examples): keep tinyhttpd connections alive after sending a file http_send_file closed the connection at EOF even when the response said "Connection: keep-alive", so every file request cost a new connection and wrk saw half of its requests go unanswered. Finish the request the same way as other responses once the whole file has been sent, and only then read the next request so pipelined responses cannot interleave with the file body. Co-authored-by: Cursor --- examples/tinyhttpd.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/examples/tinyhttpd.c b/examples/tinyhttpd.c index 5aa6693b6..51f6064fe 100644 --- a/examples/tinyhttpd.c +++ b/examples/tinyhttpd.c @@ -155,6 +155,20 @@ static int http_reply(http_conn_t* conn, return nwrite < 0 ? nwrite : msglen; } +static void http_request_done(http_conn_t* conn) { + if (conn->request.keepalive) { + // Connection: keep-alive\r\n + // reset and receive next request + memset(&conn->request, 0, sizeof(http_msg_t)); + memset(&conn->response, 0, sizeof(http_msg_t)); + conn->state = s_first_line; + hio_readline(conn->io); + } else { + // Connection: close\r\n + hio_close(conn->io); + } +} + static void http_send_file(http_conn_t* conn) { if (!conn || !conn->fp) return; // alloc filebuf @@ -167,8 +181,16 @@ static void http_send_file(http_conn_t* conn) { // read file int nread = fread(filebuf, 1, filebuflen, conn->fp); if (nread <= 0) { - // eof or error - hio_close(conn->io); + int error = ferror(conn->fp); + fclose(conn->fp); + conn->fp = NULL; + hio_setcb_write(conn->io, NULL); + if (error) { + hio_close(conn->io); + } else { + // eof: the whole file has been sent + http_request_done(conn); + } return; } // send file @@ -366,16 +388,10 @@ static void on_recv(hio_t* io, void* buf, int readbytes) { // received complete request on_request(conn); if (hio_is_closed(io)) return; - if (req->keepalive) { - // Connection: keep-alive\r\n - // reset and receive next request - memset(&conn->request, 0, sizeof(http_msg_t)); - memset(&conn->response, 0, sizeof(http_msg_t)); - conn->state = s_first_line; - hio_readline(io); - } else { - // Connection: close\r\n - hio_close(io); + // A file response finishes the request once the whole file is sent, + // which may already have happened inside on_request. + if (conn->state == s_end && conn->fp == NULL) { + http_request_done(conn); } break; default: break; From 6d504224ab371709edecb013100587aeaf2ab1d2 Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 11:28:20 +0800 Subject: [PATCH 2/8] ci(benchmark): validate benchmark results benchmark.yml ran every benchmark but never looked at the numbers. Save each result and check it with .github/scripts/benchmark_check.sh: - correctness: every echo server reports a throughput, libhv HTTP endpoints leave no more unanswered requests than connections, wrk sees no parse errors, and every endpoint answers with the expected status before it is benchmarked - performance: ratios against peers measured in the same run (nginx and the other echo servers), since absolute numbers swing by up to 2x between GitHub-hosted runners Also wait for iperf and tcp_proxy_server to listen before measuring; the direct baseline was missing in 24 of the last 27 runs because the client connected before the server was up. Co-authored-by: Cursor --- .github/scripts/benchmark_check.sh | 123 +++++++++++++++++++++++++++++ .github/workflows/benchmark.yml | 63 ++++++++++----- 2 files changed, 165 insertions(+), 21 deletions(-) create mode 100755 .github/scripts/benchmark_check.sh diff --git a/.github/scripts/benchmark_check.sh b/.github/scripts/benchmark_check.sh new file mode 100755 index 000000000..3738ed62f --- /dev/null +++ b/.github/scripts/benchmark_check.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# +# Validate the results written by .github/workflows/benchmark.yml. +# +# GitHub-hosted runners differ from run to run, so absolute numbers swing by +# up to 2x. Performance is therefore judged by ratios against peers measured in +# the same run (nginx, other echo servers), which stay within a narrow band. +# Floors are ~85% of the lowest ratio seen across 27 recent runs. +# +# usage: benchmark_check.sh [result_dir] +# result_dir holds echo.log, iperf_direct.csv, iperf_proxy.csv, wrk_.log + +cd "${1:-.}" || exit 1 + +# wrk runs with this many connections; at most one request per connection may +# be in flight when the test stops. +MAX_UNANSWERED=110 + +FLOOR_ECHO=0.84 # libhv echo / median of the other echo servers +FLOOR_HTTPD_FILE=1.13 # httpd send file / nginx send file +FLOOR_HTTPD_404=0.81 # httpd 404 / nginx 404 +FLOOR_TINY_404=0.87 # tinyhttpd 404 / nginx 404 +FLOOR_HTTPD_PING=0.89 # httpd /ping / nginx 404 +FLOOR_TINY_PING=0.92 # tinyhttpd /ping / nginx 404 +FLOOR_TCP_PROXY=0.38 # tcp_proxy_server / direct (warn only until more data) + +failed=0 +summary() { [ -n "$GITHUB_STEP_SUMMARY" ] && echo "$1" >> "$GITHUB_STEP_SUMMARY"; } +summary "| result | check | value | limit |" +summary "|---|---|---|---|" + +# wrk prints "N requests, M OK, ..." and "Requests/sec: R" +rps() { awk '/^Requests\/sec:/ {print $2}' "wrk_$1.log" 2>/dev/null; } +unanswered() { awk '/ requests, [0-9]+ OK/ {print $1 - $3}' "wrk_$1.log" 2>/dev/null; } + +# echo-servers/benchmark.sh prints " running on port P", then for each +# port "====P====" followed by "throughput = X MB/s" +echo_tp() { + awk -v want="$1" ' + / running on port / { name[$5] = $1 } + /^=+[0-9]+=+$/ { gsub(/=/, ""); port = $0 } + /^throughput = / { if (name[port] == want) print $3 }' echo.log 2>/dev/null +} + +# iperf -y C: the last field is bits per second +iperf_bps() { awk -F, 'NF > 8 {print $NF}' "iperf_$1.csv" 2>/dev/null | tail -1; } + +median() { sort -n | awk 'NF {a[++n] = $1} END {if (n) print (n % 2) ? a[(n + 1) / 2] : (a[n / 2] + a[n / 2 + 1]) / 2}'; } + +# result name value limit +record() { + printf "%-5s %-44s %-8s %s\n" "$1" "$2" "$3" "$4" + summary "| $1 | $2 | $3 | $4 |" + [ "$1" = FAIL ] && failed=$((failed + 1)) +} + +# name numerator denominator floor [warn|info] +check_ratio() { + local name=$1 num=$2 den=$3 floor=$4 mode=$5 + if [ -z "$num" ] || [ -z "$den" ]; then + record FAIL "$name" "missing" ">= $floor" + return + fi + local ratio ok + ratio=$(awk -v n="$num" -v d="$den" 'BEGIN {if (d + 0 > 0) printf "%.2f", n / d}') + ok=$(awk -v r="$ratio" -v f="$floor" 'BEGIN {print (r != "" && r + 0 >= f) ? 1 : 0}') + if [ "$mode" = info ]; then + record INFO "$name" "$ratio" "-" + elif [ "$ok" = 1 ]; then + record PASS "$name" "$ratio" ">= $floor" + elif [ "$mode" = warn ]; then + record WARN "$name" "$ratio" ">= $floor" + else + record FAIL "$name" "$ratio" ">= $floor" + fi +} + +# name value max +check_max() { + if [ -z "$2" ]; then + record FAIL "$1" "missing" "<= $3" + elif [ "$2" -le "$3" ]; then + record PASS "$1" "$2" "<= $3" + else + record FAIL "$1" "$2" "<= $3" + fi +} + +echo "== correctness ==" +for s in libevent libev libuv libhv asio poco; do + tp=$(echo_tp $s) + if [ -n "$tp" ] && [ "$tp" -gt 0 ]; then + record PASS "echo server $s" "${tp}MB/s" "> 0" + else + record FAIL "echo server $s" "${tp:-missing}" "> 0" + fi +done +for t in httpd_file tinyhttpd_file httpd_ping tinyhttpd_ping; do + check_max "$t unanswered requests" "$(unanswered $t)" $MAX_UNANSWERED +done +for f in wrk_*.log; do + if grep -q "parse error" "$f" 2>/dev/null; then + record FAIL "${f%.log}: http parse error" "found" "none" + fi +done + +echo "== performance (same-run ratios) ==" +others=$(for s in libevent libev libuv asio poco; do echo_tp $s; done | median) +check_ratio "libhv echo / median(other echo servers)" "$(echo_tp libhv)" "$others" $FLOOR_ECHO +check_ratio "httpd file / nginx file" "$(rps httpd_file)" "$(rps nginx_file)" $FLOOR_HTTPD_FILE +check_ratio "httpd 404 / nginx 404" "$(rps httpd_404)" "$(rps nginx_404)" $FLOOR_HTTPD_404 +check_ratio "tinyhttpd 404 / nginx 404" "$(rps tinyhttpd_404)" "$(rps nginx_404)" $FLOOR_TINY_404 +check_ratio "httpd /ping / nginx 404" "$(rps httpd_ping)" "$(rps nginx_404)" $FLOOR_HTTPD_PING +check_ratio "tinyhttpd /ping / nginx 404" "$(rps tinyhttpd_ping)" "$(rps nginx_404)" $FLOOR_TINY_PING +check_ratio "tcp proxy / direct (iperf)" "$(iperf_bps proxy)" "$(iperf_bps direct)" $FLOOR_TCP_PROXY warn +check_ratio "tinyhttpd file / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)" 0 info + +echo +if [ $failed -gt 0 ]; then + echo "$failed check(s) failed" + exit 1 +fi +echo "all checks passed" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 08d872217..50eb5cf15 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,10 +9,15 @@ on: paths-ignore: - '**.md' +defaults: + run: + shell: bash -eo pipefail {0} + jobs: benchmark: name: benchmark runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v7 @@ -34,14 +39,18 @@ jobs: - name: benchmark echo-servers run: | - bash echo-servers/benchmark.sh + bash echo-servers/benchmark.sh | tee echo.log - name: benchmark tcp_proxy_server run: | iperf -s -p 5001 > /dev/null & bin/tcp_proxy_server 1212 127.0.0.1:5001 & - iperf -c 127.0.0.1 -p 5001 -l 8K - iperf -c 127.0.0.1 -p 1212 -l 8K + for i in $(seq 50); do + ss -ltn | grep -q ':5001 ' && ss -ltn | grep -q ':1212 ' && break + sleep 0.1 + done + iperf -c 127.0.0.1 -p 5001 -l 8K -y C | tee iperf_direct.csv + iperf -c 127.0.0.1 -p 1212 -l 8K -y C | tee iperf_proxy.csv - name: webbench run: | @@ -49,22 +58,34 @@ jobs: sudo cp html/index.html index.html sudo cp html/index.html /var/www/html/index.html bin/httpd -c etc/httpd.conf -d - ps aux | grep nginx - ps aux | grep httpd bin/tinyhttpd 9090 & - echo -e "\n======Test nginx send file===============================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:80/ - echo -e "\n======Test libhv/httpd send file (with FileCache)========" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/ - echo -e "\n======Test libhv/tinyhttpd send file (no FileCache)======" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/ - echo -e "\n======Test nginx 404 Not Found===========================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:80/404 - echo -e "\n======Test libhv/httpd 404 Not Found=====================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/404 - echo -e "\n======Test libhv/tinyhttpd 404 Not found=================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/404 - echo -e "\n======Test libhv/httpd /ping=============================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/ping - echo -e "\n======Test libhv/tinyhttpd /ping=========================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/ping + sleep 1 + ps aux | grep -E 'nginx|httpd' | grep -v grep + expect() { + code=$(curl -s -o /dev/null -w '%{http_code}' "$2") + echo "$2 -> $code (expect $1)" + [ "$code" = "$1" ] + } + for port in 80 8080 9090; do + expect 200 http://127.0.0.1:$port/ + expect 404 http://127.0.0.1:$port/404 + done + expect 200 http://127.0.0.1:8080/ping + expect 200 http://127.0.0.1:9090/ping + run_wrk() { + name=$1; shift + echo -e "\n====== $name" + bin/wrk -c 100 -t 2 -d 10s "$@" 2>&1 | tee "wrk_$name.log" + } + run_wrk nginx_file http://127.0.0.1:80/ + run_wrk httpd_file http://127.0.0.1:8080/ + run_wrk tinyhttpd_file http://127.0.0.1:9090/ + run_wrk nginx_404 http://127.0.0.1:80/404 + run_wrk httpd_404 http://127.0.0.1:8080/404 + run_wrk tinyhttpd_404 http://127.0.0.1:9090/404 + run_wrk httpd_ping http://127.0.0.1:8080/ping + run_wrk tinyhttpd_ping http://127.0.0.1:9090/ping + + - name: check results + run: | + bash .github/scripts/benchmark_check.sh . From 50fe4d782a8f2562a4ceada834dfc631d407e2a2 Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 11:36:35 +0800 Subject: [PATCH 3/8] fix(examples): disable Nagle on tinyhttpd connections A file response goes out as two writes, the head and then the body. Now that the connection stays open, Nagle held the body until the client's delayed ACK, adding ~40 ms to every file request (2.4k req/s in the benchmark run, latency p50 41 ms). Co-authored-by: Cursor --- examples/tinyhttpd.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/tinyhttpd.c b/examples/tinyhttpd.c index 51f6064fe..b20e4af0f 100644 --- a/examples/tinyhttpd.c +++ b/examples/tinyhttpd.c @@ -413,6 +413,9 @@ static void new_conn_event(hevent_t* ev) { SOCKADDR_STR(hio_peeraddr(io), peeraddrstr)); */ + // A file response is written as a head and then the body; without this, + // Nagle holds the body until the peer's delayed ACK on keep-alive connections. + tcp_nodelay(hio_fd(io), 1); hio_setcb_close(io, on_close); hio_setcb_read(io, on_recv); hio_set_keepalive_timeout(io, HTTP_KEEPALIVE_TIMEOUT); From a68242736291b0dc52030e41e24be8f7f468d2c1 Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 12:18:39 +0800 Subject: [PATCH 4/8] ci(benchmark): keep only the ratio checks, compared at one decimal Drop the unanswered-request and per-echo-server checks, compare ratios at one decimal place, and make all six ratios hard limits: libhv echo / median >= 1.0 tcp proxy / direct (iperf) >= 0.4 httpd file (with FileCache) / nginx file >= 1.2 tinyhttpd file (no FileCache) / nginx file >= 0.8 httpd 404 / nginx 404 >= 0.8 tinyhttpd 404 / nginx 404 >= 0.8 wrk parse errors still fail the job but are only printed when found. Co-authored-by: Cursor --- .github/scripts/benchmark_check.sh | 82 ++++++++---------------------- 1 file changed, 21 insertions(+), 61 deletions(-) diff --git a/.github/scripts/benchmark_check.sh b/.github/scripts/benchmark_check.sh index 3738ed62f..7a9e36acf 100755 --- a/.github/scripts/benchmark_check.sh +++ b/.github/scripts/benchmark_check.sh @@ -4,34 +4,28 @@ # # GitHub-hosted runners differ from run to run, so absolute numbers swing by # up to 2x. Performance is therefore judged by ratios against peers measured in -# the same run (nginx, other echo servers), which stay within a narrow band. -# Floors are ~85% of the lowest ratio seen across 27 recent runs. +# the same run (nginx, the other echo servers, a direct iperf connection), +# which stay within a narrow band. Ratios are compared at one decimal place. # # usage: benchmark_check.sh [result_dir] # result_dir holds echo.log, iperf_direct.csv, iperf_proxy.csv, wrk_.log cd "${1:-.}" || exit 1 -# wrk runs with this many connections; at most one request per connection may -# be in flight when the test stops. -MAX_UNANSWERED=110 - -FLOOR_ECHO=0.84 # libhv echo / median of the other echo servers -FLOOR_HTTPD_FILE=1.13 # httpd send file / nginx send file -FLOOR_HTTPD_404=0.81 # httpd 404 / nginx 404 -FLOOR_TINY_404=0.87 # tinyhttpd 404 / nginx 404 -FLOOR_HTTPD_PING=0.89 # httpd /ping / nginx 404 -FLOOR_TINY_PING=0.92 # tinyhttpd /ping / nginx 404 -FLOOR_TCP_PROXY=0.38 # tcp_proxy_server / direct (warn only until more data) +FLOOR_ECHO=1.0 # libhv echo / median of the other echo servers +FLOOR_TCP_PROXY=0.4 # tcp_proxy_server / direct +FLOOR_HTTPD_FILE=1.2 # httpd send file (with FileCache) / nginx send file +FLOOR_TINY_FILE=0.8 # tinyhttpd send file (no FileCache) / nginx send file +FLOOR_HTTPD_404=0.8 # httpd 404 / nginx 404 +FLOOR_TINY_404=0.8 # tinyhttpd 404 / nginx 404 failed=0 summary() { [ -n "$GITHUB_STEP_SUMMARY" ] && echo "$1" >> "$GITHUB_STEP_SUMMARY"; } summary "| result | check | value | limit |" summary "|---|---|---|---|" -# wrk prints "N requests, M OK, ..." and "Requests/sec: R" -rps() { awk '/^Requests\/sec:/ {print $2}' "wrk_$1.log" 2>/dev/null; } -unanswered() { awk '/ requests, [0-9]+ OK/ {print $1 - $3}' "wrk_$1.log" 2>/dev/null; } +# wrk prints "Requests/sec: R" +rps() { awk '/^Requests\/sec:/ {print $2}' "wrk_$1.log" 2>/dev/null; } # echo-servers/benchmark.sh prints " running on port P", then for each # port "====P====" followed by "throughput = X MB/s" @@ -54,66 +48,32 @@ record() { [ "$1" = FAIL ] && failed=$((failed + 1)) } -# name numerator denominator floor [warn|info] +# name numerator denominator floor check_ratio() { - local name=$1 num=$2 den=$3 floor=$4 mode=$5 - if [ -z "$num" ] || [ -z "$den" ]; then + local name=$1 num=$2 den=$3 floor=$4 ratio + ratio=$(awk -v n="$num" -v d="$den" 'BEGIN {if (n != "" && d + 0 > 0) printf "%.1f", n / d}') + if [ -z "$ratio" ]; then record FAIL "$name" "missing" ">= $floor" - return - fi - local ratio ok - ratio=$(awk -v n="$num" -v d="$den" 'BEGIN {if (d + 0 > 0) printf "%.2f", n / d}') - ok=$(awk -v r="$ratio" -v f="$floor" 'BEGIN {print (r != "" && r + 0 >= f) ? 1 : 0}') - if [ "$mode" = info ]; then - record INFO "$name" "$ratio" "-" - elif [ "$ok" = 1 ]; then + elif awk -v r="$ratio" -v f="$floor" 'BEGIN {exit !(r + 0 >= f + 0)}'; then record PASS "$name" "$ratio" ">= $floor" - elif [ "$mode" = warn ]; then - record WARN "$name" "$ratio" ">= $floor" else record FAIL "$name" "$ratio" ">= $floor" fi } -# name value max -check_max() { - if [ -z "$2" ]; then - record FAIL "$1" "missing" "<= $3" - elif [ "$2" -le "$3" ]; then - record PASS "$1" "$2" "<= $3" - else - record FAIL "$1" "$2" "<= $3" - fi -} - -echo "== correctness ==" -for s in libevent libev libuv libhv asio poco; do - tp=$(echo_tp $s) - if [ -n "$tp" ] && [ "$tp" -gt 0 ]; then - record PASS "echo server $s" "${tp}MB/s" "> 0" - else - record FAIL "echo server $s" "${tp:-missing}" "> 0" - fi -done -for t in httpd_file tinyhttpd_file httpd_ping tinyhttpd_ping; do - check_max "$t unanswered requests" "$(unanswered $t)" $MAX_UNANSWERED -done for f in wrk_*.log; do if grep -q "parse error" "$f" 2>/dev/null; then record FAIL "${f%.log}: http parse error" "found" "none" fi done -echo "== performance (same-run ratios) ==" others=$(for s in libevent libev libuv asio poco; do echo_tp $s; done | median) -check_ratio "libhv echo / median(other echo servers)" "$(echo_tp libhv)" "$others" $FLOOR_ECHO -check_ratio "httpd file / nginx file" "$(rps httpd_file)" "$(rps nginx_file)" $FLOOR_HTTPD_FILE -check_ratio "httpd 404 / nginx 404" "$(rps httpd_404)" "$(rps nginx_404)" $FLOOR_HTTPD_404 -check_ratio "tinyhttpd 404 / nginx 404" "$(rps tinyhttpd_404)" "$(rps nginx_404)" $FLOOR_TINY_404 -check_ratio "httpd /ping / nginx 404" "$(rps httpd_ping)" "$(rps nginx_404)" $FLOOR_HTTPD_PING -check_ratio "tinyhttpd /ping / nginx 404" "$(rps tinyhttpd_ping)" "$(rps nginx_404)" $FLOOR_TINY_PING -check_ratio "tcp proxy / direct (iperf)" "$(iperf_bps proxy)" "$(iperf_bps direct)" $FLOOR_TCP_PROXY warn -check_ratio "tinyhttpd file / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)" 0 info +check_ratio "libhv echo / median" "$(echo_tp libhv)" "$others" $FLOOR_ECHO +check_ratio "tcp proxy / direct (iperf)" "$(iperf_bps proxy)" "$(iperf_bps direct)" $FLOOR_TCP_PROXY +check_ratio "httpd file (with FileCache) / nginx file" "$(rps httpd_file)" "$(rps nginx_file)" $FLOOR_HTTPD_FILE +check_ratio "tinyhttpd file (no FileCache) / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)" $FLOOR_TINY_FILE +check_ratio "httpd 404 / nginx 404" "$(rps httpd_404)" "$(rps nginx_404)" $FLOOR_HTTPD_404 +check_ratio "tinyhttpd 404 / nginx 404" "$(rps tinyhttpd_404)" "$(rps nginx_404)" $FLOOR_TINY_404 echo if [ $failed -gt 0 ]; then From 41727b843a82156f10531415e126822dc3c4ba1f Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 12:27:09 +0800 Subject: [PATCH 5/8] ci(benchmark): compare /ping against nginx Serve /ping from nginx on port 8000 (return 200 "pong") and check httpd ping / nginx ping >= 1.0 tinyhttpd ping / nginx ping >= 1.0 Co-authored-by: Cursor --- .github/scripts/benchmark_check.sh | 4 ++++ .github/workflows/benchmark.yml | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/.github/scripts/benchmark_check.sh b/.github/scripts/benchmark_check.sh index 7a9e36acf..32b032876 100755 --- a/.github/scripts/benchmark_check.sh +++ b/.github/scripts/benchmark_check.sh @@ -18,6 +18,8 @@ FLOOR_HTTPD_FILE=1.2 # httpd send file (with FileCache) / nginx send file FLOOR_TINY_FILE=0.8 # tinyhttpd send file (no FileCache) / nginx send file FLOOR_HTTPD_404=0.8 # httpd 404 / nginx 404 FLOOR_TINY_404=0.8 # tinyhttpd 404 / nginx 404 +FLOOR_HTTPD_PING=1.0 # httpd /ping / nginx /ping +FLOOR_TINY_PING=1.0 # tinyhttpd /ping / nginx /ping failed=0 summary() { [ -n "$GITHUB_STEP_SUMMARY" ] && echo "$1" >> "$GITHUB_STEP_SUMMARY"; } @@ -74,6 +76,8 @@ check_ratio "httpd file (with FileCache) / nginx file" "$(rps httpd_file)" check_ratio "tinyhttpd file (no FileCache) / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)" $FLOOR_TINY_FILE check_ratio "httpd 404 / nginx 404" "$(rps httpd_404)" "$(rps nginx_404)" $FLOOR_HTTPD_404 check_ratio "tinyhttpd 404 / nginx 404" "$(rps tinyhttpd_404)" "$(rps nginx_404)" $FLOOR_TINY_404 +check_ratio "httpd ping / nginx ping" "$(rps httpd_ping)" "$(rps nginx_ping)" $FLOOR_HTTPD_PING +check_ratio "tinyhttpd ping / nginx ping" "$(rps tinyhttpd_ping)" "$(rps nginx_ping)" $FLOOR_TINY_PING echo if [ $failed -gt 0 ]; then diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 50eb5cf15..48043ecbc 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -54,6 +54,15 @@ jobs: - name: webbench run: | + sudo tee /etc/nginx/conf.d/ping.conf > /dev/null <<'EOF' + server { + listen 8000; + location = /ping { + default_type text/plain; + return 200 "pong"; + } + } + EOF sudo nginx -c /etc/nginx/nginx.conf sudo cp html/index.html index.html sudo cp html/index.html /var/www/html/index.html @@ -70,6 +79,7 @@ jobs: expect 200 http://127.0.0.1:$port/ expect 404 http://127.0.0.1:$port/404 done + expect 200 http://127.0.0.1:8000/ping expect 200 http://127.0.0.1:8080/ping expect 200 http://127.0.0.1:9090/ping run_wrk() { @@ -83,6 +93,7 @@ jobs: run_wrk nginx_404 http://127.0.0.1:80/404 run_wrk httpd_404 http://127.0.0.1:8080/404 run_wrk tinyhttpd_404 http://127.0.0.1:9090/404 + run_wrk nginx_ping http://127.0.0.1:8000/ping run_wrk httpd_ping http://127.0.0.1:8080/ping run_wrk tinyhttpd_ping http://127.0.0.1:9090/ping From 7d377e61f265a271e5ecf3f553e29bac9ca10ab8 Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 13:25:33 +0800 Subject: [PATCH 6/8] ci(benchmark): serve nginx /ping from the default site on port 80 Write the default site explicitly (same root, index and try_files as the Ubuntu default) with a /ping location, instead of a separate server on port 8000, so every nginx test hits the same server block. Co-authored-by: Cursor --- .github/workflows/benchmark.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 48043ecbc..b1d904b32 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -54,9 +54,16 @@ jobs: - name: webbench run: | - sudo tee /etc/nginx/conf.d/ping.conf > /dev/null <<'EOF' + sudo tee /etc/nginx/sites-available/default > /dev/null <<'EOF' server { - listen 8000; + listen 80 default_server; + listen [::]:80 default_server; + root /var/www/html; + index index.html; + server_name _; + location / { + try_files $uri $uri/ =404; + } location = /ping { default_type text/plain; return 200 "pong"; @@ -78,10 +85,8 @@ jobs: for port in 80 8080 9090; do expect 200 http://127.0.0.1:$port/ expect 404 http://127.0.0.1:$port/404 + expect 200 http://127.0.0.1:$port/ping done - expect 200 http://127.0.0.1:8000/ping - expect 200 http://127.0.0.1:8080/ping - expect 200 http://127.0.0.1:9090/ping run_wrk() { name=$1; shift echo -e "\n====== $name" @@ -93,7 +98,7 @@ jobs: run_wrk nginx_404 http://127.0.0.1:80/404 run_wrk httpd_404 http://127.0.0.1:8080/404 run_wrk tinyhttpd_404 http://127.0.0.1:9090/404 - run_wrk nginx_ping http://127.0.0.1:8000/ping + run_wrk nginx_ping http://127.0.0.1:80/ping run_wrk httpd_ping http://127.0.0.1:8080/ping run_wrk tinyhttpd_ping http://127.0.0.1:9090/ping From 0e52bac6ef242d985347ef8f89e5fab03e4bb6be Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 13:30:40 +0800 Subject: [PATCH 7/8] ci(benchmark): show both sides of each ratio in the summary Add an "a / b" column (MB/s, Gbit/s or req/s) next to the ratio, in the step output and the job summary. Co-authored-by: Cursor --- .github/scripts/benchmark_check.sh | 41 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.github/scripts/benchmark_check.sh b/.github/scripts/benchmark_check.sh index 32b032876..1cacbaf2d 100755 --- a/.github/scripts/benchmark_check.sh +++ b/.github/scripts/benchmark_check.sh @@ -23,8 +23,8 @@ FLOOR_TINY_PING=1.0 # tinyhttpd /ping / nginx /ping failed=0 summary() { [ -n "$GITHUB_STEP_SUMMARY" ] && echo "$1" >> "$GITHUB_STEP_SUMMARY"; } -summary "| result | check | value | limit |" -summary "|---|---|---|---|" +summary "| result | check | a / b | ratio | limit |" +summary "|---|---|---|---|---|" # wrk prints "Requests/sec: R" rps() { awk '/^Requests\/sec:/ {print $2}' "wrk_$1.log" 2>/dev/null; } @@ -39,45 +39,46 @@ echo_tp() { } # iperf -y C: the last field is bits per second -iperf_bps() { awk -F, 'NF > 8 {print $NF}' "iperf_$1.csv" 2>/dev/null | tail -1; } +iperf_gbps() { awk -F, 'NF > 8 {printf "%.2f\n", $NF / 1e9}' "iperf_$1.csv" 2>/dev/null | tail -1; } median() { sort -n | awk 'NF {a[++n] = $1} END {if (n) print (n % 2) ? a[(n + 1) / 2] : (a[n / 2] + a[n / 2 + 1]) / 2}'; } -# result name value limit +# result name a/b ratio limit record() { - printf "%-5s %-44s %-8s %s\n" "$1" "$2" "$3" "$4" - summary "| $1 | $2 | $3 | $4 |" + printf "%-5s %-44s %-26s %-7s %s\n" "$1" "$2" "$3" "$4" "$5" + summary "| $1 | $2 | $3 | $4 | $5 |" [ "$1" = FAIL ] && failed=$((failed + 1)) } -# name numerator denominator floor +# name numerator denominator floor unit check_ratio() { - local name=$1 num=$2 den=$3 floor=$4 ratio + local name=$1 num=$2 den=$3 floor=$4 unit=$5 values ratio + values="${num:-?} / ${den:-?} $unit" ratio=$(awk -v n="$num" -v d="$den" 'BEGIN {if (n != "" && d + 0 > 0) printf "%.1f", n / d}') if [ -z "$ratio" ]; then - record FAIL "$name" "missing" ">= $floor" + record FAIL "$name" "$values" "missing" ">= $floor" elif awk -v r="$ratio" -v f="$floor" 'BEGIN {exit !(r + 0 >= f + 0)}'; then - record PASS "$name" "$ratio" ">= $floor" + record PASS "$name" "$values" "$ratio" ">= $floor" else - record FAIL "$name" "$ratio" ">= $floor" + record FAIL "$name" "$values" "$ratio" ">= $floor" fi } for f in wrk_*.log; do if grep -q "parse error" "$f" 2>/dev/null; then - record FAIL "${f%.log}: http parse error" "found" "none" + record FAIL "${f%.log}: http parse error" "found" "-" "none" fi done others=$(for s in libevent libev libuv asio poco; do echo_tp $s; done | median) -check_ratio "libhv echo / median" "$(echo_tp libhv)" "$others" $FLOOR_ECHO -check_ratio "tcp proxy / direct (iperf)" "$(iperf_bps proxy)" "$(iperf_bps direct)" $FLOOR_TCP_PROXY -check_ratio "httpd file (with FileCache) / nginx file" "$(rps httpd_file)" "$(rps nginx_file)" $FLOOR_HTTPD_FILE -check_ratio "tinyhttpd file (no FileCache) / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)" $FLOOR_TINY_FILE -check_ratio "httpd 404 / nginx 404" "$(rps httpd_404)" "$(rps nginx_404)" $FLOOR_HTTPD_404 -check_ratio "tinyhttpd 404 / nginx 404" "$(rps tinyhttpd_404)" "$(rps nginx_404)" $FLOOR_TINY_404 -check_ratio "httpd ping / nginx ping" "$(rps httpd_ping)" "$(rps nginx_ping)" $FLOOR_HTTPD_PING -check_ratio "tinyhttpd ping / nginx ping" "$(rps tinyhttpd_ping)" "$(rps nginx_ping)" $FLOOR_TINY_PING +check_ratio "libhv echo / median" "$(echo_tp libhv)" "$others" $FLOOR_ECHO MB/s +check_ratio "tcp proxy / direct (iperf)" "$(iperf_gbps proxy)" "$(iperf_gbps direct)" $FLOOR_TCP_PROXY Gbit/s +check_ratio "httpd file (with FileCache) / nginx file" "$(rps httpd_file)" "$(rps nginx_file)" $FLOOR_HTTPD_FILE req/s +check_ratio "tinyhttpd file (no FileCache) / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)" $FLOOR_TINY_FILE req/s +check_ratio "httpd 404 / nginx 404" "$(rps httpd_404)" "$(rps nginx_404)" $FLOOR_HTTPD_404 req/s +check_ratio "tinyhttpd 404 / nginx 404" "$(rps tinyhttpd_404)" "$(rps nginx_404)" $FLOOR_TINY_404 req/s +check_ratio "httpd ping / nginx ping" "$(rps httpd_ping)" "$(rps nginx_ping)" $FLOOR_HTTPD_PING req/s +check_ratio "tinyhttpd ping / nginx ping" "$(rps tinyhttpd_ping)" "$(rps nginx_ping)" $FLOOR_TINY_PING req/s echo if [ $failed -gt 0 ]; then From 9d365026059fbcde9536b4c3450ffcbe61bef9e6 Mon Sep 17 00:00:00 2001 From: ithewei Date: Tue, 13 Oct 2026 13:34:16 +0800 Subject: [PATCH 8/8] ci(benchmark): lower httpd ping / nginx ping limit to 0.8 httpd /ping runs at 0.95-0.97 of nginx's return 200 "pong", so 1.0 failed intermittently. Co-authored-by: Cursor --- .github/scripts/benchmark_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/benchmark_check.sh b/.github/scripts/benchmark_check.sh index 1cacbaf2d..dbd2ca72a 100755 --- a/.github/scripts/benchmark_check.sh +++ b/.github/scripts/benchmark_check.sh @@ -18,7 +18,7 @@ FLOOR_HTTPD_FILE=1.2 # httpd send file (with FileCache) / nginx send file FLOOR_TINY_FILE=0.8 # tinyhttpd send file (no FileCache) / nginx send file FLOOR_HTTPD_404=0.8 # httpd 404 / nginx 404 FLOOR_TINY_404=0.8 # tinyhttpd 404 / nginx 404 -FLOOR_HTTPD_PING=1.0 # httpd /ping / nginx /ping +FLOOR_HTTPD_PING=0.8 # httpd /ping / nginx /ping FLOOR_TINY_PING=1.0 # tinyhttpd /ping / nginx /ping failed=0