ci(benchmark): validate benchmark results

.github/scripts/benchmark_check.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+#
+# Validate the results written by .github/workflows/benchmark.yml.
+#
+# GitHub-hosted runners differ from run to run, so absolute numbers swing by
+# up to 2x. Performance is therefore judged by ratios against peers measured in
+# the same run (nginx, the other echo servers, a direct iperf connection),
+# which stay within a narrow band. Ratios are compared at one decimal place.
+#
+# usage: benchmark_check.sh [result_dir]
+#   result_dir holds echo.log, iperf_direct.csv, iperf_proxy.csv, wrk_<name>.log
+
+cd "${1:-.}" || exit 1
+
+FLOOR_ECHO=1.0          # libhv echo / median of the other echo servers
+FLOOR_TCP_PROXY=0.4     # tcp_proxy_server / direct
+FLOOR_HTTPD_FILE=1.2    # httpd send file (with FileCache) / nginx send file
+FLOOR_TINY_FILE=0.8     # tinyhttpd send file (no FileCache) / nginx send file
+FLOOR_HTTPD_404=0.8     # httpd 404 / nginx 404
+FLOOR_TINY_404=0.8      # tinyhttpd 404 / nginx 404
+FLOOR_HTTPD_PING=0.8    # httpd /ping / nginx /ping
+FLOOR_TINY_PING=1.0     # tinyhttpd /ping / nginx /ping
+
+failed=0
+summary() { [ -n "$GITHUB_STEP_SUMMARY" ] && echo "$1" >> "$GITHUB_STEP_SUMMARY"; }
+summary "| result | check | a / b | ratio | limit |"
+summary "|---|---|---|---|---|"
+
+# wrk prints "Requests/sec: R"
+rps() { awk '/^Requests\/sec:/ {print $2}' "wrk_$1.log" 2>/dev/null; }
+
+# echo-servers/benchmark.sh prints "<name> running on port P", then for each
+# port "====P====" followed by "throughput = X MB/s"
+echo_tp() {
+    awk -v want="$1" '
+        / running on port / { name[$5] = $1 }
+        /^=+[0-9]+=+$/      { gsub(/=/, ""); port = $0 }
+        /^throughput = /    { if (name[port] == want) print $3 }' echo.log 2>/dev/null
+}
+
+# iperf -y C: the last field is bits per second
+iperf_gbps() { awk -F, 'NF > 8 {printf "%.2f\n", $NF / 1e9}' "iperf_$1.csv" 2>/dev/null | tail -1; }
+
+median() { sort -n | awk 'NF {a[++n] = $1} END {if (n) print (n % 2) ? a[(n + 1) / 2] : (a[n / 2] + a[n / 2 + 1]) / 2}'; }
+
+# result name a/b ratio limit
+record() {
+    printf "%-5s %-44s %-26s %-7s %s\n" "$1" "$2" "$3" "$4" "$5"
+    summary "| $1 | $2 | $3 | $4 | $5 |"
+    [ "$1" = FAIL ] && failed=$((failed + 1))
+}
+
+# name numerator denominator floor unit
+check_ratio() {
+    local name=$1 num=$2 den=$3 floor=$4 unit=$5 values ratio
+    values="${num:-?} / ${den:-?} $unit"
+    ratio=$(awk -v n="$num" -v d="$den" 'BEGIN {if (n != "" && d + 0 > 0) printf "%.1f", n / d}')
+    if [ -z "$ratio" ]; then
+        record FAIL "$name" "$values" "missing" ">= $floor"
+    elif awk -v r="$ratio" -v f="$floor" 'BEGIN {exit !(r + 0 >= f + 0)}'; then
+        record PASS "$name" "$values" "$ratio" ">= $floor"
+    else
+        record FAIL "$name" "$values" "$ratio" ">= $floor"
+    fi
+}
+
+for f in wrk_*.log; do
+    if grep -q "parse error" "$f" 2>/dev/null; then
+        record FAIL "${f%.log}: http parse error" "found" "-" "none"
+    fi
+done
+
+others=$(for s in libevent libev libuv asio poco; do echo_tp $s; done | median)
+check_ratio "libhv echo / median"                        "$(echo_tp libhv)"      "$others"              $FLOOR_ECHO       MB/s
+check_ratio "tcp proxy / direct (iperf)"                 "$(iperf_gbps proxy)"   "$(iperf_gbps direct)" $FLOOR_TCP_PROXY  Gbit/s
+check_ratio "httpd file (with FileCache) / nginx file"   "$(rps httpd_file)"     "$(rps nginx_file)"    $FLOOR_HTTPD_FILE req/s
+check_ratio "tinyhttpd file (no FileCache) / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)"    $FLOOR_TINY_FILE  req/s
+check_ratio "httpd 404 / nginx 404"                      "$(rps httpd_404)"      "$(rps nginx_404)"     $FLOOR_HTTPD_404  req/s
+check_ratio "tinyhttpd 404 / nginx 404"                  "$(rps tinyhttpd_404)"  "$(rps nginx_404)"     $FLOOR_TINY_404   req/s
+check_ratio "httpd ping / nginx ping"                    "$(rps httpd_ping)"     "$(rps nginx_ping)"    $FLOOR_HTTPD_PING req/s
+check_ratio "tinyhttpd ping / nginx ping"                "$(rps tinyhttpd_ping)" "$(rps nginx_ping)"    $FLOOR_TINY_PING  req/s
+
+echo
+if [ $failed -gt 0 ]; then
+    echo "$failed check(s) failed"
+    exit 1
+fi
+echo "all checks passed"

.github/workflows/benchmark.yml

@@ -9,10 +9,15 @@ on:
     paths-ignore:
       - '**.md'
 
+defaults:
+  run:
+    shell: bash -eo pipefail {0}
+
 jobs:
   benchmark:
     name: benchmark
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v7
 
@@ -34,37 +39,69 @@ jobs:
 
       - name: benchmark echo-servers
         run: |
-          bash echo-servers/benchmark.sh
+          bash echo-servers/benchmark.sh | tee echo.log
 
       - name: benchmark tcp_proxy_server
         run: |
           iperf -s -p 5001 > /dev/null &
           bin/tcp_proxy_server 1212 127.0.0.1:5001 &
-          iperf -c 127.0.0.1 -p 5001 -l 8K
-          iperf -c 127.0.0.1 -p 1212 -l 8K
+          for i in $(seq 50); do
+            ss -ltn | grep -q ':5001 ' && ss -ltn | grep -q ':1212 ' && break
+            sleep 0.1
+          done
+          iperf -c 127.0.0.1 -p 5001 -l 8K -y C | tee iperf_direct.csv
+          iperf -c 127.0.0.1 -p 1212 -l 8K -y C | tee iperf_proxy.csv
 
       - name: webbench
         run: |
+          sudo tee /etc/nginx/sites-available/default > /dev/null <<'EOF'
+          server {
+              listen 80 default_server;
+              listen [::]:80 default_server;
+              root /var/www/html;
+              index index.html;
+              server_name _;
+              location / {
+                  try_files $uri $uri/ =404;
+              }
+              location = /ping {
+                  default_type text/plain;
+                  return 200 "pong";
+              }
+          }
+          EOF
           sudo nginx -c /etc/nginx/nginx.conf
           sudo cp html/index.html index.html
           sudo cp html/index.html /var/www/html/index.html
           bin/httpd -c etc/httpd.conf -d
-          ps aux | grep nginx
-          ps aux | grep httpd
           bin/tinyhttpd 9090 &
-          echo -e "\n======Test nginx send file==============================="
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:80/
-          echo -e "\n======Test libhv/httpd send file (with FileCache)========"
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/
-          echo -e "\n======Test libhv/tinyhttpd send file (no FileCache)======"
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/
-          echo -e "\n======Test nginx 404 Not Found==========================="
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:80/404
-          echo -e "\n======Test libhv/httpd 404 Not Found====================="
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/404
-          echo -e "\n======Test libhv/tinyhttpd 404 Not found================="
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/404
-          echo -e "\n======Test libhv/httpd /ping============================="
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/ping
-          echo -e "\n======Test libhv/tinyhttpd /ping========================="
-          bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/ping
+          sleep 1
+          ps aux | grep -E 'nginx|httpd' | grep -v grep
+          expect() {
+            code=$(curl -s -o /dev/null -w '%{http_code}' "$2")
+            echo "$2 -> $code (expect $1)"
+            [ "$code" = "$1" ]
+          }
+          for port in 80 8080 9090; do
+            expect 200 http://127.0.0.1:$port/
+            expect 404 http://127.0.0.1:$port/404
+            expect 200 http://127.0.0.1:$port/ping
+          done
+          run_wrk() {
+            name=$1; shift
+            echo -e "\n====== $name"
+            bin/wrk -c 100 -t 2 -d 10s "$@" 2>&1 | tee "wrk_$name.log"
+          }
+          run_wrk nginx_file     http://127.0.0.1:80/
+          run_wrk httpd_file     http://127.0.0.1:8080/
+          run_wrk tinyhttpd_file http://127.0.0.1:9090/
+          run_wrk nginx_404      http://127.0.0.1:80/404
+          run_wrk httpd_404      http://127.0.0.1:8080/404
+          run_wrk tinyhttpd_404  http://127.0.0.1:9090/404
+          run_wrk nginx_ping     http://127.0.0.1:80/ping
+          run_wrk httpd_ping     http://127.0.0.1:8080/ping
+          run_wrk tinyhttpd_ping http://127.0.0.1:9090/ping
+
+      - name: check results
+        run: |
+          bash .github/scripts/benchmark_check.sh .

examples/tinyhttpd.c

@@ -155,6 +155,20 @@ static int http_reply(http_conn_t* conn,
     return nwrite < 0 ? nwrite : msglen;
 }
 
+static void http_request_done(http_conn_t* conn) {
+    if (conn->request.keepalive) {
+        // Connection: keep-alive\r\n
+        // reset and receive next request
+        memset(&conn->request,  0, sizeof(http_msg_t));
+        memset(&conn->response, 0, sizeof(http_msg_t));
+        conn->state = s_first_line;
+        hio_readline(conn->io);
+    } else {
+        // Connection: close\r\n
+        hio_close(conn->io);
+    }
+}
+
 static void http_send_file(http_conn_t* conn) {
     if (!conn || !conn->fp) return;
     // alloc filebuf
@@ -167,8 +181,16 @@ static void http_send_file(http_conn_t* conn) {
     // read file
     int nread = fread(filebuf, 1, filebuflen, conn->fp);
     if (nread <= 0) {
-        // eof or error
-        hio_close(conn->io);
+        int error = ferror(conn->fp);
+        fclose(conn->fp);
+        conn->fp = NULL;
+        hio_setcb_write(conn->io, NULL);
+        if (error) {
+            hio_close(conn->io);
+        } else {
+            // eof: the whole file has been sent
+            http_request_done(conn);
+        }
         return;
     }
     // send file
@@ -366,16 +388,10 @@ static void on_recv(hio_t* io, void* buf, int readbytes) {
         // received complete request
         on_request(conn);
         if (hio_is_closed(io)) return;
-        if (req->keepalive) {
-            // Connection: keep-alive\r\n
-            // reset and receive next request
-            memset(&conn->request,  0, sizeof(http_msg_t));
-            memset(&conn->response, 0, sizeof(http_msg_t));
-            conn->state = s_first_line;
-            hio_readline(io);
-        } else {
-            // Connection: close\r\n
-            hio_close(io);
+        // A file response finishes the request once the whole file is sent,
+        // which may already have happened inside on_request.
+        if (conn->state == s_end && conn->fp == NULL) {
+            http_request_done(conn);
         }
         break;
     default: break;
@@ -397,6 +413,9 @@ static void new_conn_event(hevent_t* ev) {
             SOCKADDR_STR(hio_peeraddr(io), peeraddrstr));
     */
 
+    // A file response is written as a head and then the body; without this,
+    // Nagle holds the body until the peer's delayed ACK on keep-alive connections.
+    tcp_nodelay(hio_fd(io), 1);
     hio_setcb_close(io, on_close);
     hio_setcb_read(io, on_recv);
     hio_set_keepalive_timeout(io, HTTP_KEEPALIVE_TIMEOUT);