diff --git a/.github/scripts/benchmark_check.sh b/.github/scripts/benchmark_check.sh new file mode 100755 index 000000000..dbd2ca72a --- /dev/null +++ b/.github/scripts/benchmark_check.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Validate the results written by .github/workflows/benchmark.yml. +# +# GitHub-hosted runners differ from run to run, so absolute numbers swing by +# up to 2x. Performance is therefore judged by ratios against peers measured in +# the same run (nginx, the other echo servers, a direct iperf connection), +# which stay within a narrow band. Ratios are compared at one decimal place. +# +# usage: benchmark_check.sh [result_dir] +# result_dir holds echo.log, iperf_direct.csv, iperf_proxy.csv, wrk_.log + +cd "${1:-.}" || exit 1 + +FLOOR_ECHO=1.0 # libhv echo / median of the other echo servers +FLOOR_TCP_PROXY=0.4 # tcp_proxy_server / direct +FLOOR_HTTPD_FILE=1.2 # httpd send file (with FileCache) / nginx send file +FLOOR_TINY_FILE=0.8 # tinyhttpd send file (no FileCache) / nginx send file +FLOOR_HTTPD_404=0.8 # httpd 404 / nginx 404 +FLOOR_TINY_404=0.8 # tinyhttpd 404 / nginx 404 +FLOOR_HTTPD_PING=0.8 # httpd /ping / nginx /ping +FLOOR_TINY_PING=1.0 # tinyhttpd /ping / nginx /ping + +failed=0 +summary() { [ -n "$GITHUB_STEP_SUMMARY" ] && echo "$1" >> "$GITHUB_STEP_SUMMARY"; } +summary "| result | check | a / b | ratio | limit |" +summary "|---|---|---|---|---|" + +# wrk prints "Requests/sec: R" +rps() { awk '/^Requests\/sec:/ {print $2}' "wrk_$1.log" 2>/dev/null; } + +# echo-servers/benchmark.sh prints " running on port P", then for each +# port "====P====" followed by "throughput = X MB/s" +echo_tp() { + awk -v want="$1" ' + / running on port / { name[$5] = $1 } + /^=+[0-9]+=+$/ { gsub(/=/, ""); port = $0 } + /^throughput = / { if (name[port] == want) print $3 }' echo.log 2>/dev/null +} + +# iperf -y C: the last field is bits per second +iperf_gbps() { awk -F, 'NF > 8 {printf "%.2f\n", $NF / 1e9}' "iperf_$1.csv" 2>/dev/null | tail -1; } + +median() { sort -n | awk 'NF {a[++n] = $1} END {if (n) print (n % 2) ? a[(n + 1) / 2] : (a[n / 2] + a[n / 2 + 1]) / 2}'; } + +# result name a/b ratio limit +record() { + printf "%-5s %-44s %-26s %-7s %s\n" "$1" "$2" "$3" "$4" "$5" + summary "| $1 | $2 | $3 | $4 | $5 |" + [ "$1" = FAIL ] && failed=$((failed + 1)) +} + +# name numerator denominator floor unit +check_ratio() { + local name=$1 num=$2 den=$3 floor=$4 unit=$5 values ratio + values="${num:-?} / ${den:-?} $unit" + ratio=$(awk -v n="$num" -v d="$den" 'BEGIN {if (n != "" && d + 0 > 0) printf "%.1f", n / d}') + if [ -z "$ratio" ]; then + record FAIL "$name" "$values" "missing" ">= $floor" + elif awk -v r="$ratio" -v f="$floor" 'BEGIN {exit !(r + 0 >= f + 0)}'; then + record PASS "$name" "$values" "$ratio" ">= $floor" + else + record FAIL "$name" "$values" "$ratio" ">= $floor" + fi +} + +for f in wrk_*.log; do + if grep -q "parse error" "$f" 2>/dev/null; then + record FAIL "${f%.log}: http parse error" "found" "-" "none" + fi +done + +others=$(for s in libevent libev libuv asio poco; do echo_tp $s; done | median) +check_ratio "libhv echo / median" "$(echo_tp libhv)" "$others" $FLOOR_ECHO MB/s +check_ratio "tcp proxy / direct (iperf)" "$(iperf_gbps proxy)" "$(iperf_gbps direct)" $FLOOR_TCP_PROXY Gbit/s +check_ratio "httpd file (with FileCache) / nginx file" "$(rps httpd_file)" "$(rps nginx_file)" $FLOOR_HTTPD_FILE req/s +check_ratio "tinyhttpd file (no FileCache) / nginx file" "$(rps tinyhttpd_file)" "$(rps nginx_file)" $FLOOR_TINY_FILE req/s +check_ratio "httpd 404 / nginx 404" "$(rps httpd_404)" "$(rps nginx_404)" $FLOOR_HTTPD_404 req/s +check_ratio "tinyhttpd 404 / nginx 404" "$(rps tinyhttpd_404)" "$(rps nginx_404)" $FLOOR_TINY_404 req/s +check_ratio "httpd ping / nginx ping" "$(rps httpd_ping)" "$(rps nginx_ping)" $FLOOR_HTTPD_PING req/s +check_ratio "tinyhttpd ping / nginx ping" "$(rps tinyhttpd_ping)" "$(rps nginx_ping)" $FLOOR_TINY_PING req/s + +echo +if [ $failed -gt 0 ]; then + echo "$failed check(s) failed" + exit 1 +fi +echo "all checks passed" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 08d872217..b1d904b32 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,10 +9,15 @@ on: paths-ignore: - '**.md' +defaults: + run: + shell: bash -eo pipefail {0} + jobs: benchmark: name: benchmark runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v7 @@ -34,37 +39,69 @@ jobs: - name: benchmark echo-servers run: | - bash echo-servers/benchmark.sh + bash echo-servers/benchmark.sh | tee echo.log - name: benchmark tcp_proxy_server run: | iperf -s -p 5001 > /dev/null & bin/tcp_proxy_server 1212 127.0.0.1:5001 & - iperf -c 127.0.0.1 -p 5001 -l 8K - iperf -c 127.0.0.1 -p 1212 -l 8K + for i in $(seq 50); do + ss -ltn | grep -q ':5001 ' && ss -ltn | grep -q ':1212 ' && break + sleep 0.1 + done + iperf -c 127.0.0.1 -p 5001 -l 8K -y C | tee iperf_direct.csv + iperf -c 127.0.0.1 -p 1212 -l 8K -y C | tee iperf_proxy.csv - name: webbench run: | + sudo tee /etc/nginx/sites-available/default > /dev/null <<'EOF' + server { + listen 80 default_server; + listen [::]:80 default_server; + root /var/www/html; + index index.html; + server_name _; + location / { + try_files $uri $uri/ =404; + } + location = /ping { + default_type text/plain; + return 200 "pong"; + } + } + EOF sudo nginx -c /etc/nginx/nginx.conf sudo cp html/index.html index.html sudo cp html/index.html /var/www/html/index.html bin/httpd -c etc/httpd.conf -d - ps aux | grep nginx - ps aux | grep httpd bin/tinyhttpd 9090 & - echo -e "\n======Test nginx send file===============================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:80/ - echo -e "\n======Test libhv/httpd send file (with FileCache)========" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/ - echo -e "\n======Test libhv/tinyhttpd send file (no FileCache)======" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/ - echo -e "\n======Test nginx 404 Not Found===========================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:80/404 - echo -e "\n======Test libhv/httpd 404 Not Found=====================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/404 - echo -e "\n======Test libhv/tinyhttpd 404 Not found=================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/404 - echo -e "\n======Test libhv/httpd /ping=============================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:8080/ping - echo -e "\n======Test libhv/tinyhttpd /ping=========================" - bin/wrk -c 100 -t 2 -d 10s http://127.0.0.1:9090/ping + sleep 1 + ps aux | grep -E 'nginx|httpd' | grep -v grep + expect() { + code=$(curl -s -o /dev/null -w '%{http_code}' "$2") + echo "$2 -> $code (expect $1)" + [ "$code" = "$1" ] + } + for port in 80 8080 9090; do + expect 200 http://127.0.0.1:$port/ + expect 404 http://127.0.0.1:$port/404 + expect 200 http://127.0.0.1:$port/ping + done + run_wrk() { + name=$1; shift + echo -e "\n====== $name" + bin/wrk -c 100 -t 2 -d 10s "$@" 2>&1 | tee "wrk_$name.log" + } + run_wrk nginx_file http://127.0.0.1:80/ + run_wrk httpd_file http://127.0.0.1:8080/ + run_wrk tinyhttpd_file http://127.0.0.1:9090/ + run_wrk nginx_404 http://127.0.0.1:80/404 + run_wrk httpd_404 http://127.0.0.1:8080/404 + run_wrk tinyhttpd_404 http://127.0.0.1:9090/404 + run_wrk nginx_ping http://127.0.0.1:80/ping + run_wrk httpd_ping http://127.0.0.1:8080/ping + run_wrk tinyhttpd_ping http://127.0.0.1:9090/ping + + - name: check results + run: | + bash .github/scripts/benchmark_check.sh . diff --git a/examples/tinyhttpd.c b/examples/tinyhttpd.c index 5aa6693b6..b20e4af0f 100644 --- a/examples/tinyhttpd.c +++ b/examples/tinyhttpd.c @@ -155,6 +155,20 @@ static int http_reply(http_conn_t* conn, return nwrite < 0 ? nwrite : msglen; } +static void http_request_done(http_conn_t* conn) { + if (conn->request.keepalive) { + // Connection: keep-alive\r\n + // reset and receive next request + memset(&conn->request, 0, sizeof(http_msg_t)); + memset(&conn->response, 0, sizeof(http_msg_t)); + conn->state = s_first_line; + hio_readline(conn->io); + } else { + // Connection: close\r\n + hio_close(conn->io); + } +} + static void http_send_file(http_conn_t* conn) { if (!conn || !conn->fp) return; // alloc filebuf @@ -167,8 +181,16 @@ static void http_send_file(http_conn_t* conn) { // read file int nread = fread(filebuf, 1, filebuflen, conn->fp); if (nread <= 0) { - // eof or error - hio_close(conn->io); + int error = ferror(conn->fp); + fclose(conn->fp); + conn->fp = NULL; + hio_setcb_write(conn->io, NULL); + if (error) { + hio_close(conn->io); + } else { + // eof: the whole file has been sent + http_request_done(conn); + } return; } // send file @@ -366,16 +388,10 @@ static void on_recv(hio_t* io, void* buf, int readbytes) { // received complete request on_request(conn); if (hio_is_closed(io)) return; - if (req->keepalive) { - // Connection: keep-alive\r\n - // reset and receive next request - memset(&conn->request, 0, sizeof(http_msg_t)); - memset(&conn->response, 0, sizeof(http_msg_t)); - conn->state = s_first_line; - hio_readline(io); - } else { - // Connection: close\r\n - hio_close(io); + // A file response finishes the request once the whole file is sent, + // which may already have happened inside on_request. + if (conn->state == s_end && conn->fp == NULL) { + http_request_done(conn); } break; default: break; @@ -397,6 +413,9 @@ static void new_conn_event(hevent_t* ev) { SOCKADDR_STR(hio_peeraddr(io), peeraddrstr)); */ + // A file response is written as a head and then the body; without this, + // Nagle holds the body until the peer's delayed ACK on keep-alive connections. + tcp_nodelay(hio_fd(io), 1); hio_setcb_close(io, on_close); hio_setcb_read(io, on_recv); hio_set_keepalive_timeout(io, HTTP_KEEPALIVE_TIMEOUT);