#!/bin/bash
#
# Rough measurement of what the paivana reverse proxy costs, and of
# what it costs to be turned away by the paywall.
#
# Runs N parallel curl clients against a fixed-size static page for a
# fixed period, and reports completed requests, requests/s and MB/s.
# There are three things it can point them at:
#
#   direct    straight at the Rust upstream
#   proxy     through paivana-httpd in front of that same upstream
#   paywall   at paivana's own paywall page, with no upstream involved
#
# direct-vs-proxy is what the proxy costs, and is the number this
# started out to produce.  paywall is the other side of paivana: the
# 402 page an unpaid client is sent to, which paivana renders from its
# template once and then serves out of its own response cache, so it
# measures paivana answering by itself -- the cheapest thing it does,
# and the one an unauthenticated client can ask for without limit.
#
# Nothing is written to disk: the upstream's page comes from a buffer
# built once at its startup, the paywall page from paivana's cache, and
# every client discards the body.  The whole thing runs on loopback
# against one upstream process and one paivana process.
#
# Usage: benchmark.sh [-c CLIENTS] [-s SIZE] [-d SECONDS] [-m MODE]
#
#   -c N      parallel curl clients (default 8)
#   -s SIZE   page size in bytes; k/M suffixes accepted (default 64k).
#             Capped at 10M, which is the largest /large/ the upstream
#             will serve.  Does not apply to paywall mode, where the
#             page size is whatever the template renders to.
#   -d SECS   how long to run each mode for (default 10)
#   -m MODE   direct, proxy, paywall, both (= direct + proxy, the
#             default) or all
#   -b PORT   first of the three ports used (default 18600, or
#             $PAIVANA_BENCH_PORT_BASE)
#   -h        this help
#
# Environment: PAIVANA_HTTPD, SRCDIR, BUILDDIR and KEEP_TMP have the
# same meaning as in test_reverse_proxy.sh.  PAIVANA_BENCH_MERCHANT_URL
# and PAIVANA_BENCH_MERCHANT_TOKEN point paywall mode at a real
# merchant backend instead of the stub (see below).
#
# Exits 77 (meson: SKIP) when a binary or a port is missing, and 1 when
# any request failed -- a run with failures has not measured throughput,
# it has measured how fast something can be refused.
#
# ---------------------------------------------------------------------
# Four things about the setup shape the numbers, and are printed with
# them so nobody has to read this file to interpret a result:
#
#  * Every request gets a fresh TCP connection, in BOTH modes.  Not a
#    choice: upstream_rs answers one request per connection and closes
#    (see its client_loop()), so the direct arm could not keep-alive
#    even if asked.  paivana's client side would, though -- MHD strips
#    the upstream's hop-by-hop `Connection: close' and decides the
#    client connection's fate itself -- so the clients send
#    `Connection: close' to hold both arms to the same behaviour.
#    Otherwise the proxy would be credited with connection reuse the
#    thing it is being compared against cannot do.
#
#    The bias this leaves is a known one: a TCP setup and teardown is
#    charged to both arms, so it is a constant added to both sides of
#    the ratio, which pulls the ratio toward 1.  The proxy's real
#    relative cost is therefore a little worse than what is reported,
#    never better.  The same goes for the share of each run that is
#    curl's own process startup (see BATCH_LO below): a per-request
#    constant present in both arms, pulling the same way.
#
#    How large that share is depends on how much of the machine the
#    clients get, which is why the same box has produced both 0.20x
#    and 0.35x for the same configuration: under contention the
#    fastest arm loses the most.  The reported CPU is the tell -- an
#    origin at 3 cores and one at 1.5 are not the same measurement.
#
#  * paivana is single-threaded by construction -- one GNUnet scheduler
#    driving MHD and libcurl, no threads -- while upstream_rs spawns a
#    thread per connection.  On a multi-core box the direct arm is
#    therefore free to use every core and the proxy arm is not.  That
#    is a real property of the proxy and not an artefact, but it does
#    mean the ratio is a function of how many cores the machine has, so
#    CPU seconds and the core count are reported alongside.
#
#  * The paywall page is not the same size as -s, so paywall mode is
#    comparable to the other two in requests/s and not in MB/s.  Its
#    size is measured and printed rather than assumed.
#
#  * MB is 10^6 bytes.
#
# On the merchant backend in paywall mode: paivana does not open its
# listen socket until it has fetched a template from one, so paywall
# mode needs a backend before it can measure anything.  It starts
# merchant_stub, which answers the two GETs of that startup exchange
# and nothing else.  That is not a shortcut around a real backend, it
# is the whole of what a real backend would do here: the template is
# fetched once, the page is rendered locally from it, and the rendered
# response is cached, so from the first measured request onwards a live
# merchant would be exactly as idle as the stub.  Nothing in this
# benchmark buys anything.  For anyone who would rather check that than
# take it: set PAIVANA_BENCH_MERCHANT_URL (and, if it wants one,
# PAIVANA_BENCH_MERCHANT_TOKEN) and paivana is pointed at that instead
# -- it needs an instance carrying a `paivana' template whose ID is
# `premium'.
# ---------------------------------------------------------------------

set -u

# EPOCHREALTIME and every printf %f below assume a '.' radix character.
export LC_ALL=C

CLIENTS=8
SIZE=65536
DURATION=10
MODE=both
PORT_BASE="${PAIVANA_BENCH_PORT_BASE:-18600}"

# The upstream will not serve more than this from /large/; asking for
# more silently gets you this much, which would then be reported as the
# size that was asked for.
SIZE_MAX=$((10 * 1024 * 1024))

# Prints the comment block above, from the first line of prose to the
# `# ---' divider.  Delimited rather than given as a line range: a line
# range silently starts printing the wrong thing the first time anyone
# adds a paragraph up there, and it already had.
function usage() {
    awk 'NR < 3 { next } /^# -----/ { exit } { sub(/^# ?/, ""); print }' "$0"
    exit "${1:-0}"
}

function die() {
    echo "FAIL: $*" >&2
    exit 1
}

function parse_size() {
    # Accepts 4096, 64k, 2M (also 64K / 2m).
    local s="$1" mult=1

    case "$s" in
        *k | *K) mult=1024; s="${s%[kK]}" ;;
        *m | *M) mult=$((1024 * 1024)); s="${s%[mM]}" ;;
    esac
    case "$s" in
        '' | *[!0-9]*) die "not a size: $1" ;;
    esac
    echo $((s * mult))
}

while getopts "c:s:d:m:b:h" opt;
do
    case "$opt" in
        c) CLIENTS="$OPTARG" ;;
        s) SIZE="$(parse_size "$OPTARG")" || exit 1 ;;
        d) DURATION="$OPTARG" ;;
        m) MODE="$OPTARG" ;;
        b) PORT_BASE="$OPTARG" ;;
        h) usage 0 ;;
        *) usage 1 ;;
    esac
done

case "$CLIENTS" in '' | *[!0-9]*) die "-c wants a count, got '$CLIENTS'" ;; esac
case "$DURATION" in '' | *[!0-9]*) die "-d wants seconds, got '$DURATION'" ;; esac
case "$PORT_BASE" in '' | *[!0-9]*) die "-b wants a port, got '$PORT_BASE'" ;; esac
[ "$CLIENTS" -ge 1 ] || die "-c must be at least 1"
[ "$DURATION" -ge 1 ] || die "-d must be at least 1"
[ "$SIZE" -le "$SIZE_MAX" ] ||
    die "-s is capped at $SIZE_MAX bytes; upstream_rs truncates /large/ there," \
        "so a larger request would be reported at a size it never served"
case "$MODE" in
    direct | proxy | paywall | both | all) ;;
    *) die "-m wants direct, proxy, paywall, both or all, got '$MODE'" ;;
esac

# Which arms this run consists of.  `both' is direct+proxy, which is
# what it meant before paywall mode existed and what `make benchmark'
# still gets; `all' is the three.
DO_DIRECT=0; DO_PROXY=0; DO_PAYWALL=0
case "$MODE" in
    direct) DO_DIRECT=1 ;;
    proxy) DO_PROXY=1 ;;
    paywall) DO_PAYWALL=1 ;;
    both) DO_DIRECT=1; DO_PROXY=1 ;;
    all) DO_DIRECT=1; DO_PROXY=1; DO_PAYWALL=1 ;;
esac

# A sanitized build measures the sanitizer.  Refusing is more useful
# than publishing a number that is a factor of several out.
if [ "${PAIVANA_SANITIZED:-0}" = "1" ];
then
    echo "SKIP: this is a sanitizer build; its timings measure ASan," \
         "not paivana" >&2
    exit 77
fi

function here() {
    cd -- "$(dirname -- "$0")" && pwd
}

SRCDIR="${SRCDIR:-$(here)}"
BUILDDIR="${BUILDDIR:-$PWD}"

PAIVANA_HTTPD="${PAIVANA_HTTPD:-$BUILDDIR/../backend/paivana-httpd}"
if [ ! -x "$PAIVANA_HTTPD" ] && [ -x "$SRCDIR/../backend/paivana-httpd" ];
then
    PAIVANA_HTTPD="$SRCDIR/../backend/paivana-httpd"
fi
UPSTREAM_RS="$BUILDDIR/upstream_rs"
MERCHANT_STUB="$BUILDDIR/merchant_stub"

# The template ID merchant_stub serves; also the last path segment of
# the URL paywall mode measures, so the two have to agree.  Overridable
# only because PAIVANA_BENCH_MERCHANT_URL points at a backend whose
# template is named by whoever set it up.
TEMPLATE_ID="${PAIVANA_BENCH_TEMPLATE_ID:-premium}"

# Set from the environment: a live merchant backend to use instead of
# merchant_stub.  Empty means "start the stub".
MERCHANT_URL="${PAIVANA_BENCH_MERCHANT_URL:-}"
# Not a credential: it is what the stub is told to expect on the same
# command line.  A real backend's token comes from the environment and
# is never written to the log or the config we generate for it.
MERCHANT_TOKEN="${PAIVANA_BENCH_MERCHANT_TOKEN:-secret-token:benchmark-stub}"

if [ "$DO_DIRECT" = 1 ] || [ "$DO_PROXY" = 1 ];
then
    if [ ! -x "$UPSTREAM_RS" ];
    then
        echo "SKIP: upstream_rs not built (no rustc?), nothing to serve the" \
             "page these modes measure" >&2
        exit 77
    fi
fi
if [ "$MODE" != direct ] && [ ! -x "$PAIVANA_HTTPD" ];
then
    echo "SKIP: paivana-httpd not found at $PAIVANA_HTTPD" >&2
    exit 77
fi
command -v curl >/dev/null 2>&1 || { echo "SKIP: no curl" >&2; exit 77; }

# What paywall mode needs beyond the above: something to answer
# paivana's startup template fetch, and the paywall template itself.
# The latter is generated into the build tree, so a source checkout
# alone does not have it -- and PAIVANA_PREFIX below stages it rather
# than requiring `make install', which is what test_paywall.sh does for
# the same reason.
PAYWALL_TEMPLATE=""
if [ "$DO_PAYWALL" = 1 ];
then
    if [ -z "$MERCHANT_URL" ] && [ ! -x "$MERCHANT_STUB" ];
    then
        echo "SKIP: merchant_stub not built (no rustc?) and no" \
             "PAIVANA_BENCH_MERCHANT_URL, so the paywall cannot be made active" >&2
        exit 77
    fi
    for cand in "$BUILDDIR/../frontend/paywall.en.must" \
                "$SRCDIR/../frontend/paywall.en.must";
    do
        [ -r "$cand" ] && { PAYWALL_TEMPLATE="$cand"; break; }
    done
    if [ -z "$PAYWALL_TEMPLATE" ];
    then
        echo "SKIP: the paywall template has not been built, so there is no" \
             "paywall page to measure" >&2
        exit 77
    fi
fi

RS_PORT=$((PORT_BASE + 1))
PAIVANA_PORT=$((PORT_BASE + 2))
MERCHANT_PORT=$((PORT_BASE + 3))

SCRATCH="$(mktemp -d -t paivana-bench.XXXXXX)"
mkdir -p "$SCRATCH/logs" "$SCRATCH/out"

# Same reason as in test_reverse_proxy.sh: paivana would otherwise pull
# config.d out of the install prefix rather than the build tree.
BASE_CONFIG_DIR="$SCRATCH/configd"
mkdir -p "$BASE_CONFIG_DIR"
export PAIVANA_BASE_CONFIG="$BASE_CONFIG_DIR"

RS_PID=""
PAIVANA_PID=""
MERCHANT_PID=""
WORKERS=()

function cleanup() {
    set +e
    for p in "${WORKERS[@]:-}";
    do
        [ -n "$p" ] && kill -KILL "$p" 2>/dev/null
    done
    for p in "$PAIVANA_PID" "$RS_PID" "$MERCHANT_PID";
    do
        [ -n "$p" ] && kill -TERM "$p" 2>/dev/null
    done
    sleep 0.2
    for p in "$PAIVANA_PID" "$RS_PID" "$MERCHANT_PID";
    do
        [ -n "$p" ] && kill -KILL "$p" 2>/dev/null
    done
    if [ "${KEEP_TMP:-0}" = "1" ];
    then
        echo "Temp files kept in $SCRATCH" >&2
    else
        rm -rf "$SCRATCH"
    fi
}
trap cleanup EXIT
trap 'echo "FAIL: interrupted" >&2; exit 1' INT TERM

# The /dev/tcp probe runs in a subshell on purpose: a failing `exec'
# redirection kills a non-interactive bash outright, and 2>/dev/null
# does not save it.
function port_is_free() {
    if ( exec 7<>"/dev/tcp/127.0.0.1/$1" ) 2>/dev/null;
    then
        return 1
    fi
    return 0
}

function wait_for_port() {
    local port="$1" pid="$2" tries=50

    while [ "$tries" -gt 0 ];
    do
        port_is_free "$port" || return 0
        kill -0 "$pid" 2>/dev/null || return 1
        sleep 0.1
        tries=$((tries - 1))
    done
    return 1
}

PORTS_NEEDED=()
if [ "$DO_DIRECT" = 1 ] || [ "$DO_PROXY" = 1 ];
then
    PORTS_NEEDED+=("$RS_PORT")
fi
if [ "$MODE" != direct ];
then
    PORTS_NEEDED+=("$PAIVANA_PORT")
fi
if [ "$DO_PAYWALL" = 1 ] && [ -z "$MERCHANT_URL" ];
then
    PORTS_NEEDED+=("$MERCHANT_PORT")
fi
for p in "${PORTS_NEEDED[@]}";
do
    port_is_free "$p" && continue
    echo "SKIP: port $p is already in use; re-run with -b pointing at a" \
         "free block of three (current base: $PORT_BASE)" >&2
    exit 77
done

# Microseconds as an integer.  ${EPOCHREALTIME} always has six decimal
# places, so deleting the radix character is the whole conversion, and
# it costs no fork -- which matters in the worker loop below, where a
# $(date) per iteration would be a measurable part of what is being
# measured.
function now_us() {
    local t="${EPOCHREALTIME}"
    echo "${t/./}"
}

CLK_TCK="$(getconf CLK_TCK 2>/dev/null || echo 100)"

function cpu_jiffies() {
    local pid="$1" st
    local -a f

    if [ -z "$pid" ] || [ ! -r "/proc/$pid/stat" ];
    then
        echo 0
        return 0
    fi
    st="$(< "/proc/$pid/stat")" || { echo 0; return 0; }
    # The comm field is parenthesised and may contain spaces, so it is
    # cut off rather than counted through.  What is left starts at
    # field 3, which puts utime (14) and stime (15) at 11 and 12.
    st="${st#*") "}"
    # shellcheck disable=SC2206 # splitting on whitespace is the point
    f=($st)
    echo $(( ${f[11]:-0} + ${f[12]:-0} ))
}

function start_upstream() {
    ( exec "$UPSTREAM_RS" "$RS_PORT" ) >"$SCRATCH/logs/rs.log" 2>&1 &
    RS_PID=$!
    wait_for_port "$RS_PORT" "$RS_PID" ||
        { tail -n 20 "$SCRATCH/logs/rs.log" >&2
          die "upstream_rs did not come up on port $RS_PORT"; }
}

function start_merchant_stub() {
    # Only when we are providing the backend ourselves; with
    # PAIVANA_BENCH_MERCHANT_URL set there is nothing to start.
    [ -z "$MERCHANT_URL" ] || return 0
    ( exec "$MERCHANT_STUB" "$MERCHANT_PORT" "$MERCHANT_TOKEN" \
      ) >"$SCRATCH/logs/merchant.log" 2>&1 &
    MERCHANT_PID=$!
    MERCHANT_URL="http://127.0.0.1:$MERCHANT_PORT/"
    wait_for_port "$MERCHANT_PORT" "$MERCHANT_PID" ||
        { tail -n 20 "$SCRATCH/logs/merchant.log" >&2
          die "merchant_stub did not come up on port $MERCHANT_PORT"; }
}

# $1 = paywall (paywall on, merchant backend configured) or proxy
# (`-n', no paywall at all).  Restarting between the two arms rather
# than running one daemon is deliberate: `-n' is read once at startup
# and decides whether the paywall check runs at all, so a single
# process cannot be both.
function start_paivana() {
    local kind="$1"
    local -a args=(-c "$SCRATCH/paivana.conf" -L ERROR)

    {
        echo "# Generated by benchmark.sh."
        echo "[paivana]"
        # Required even in paywall mode, where nothing is ever
        # forwarded and so nothing has to be listening there.
        echo "DESTINATION_BASE_URL = http://127.0.0.1:$RS_PORT/"
        echo "SERVE = tcp"
        echo "PORT = $PAIVANA_PORT"
        echo "BASE_URL = http://localhost:$PAIVANA_PORT/"
        echo "SECRET = paivana-benchmark"
        if [ "$kind" = paywall ];
        then
            echo "MERCHANT_BACKEND_URL = $MERCHANT_URL"
            echo "MERCHANT_ACCESS_TOKEN = $MERCHANT_TOKEN"
        fi
    } > "$SCRATCH/paivana.conf"
    # The file names a bearer token, so it is readable by us alone --
    # $SCRATCH is a mktemp -d, but the token can also come from the
    # environment and pointing at a real backend must not be a way to
    # publish its token to everyone on the machine.
    chmod 600 "$SCRATCH/paivana.conf"
    if [ "$kind" != paywall ];
    then
        args+=(-n)
    fi
    ( if [ "$kind" = paywall ]; then export PAIVANA_PREFIX="$SCRATCH/prefix/"; fi
      exec "$PAIVANA_HTTPD" "${args[@]}" ) >"$SCRATCH/logs/paivana.log" 2>&1 &
    PAIVANA_PID=$!
    # In paywall mode this waits out the template fetch as well: the
    # listen socket is not bound until the backend has answered, so a
    # port that never opens is as likely to be a merchant problem as a
    # paivana one, which is why the log tail is printed either way.
    wait_for_port "$PAIVANA_PORT" "$PAIVANA_PID" ||
        { tail -n 20 "$SCRATCH/logs/paivana.log" >&2
          die "paivana-httpd did not come up on port $PAIVANA_PORT"; }
}

function stop_paivana() {
    [ -n "$PAIVANA_PID" ] || return 0
    kill -TERM "$PAIVANA_PID" 2>/dev/null
    wait "$PAIVANA_PID" 2>/dev/null
    PAIVANA_PID=""
}

# A worker hands several URLs to one curl invocation, because a fork
# per request would put bash's process creation into the measurement.
# But it only looks at the clock between invocations, so the batch size
# is also the granularity at which the run can stop -- and the right
# size cannot be computed up front: how long a batch takes depends on
# the per-worker request rate, which falls as -c rises and is exactly
# what is being measured.  A size picked from the page size alone
# overshot a 3 s run by 7% at -c 32 and would have been far worse at
# -c 200.
#
# So the workers converge on it instead: start small, double while a
# batch comes in under BATCH_LO, halve when it exceeds BATCH_HI.  That
# bounds the overshoot at roughly BATCH_HI no matter what -c and -s
# are, and costs a handful of extra invocations at the start.
#
# The bounds are set by what one curl invocation costs before it sends
# anything: measured at 12.5 ms on the machine this was written on,
# which is a lot of dynamic linking (openssl, nghttp2, brotli, zstd,
# ldap...).  Holding a batch to at least BATCH_LO keeps that under
# about 6% of the batch; at a fixed batch of 8 it was 44%, and the
# reported rate came out at half the truth.
BATCH_MAX=256
BATCH_LO=200000     # us; below this curl's own startup is too large a share
BATCH_HI=600000     # us; above this the deadline granularity is too coarse

# -w goes to stderr, not stdout, for a reason that is easy to trip
# over: -o is a *per-URL* option, so with many URLs on one command line
# only the first body would be redirected and the rest would land in
# the middle of the statistics.  %{stderr} splits the two streams
# instead, letting stdout be discarded wholesale.  stderr is unbuffered,
# so a line is on disk the moment the transfer ends.
#
# -m 30 keeps a hung transfer from wedging a worker until the deadline:
# the loop only reaches its clock check between invocations.
CURL_ARGS=(
    -s
    -m 30
    -H 'Connection: close'
    -w '%{stderr}%{http_code} %{size_download}
'
)

function worker() {
    local out="$1" deadline_us="$2"; shift 2
    local t t0 dt n=8

    [ "$n" -gt "$BATCH_MAX" ] && n="$BATCH_MAX"
    while :;
    do
        t="${EPOCHREALTIME}"; t0="${t/./}"
        [ "$t0" -ge "$deadline_us" ] && break
        curl "${CURL_ARGS[@]}" "${@:1:$n}" >/dev/null 2>>"$out"
        t="${EPOCHREALTIME}"; dt=$(( ${t/./} - t0 ))
        if [ "$dt" -lt "$BATCH_LO" ] && [ "$n" -lt "$BATCH_MAX" ];
        then
            n=$((n * 2))
            [ "$n" -gt "$BATCH_MAX" ] && n="$BATCH_MAX"
        elif [ "$dt" -gt "$BATCH_HI" ] && [ "$n" -gt 1 ];
        then
            n=$((n / 2))
        fi
    done
}

# Results of the last run_mode(), for the comparison at the end.
R_OK=0; R_BAD=0; R_BYTES=0; R_SECS=0; R_RATE=0; R_MBPS=0; R_PAGE=0

function run_mode() {
    # $1 = label, $2 = URL, $3 = expected status, $4.. = pids whose CPU
    # time to account
    local label="$1" url="$2" want_status="$3"; shift 3
    local pids=("$@")
    local outdir="$SCRATCH/out/$label"
    local i start_us end_us deadline_us
    local -a urls cpu0 cpu1

    rm -rf "$outdir"; mkdir -p "$outdir"

    urls=()
    for ((i = 0; i < BATCH_MAX; i++));
    do
        urls+=("$url")
    done

    # Warm up: fault in the upstream's page buffer, or render and cache
    # the paywall page, so that neither is charged to the run.  It also
    # establishes the size every later response is checked against --
    # measured rather than assumed, because in paywall mode the size is
    # whatever the template renders to and nothing here knows it up
    # front.
    local warm
    warm="$(curl -s -m 30 -H 'Connection: close' -o /dev/null \
                 -w '%{http_code} %{size_download}' "$url")" ||
        die "$label: the very first request to $url failed"
    read -r i R_PAGE <<<"$warm"
    [ "$i" = "$want_status" ] ||
        die "$label: $url answered $i, expected $want_status" \
            "-- benchmarking that would measure the wrong response"
    [ "$R_PAGE" -gt 0 ] ||
        die "$label: $url returned an empty body; there is nothing to measure"
    # Note the redirection rather than -o: -o is per-URL, so with two
    # URLs it would discard one body and print the other.
    curl -s -m 30 -H 'Connection: close' "$url" "$url" >/dev/null ||
        die "$label: the warm-up request to $url failed"

    cpu0=()
    for p in "${pids[@]}";
    do
        cpu0+=("$(cpu_jiffies "$p")")
    done

    start_us="$(now_us)"
    deadline_us=$((start_us + DURATION * 1000000))
    WORKERS=()
    for ((i = 0; i < CLIENTS; i++));
    do
        worker "$outdir/w$i.err" "$deadline_us" "${urls[@]}" &
        WORKERS+=($!)
    done
    for p in "${WORKERS[@]}";
    do
        wait "$p"
    done
    WORKERS=()
    end_us="$(now_us)"

    cpu1=()
    for p in "${pids[@]}";
    do
        cpu1+=("$(cpu_jiffies "$p")")
    done

    local elapsed_us=$((end_us - start_us))
    local stats
    stats="$(cat "$outdir"/w*.err 2>/dev/null |
        awk -v want="$R_PAGE" -v st="$want_status" '
            {
                # A short body is as much a failure as a 502: it would
                # otherwise be counted as a request served and its
                # missing bytes as throughput not achieved.  The status
                # is the one the warm-up got, so in paywall mode a 402
                # is the success case and a 200 would be the failure.
                if (st == $1 && $2 + 0 == want) { ok++; bytes += $2 }
                else { bad++; codes[$1]++ }
            }
            END {
                brk = ""
                for (c in codes) brk = brk " " c "=" codes[c]
                printf "%d %d %d%s\n", ok + 0, bad + 0, bytes + 0, brk
            }')"
    local brk
    read -r R_OK R_BAD R_BYTES brk <<<"$stats"

    R_SECS="$(awk -v u="$elapsed_us" 'BEGIN { printf "%.2f", u / 1000000 }')"
    # The parentheses around each ternary are load-bearing: inside a
    # printf argument list awk reads a bare `>' as an output
    # redirection, not as a comparison.
    R_RATE="$(awk -v n="$R_OK" -v u="$elapsed_us" \
        'BEGIN { printf "%.1f", (u > 0 ? n * 1000000 / u : 0) }')"
    # bytes per microsecond is already megabytes (10^6) per second.
    R_MBPS="$(awk -v b="$R_BYTES" -v u="$elapsed_us" \
        'BEGIN { printf "%.1f", (u > 0 ? b / u : 0) }')"

    echo
    echo "$label"
    # Per-mode rather than only in the banner: the paywall page is
    # whatever the template renders to and is not the -s size, so a
    # single "page:" line at the top would be wrong for one of the arms.
    printf '  page         : %s bytes, HTTP %s\n' "$R_PAGE" "$want_status"
    printf '  requests     : %s ok, %s failed%s\n' \
           "$R_OK" "$R_BAD" "${brk:+ ($brk)}"
    printf '  elapsed      : %s s\n' "$R_SECS"
    printf '  requests/s   : %s\n' "$R_RATE"
    printf '  throughput   : %s MB/s\n' "$R_MBPS"

    local n=0 line=""
    for p in "${pids[@]}";
    do
        local d=$(( ${cpu1[$n]} - ${cpu0[$n]} ))
        line="$line${line:+, }$(awk -v j="$d" -v hz="$CLK_TCK" -v s="$R_SECS" \
            -v name="${PIDNAMES[$n]}" \
            'BEGIN { printf "%s %.2f s (%.2f cores)", name, j / hz,
                     (s > 0 ? j / hz / s : 0) }')"
        n=$((n + 1))
    done
    printf '  server cpu   : %s\n' "$line"
}

if [ "$DO_DIRECT" = 1 ] || [ "$DO_PROXY" = 1 ];
then
    start_upstream
fi

echo "paivana benchmark"
if [ "$DO_DIRECT" = 1 ] || [ "$DO_PROXY" = 1 ];
then
    printf '  page         : %s bytes, /large/%s on upstream_rs (static, no disk)\n' \
           "$SIZE" "$SIZE"
fi
if [ "$DO_PAYWALL" = 1 ];
then
    printf '  paywall page : /.well-known/paivana/templates/%s, rendered once\n' \
           "$TEMPLATE_ID"
    printf '                 and served from paivana'"'"'s own cache; size measured below\n'
fi
printf '  clients      : %s parallel curl workers, up to %s URLs per invocation,\n' \
       "$CLIENTS" "$BATCH_MAX"
printf '                 one fresh connection per request in every mode\n'
printf '  duration     : %s s per mode\n' "$DURATION"
printf '  machine      : %s CPUs\n' "$(nproc 2>/dev/null || echo '?')"

FAILED=0

if [ "$DO_DIRECT" = 1 ];
then
    PIDNAMES=(upstream_rs)
    run_mode "direct (curl -> upstream_rs)" \
             "http://127.0.0.1:$RS_PORT/large/$SIZE" 200 "$RS_PID"
    [ "$R_PAGE" -eq "$SIZE" ] ||
        die "upstream_rs served $R_PAGE bytes where $SIZE were asked for"
    D_RATE="$R_RATE"; D_MBPS="$R_MBPS"; D_OK="$R_OK"
    [ "$R_BAD" -gt 0 ] && FAILED=1
fi

if [ "$DO_PROXY" = 1 ];
then
    start_paivana proxy
    PIDNAMES=(paivana-httpd upstream_rs)
    run_mode "proxy (curl -> paivana-httpd -> upstream_rs)" \
             "http://127.0.0.1:$PAIVANA_PORT/large/$SIZE" 200 \
             "$PAIVANA_PID" "$RS_PID"
    P_RATE="$R_RATE"; P_MBPS="$R_MBPS"; P_OK="$R_OK"
    [ "$R_BAD" -gt 0 ] && FAILED=1
    stop_paivana
fi

if [ "$DO_PAYWALL" = 1 ];
then
    # paivana loads the paywall page from its installation prefix, so a
    # build tree alone is not enough; stage the one file into a prefix
    # of our own rather than demanding `make install'.  Same trick, and
    # the same PAIVANA_PREFIX, as test_paywall.sh.
    mkdir -p "$SCRATCH/prefix/share/paivana/templates"
    cp "$PAYWALL_TEMPLATE" "$SCRATCH/prefix/share/paivana/templates/" ||
        die "cannot stage $PAYWALL_TEMPLATE"
    start_merchant_stub
    start_paivana paywall
    # upstream_rs is not in this path at all -- the paywall answers
    # before anything is forwarded -- so it is not accounted, and
    # merchant_stub is not either: it saw its two requests before the
    # clock started and is idle from here on.  If that is wrong, its
    # log says so.
    PIDNAMES=(paivana-httpd)
    run_mode "paywall (curl -> paivana-httpd, no upstream)" \
             "http://127.0.0.1:$PAIVANA_PORT/.well-known/paivana/templates/$TEMPLATE_ID" \
             402 "$PAIVANA_PID"
    # No W_MBPS: the comparison below is deliberately requests/s only,
    # since the two arms serve pages of different sizes.
    W_RATE="$R_RATE"; W_OK="$R_OK"; W_PAGE="$R_PAGE"
    [ "$R_BAD" -gt 0 ] && FAILED=1
    stop_paivana
fi

if [ "$DO_DIRECT" = 1 ] && [ "$DO_PROXY" = 1 ];
then
    echo
    echo "cost of the reverse proxy"
    awk -v dr="$D_RATE" -v pr="$P_RATE" -v dm="$D_MBPS" -v pm="$P_MBPS" '
        BEGIN {
            printf "%s", "  requests/s   : " pr " through paivana vs " dr \
                         " direct"
            if (dr > 0 && pr > 0)
                printf "  ->  %.2fx of direct (%.2fx slower)", pr / dr, dr / pr
            printf "\n"
            printf "  throughput   : %s vs %s MB/s\n", pm, dm
        }'
    echo
    echo "  Read as a rough figure for this machine, not a verdict.  The"
    echo "  direct arm is free to use every core and paivana is"
    echo "  single-threaded, so the ratio depends on the core count.  Both"
    echo "  arms are charged a TCP setup and a share of curl's own startup"
    echo "  per request; those are constants on both sides, so they pull the"
    echo "  ratio toward 1 and the proxy's real relative cost is a little"
    echo "  worse than shown, never better.  The clients compete with the"
    echo "  servers for this machine, so on a busy one the fastest arm loses"
    echo "  the most and the ratio flatters the proxy further; the core"
    echo "  counts above are what say which of those you got.  See the top"
    echo "  of this script."
fi

if [ "$DO_PAYWALL" = 1 ] && [ "$DO_PROXY" = 1 ];
then
    echo
    echo "cost of the paywall, against the same paivana forwarding"
    # Requests/s only.  The two arms serve different-sized pages -- the
    # paywall page is the template's size and the proxied one is -s --
    # so their MB/s are not two measurements of the same thing, and
    # printing a ratio of them would invite exactly that reading.  Both
    # sizes are shown instead.
    awk -v wr="$W_RATE" -v pr="$P_RATE" -v wp="$W_PAGE" -v sz="$SIZE" '
        BEGIN {
            printf "%s", "  requests/s   : " wr " turned away at the paywall vs " \
                         pr " forwarded"
            if (pr > 0 && wr > 0)
                printf "  ->  %.2fx", wr / pr
            printf "\n"
            printf "  page sizes   : %d bytes paywall vs %d bytes forwarded\n", wp, sz
        }'
    echo
    echo "  The paywall arm has no upstream in the path at all -- paivana"
    echo "  answers out of its own memory -- so the only things bounding it"
    echo "  are its single thread and the clients.  Which of the two it is"
    echo "  is in the core count above: if the rate stays flat as -c rises"
    echo "  while paivana sits below 1.0 cores, the thread is spending the"
    echo "  difference waiting on the kernel and more clients will not move"
    echo "  the number.  Note also that this page is the one an"
    echo "  unauthenticated client can ask for without limit -- no cookie,"
    echo "  no payment -- which is what makes its rate worth knowing."
fi

if [ "$FAILED" -ne 0 ];
then
    echo
    echo "FAIL: some requests did not complete; the rates above are not a" \
         "measurement of throughput" >&2
    # A code of 000 is curl reporting that the transfer never happened,
    # and by far the likeliest reason here is the client running out of
    # ephemeral ports: one connection per request against a fixed
    # server port means the 4-tuple is pinned for the TIME_WAIT
    # duration, and a few runs back to back can put tens of thousands
    # of them there.  Worth saying, because it is an exhausted local
    # port range and not anything paivana did.
    if grep -qs '^000 ' "$SCRATCH"/out/*/w*.err;
    then
        echo "NOTE: transfers that never connected (000) usually mean the" \
             "local ephemeral port range" \
             "($(tr '\t' '-' < /proc/sys/net/ipv4/ip_local_port_range \
                 2>/dev/null))" \
             "is full of TIME_WAIT sockets" \
             "($(ss -tan state time-wait 2>/dev/null | wc -l) right now)." \
             "Wait a minute and re-run." >&2
    fi
    exit 1
fi

if [ "${D_OK:-1}" -eq 0 ] || [ "${P_OK:-1}" -eq 0 ] || [ "${W_OK:-1}" -eq 0 ];
then
    echo
    echo "FAIL: no request completed at all" >&2
    exit 1
fi

exit 0
