Add Compose-based CLI network benchmarks

2026-07-14 00:26:00 +00:00 · 2026-07-08 06:30:12 +00:00
parent 78eb6ec3bc
commit ef655a2976
14 changed files with 704 additions and 12 deletions
@@ -0,0 +1 @@
+bench-results/
@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1
+
+FROM node:24-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl unzip python3 make g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV DENO_INSTALL=/usr/local
+RUN curl -fsSL https://deno.land/install.sh | sh
+
+WORKDIR /workspace
+
+COPY package.json package-lock.json ./
+COPY src/apps/cli/package.json ./src/apps/cli/package.json
+COPY src/apps/webapp/package.json ./src/apps/webapp/package.json
+COPY src/apps/webpeer/package.json ./src/apps/webpeer/package.json
+RUN npm ci
+
+COPY . .
+RUN npm run build -w self-hosted-livesync-cli
+
+WORKDIR /workspace/src/apps/cli/testdeno
+
+RUN deno cache --lock=deno.lock \
+    bench-network-cases.ts \
+    bench-latency-sweep.ts \
+    bench-p2p.ts \
+    bench-couchdb.ts
+
+COPY test/bench-network/run-bench.sh /usr/local/bin/run-livesync-bench
+RUN chmod +x /usr/local/bin/run-livesync-bench
+
+CMD ["run-livesync-bench"]
@@ -0,0 +1,90 @@
+# Network benchmark package
+
+This directory packages the CLI benchmark cases with Docker Compose. It is
+intended for reproducible local benchmark runs where CouchDB, the Nostr
+signalling relay, optional TURN, and the benchmark runner are fixed by the
+Compose file.
+
+## Quick smoke run
+
+From the repository root:
+
+```bash
+docker compose -f test/bench-network/compose.yml run --rm bench-runner
+```
+
+By default this runs:
+
+- `couchdb-baseline`
+- `p2p-direct-local`
+
+The dataset is intentionally small by default. Results are written to
+`test/bench-network/bench-results/`.
+
+## GitHub Actions smoke run
+
+`.github/workflows/cli-p2p-compose-smoke.yml` provides a manual
+`workflow_dispatch` smoke run for the same Compose package. It is intentionally
+not a required check yet, because WebRTC peer discovery can still be slow or
+environment-sensitive on GitHub-hosted runners. Keep the dataset small and use
+the uploaded JSON artefact to inspect whether failures are caused by peer
+discovery, synchronisation, CouchDB startup, or Docker networking.
+
+## Select cases
+
+```bash
+BENCH_CASES=couchdb-baseline,p2p-direct-local,p2p-user-turn \
+docker compose -f test/bench-network/compose.yml --profile turn run --rm bench-runner
+```
+
+Available local cases:
+
+- `couchdb-baseline`
+- `p2p-direct-local`
+- `couchdb-tethering-vpn-proxy`
+- `p2p-smartphone-vpn-direct`
+- `p2p-user-turn`
+
+`p2p-smartphone-vpn-direct` is a structural case name. When it is run inside
+this Compose package it is not a real smartphone tethering/VPN measurement; it
+uses the local Compose network. Use it only for wiring checks unless the runner
+is executed in an actual tethered/VPN environment.
+
+## Dataset and latency controls
+
+```bash
+BENCH_MD_FILE_COUNT=100 \
+BENCH_MD_MIN_SIZE_BYTES=512 \
+BENCH_MD_MAX_SIZE_BYTES=2048 \
+BENCH_BIN_FILE_COUNT=25 \
+BENCH_BIN_SIZE_BYTES=8192 \
+BENCH_COUCHDB_RTT_MS=20 \
+BENCH_PEERS_TIMEOUT=60 \
+docker compose -f test/bench-network/compose.yml run --rm bench-runner
+```
+
+The current CouchDB latency model is the existing HTTP proxy inside
+`bench-couchdb.ts`. It models a remote database path with additional request
+latency, but it does not model packet loss, jitter, MTU, bandwidth limits,
+bufferbloat, or VPN encapsulation.
+
+## Latency sweep
+
+To run P2P once and CouchDB at several requested RTT values:
+
+```bash
+BENCH_COMMAND=latency-sweep \
+BENCH_SWEEP_RTT_MS=20,50,100,150,300 \
+BENCH_MD_FILE_COUNT=100 \
+BENCH_MD_MIN_SIZE_BYTES=512 \
+BENCH_MD_MAX_SIZE_BYTES=2048 \
+BENCH_BIN_FILE_COUNT=25 \
+BENCH_BIN_SIZE_BYTES=8192 \
+BENCH_SYNC_TIMEOUT=300 \
+BENCH_PEERS_TIMEOUT=60 \
+docker compose -f test/bench-network/compose.yml run --rm bench-runner
+```
+
+This sweep is useful for finding where the remote CouchDB path falls behind the
+local direct P2P path in the current HTTP-proxy latency model. It should not be
+presented as a full smartphone/VPN model.
@@ -0,0 +1,93 @@
+services:
+  couchdb:
+    image: couchdb:3.5.0
+    environment:
+      COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin}
+      COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword}
+      COUCHDB_SINGLE_NODE: "true"
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -fsS -u ${BENCH_COUCHDB_USER:-admin}:${BENCH_COUCHDB_PASSWORD:-testpassword} http://127.0.0.1:5984/_up >/dev/null",
+        ]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+
+  nostr-relay:
+    image: ghcr.io/hoytech/strfry:latest
+    entrypoint: sh
+    command:
+      - -lc
+      - |
+        cat > /tmp/strfry.conf <<'EOF'
+        db = "./strfry-db/"
+
+        relay {
+          bind = "0.0.0.0"
+          port = 7777
+          nofiles = 100000
+
+          info {
+            name = "livesync bench relay"
+            description = "local relay for livesync compose benchmarks"
+          }
+
+          maxWebsocketPayloadSize = 131072
+          autoPingSeconds = 55
+
+          writePolicy {
+            plugin = ""
+          }
+        }
+        EOF
+        exec /app/strfry --config /tmp/strfry.conf relay
+    tmpfs:
+      - /app/strfry-db:rw,size=256m
+
+  coturn:
+    image: coturn/coturn:latest
+    command:
+      - --log-file=stdout
+      - --listening-port=3478
+      - --user=${BENCH_TURN_USERNAME:-testuser}:${BENCH_TURN_CREDENTIAL:-testpass}
+      - --realm=${BENCH_TURN_REALM:-livesync.test}
+    profiles:
+      - turn
+
+  bench-runner:
+    build:
+      context: ../..
+      dockerfile: test/bench-network/Dockerfile.runner
+    depends_on:
+      couchdb:
+        condition: service_healthy
+      nostr-relay:
+        condition: service_started
+    environment:
+      BENCH_COMMAND: ${BENCH_COMMAND:-cases}
+      BENCH_CASES: ${BENCH_CASES:-couchdb-baseline,p2p-direct-local}
+      BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results
+      BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results
+      BENCH_SWEEP_RTT_MS: ${BENCH_SWEEP_RTT_MS:-20,50,100,150,300}
+      BENCH_SWEEP_INCLUDE_P2P: ${BENCH_SWEEP_INCLUDE_P2P:-true}
+      BENCH_COUCHDB_MANAGED: "false"
+      BENCH_COUCHDB_BACKEND_URI: http://couchdb:5984
+      BENCH_COUCHDB_URI: http://127.0.0.1:15989
+      BENCH_COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin}
+      BENCH_COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword}
+      BENCH_RELAY: ws://nostr-relay:7777/
+      BENCH_LOCAL_TURN_SERVERS: turn:coturn:3478
+      BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20}
+      BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512}
+      BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
+      BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
+      BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
+      BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-20}
+      BENCH_TETHERING_VPN_RTT_MS: ${BENCH_TETHERING_VPN_RTT_MS:-120}
+      BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
+      BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
+      BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0}
+    volumes:
+      - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results
@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+set -eu
+
+case "${BENCH_COMMAND:-cases}" in
+  cases)
+    exec deno task bench:cases
+    ;;
+  latency-sweep)
+    exec deno task bench:latency-sweep
+    ;;
+  *)
+    echo "Unknown BENCH_COMMAND: ${BENCH_COMMAND}" >&2
+    echo "Expected one of: cases, latency-sweep" >&2
+    exit 2
+    ;;
+esac