diff --git a/src/apps/cli/testdeno/bench-network-cases.ts b/src/apps/cli/testdeno/bench-network-cases.ts index c460bd1..dfb6e3f 100644 --- a/src/apps/cli/testdeno/bench-network-cases.ts +++ b/src/apps/cli/testdeno/bench-network-cases.ts @@ -12,6 +12,15 @@ function readEnvString(name: string, fallback: string): string { return value && value.length > 0 ? value : fallback; } +function readEnvInteger(name: string, fallback: number): number { + const value = readEnvString(name, String(fallback)); + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${name} must be a positive integer, got '${value}'`); + } + return parsed; +} + function timestamp(): string { const d = new Date(); const pad = (n: number) => String(n).padStart(2, "0"); @@ -41,6 +50,7 @@ function buildCases(): BenchmarkCase[] { const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120"); const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478"); const shimCouchdbUri = readEnvString("BENCH_SHIM_COUCHDB_URI", "http://couchdb-shim:5984"); + const signallingShimRelay = readEnvString("BENCH_SIGNAL_SHIM_RELAY", "ws://p2p-signalling-shim:7777/"); return [ { @@ -136,6 +146,44 @@ function buildCases(): BenchmarkCase[] { "structural-placeholder-only; selected ICE pair may be collected, but the path is not shaped", }, }, + { + name: "p2p-signalling-netem-home-wifi", + runner: "p2p", + description: + "Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.", + dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim", + trustBoundary: "Nostr signalling metadata through constrained network shim; no TURN relay", + env: { + ...base, + BENCH_CASE: "p2p-signalling-netem-home-wifi", + BENCH_RELAY: signallingShimRelay, + BENCH_TURN_SERVERS: "", + BENCH_SIMULATION_TIER: "2", + BENCH_NETWORK_PROFILE: "home-wifi", + BENCH_NETWORK_MODEL: "compose-netem-signalling-shim", + BENCH_P2P_CANDIDATE_PATH_VERIFICATION: + "selected ICE pair collected; only Nostr signalling path is shaped", + }, + }, + { + name: "p2p-signalling-netem-tethering-vpn", + runner: "p2p", + description: + "Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.", + dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim", + trustBoundary: "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay", + env: { + ...base, + BENCH_CASE: "p2p-signalling-netem-tethering-vpn", + BENCH_RELAY: signallingShimRelay, + BENCH_TURN_SERVERS: "", + BENCH_SIMULATION_TIER: "2", + BENCH_NETWORK_PROFILE: "tethering-vpn", + BENCH_NETWORK_MODEL: "compose-netem-signalling-shim", + BENCH_P2P_CANDIDATE_PATH_VERIFICATION: + "selected ICE pair collected; only Nostr signalling path is shaped", + }, + }, { name: "p2p-user-turn", runner: "p2p", @@ -156,16 +204,25 @@ function buildCases(): BenchmarkCase[] { ]; } -async function runCase(testCase: BenchmarkCase, outputDir: string): Promise> { - const resultPath = `${outputDir}/${testCase.name}.json`; +async function runCase( + testCase: BenchmarkCase, + outputDir: string, + repeatIndex: number, + repeatCount: number +): Promise> { + const suffix = repeatCount > 1 ? `-r${String(repeatIndex).padStart(2, "0")}` : ""; + const resultPath = `${outputDir}/${testCase.name}${suffix}.json`; const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb"; const env = { ...Deno.env.toObject(), ...testCase.env, BENCH_RESULT_JSON: resultPath, + BENCH_REPEAT_INDEX: String(repeatIndex), + BENCH_REPEAT_COUNT: String(repeatCount), }; - console.log(`[bench-cases] running ${testCase.name}: ${testCase.description}`); + const repeatLabel = repeatCount > 1 ? ` (${repeatIndex}/${repeatCount})` : ""; + console.log(`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`); const command = new Deno.Command("deno", { args: ["task", taskName], cwd: import.meta.dirname, @@ -184,6 +241,9 @@ async function runCase(testCase: BenchmarkCase, outputDir: string): Promise; return { ...testCase, + repeatIndex, + repeatCount, + resultPath, result, }; } @@ -211,11 +271,13 @@ async function main(): Promise { const allCases = buildCases(); const cases = selectCases(allCases); + const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1); await Deno.writeTextFile( `${outputDir}/case-manifest.json`, JSON.stringify( { generatedAt: new Date().toISOString(), + repeatCount, selectedCases: cases, availableCases: allCases, }, @@ -226,12 +288,15 @@ async function main(): Promise { const results: Record[] = []; for (const testCase of cases) { - results.push(await runCase(testCase, outputDir)); + for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) { + results.push(await runCase(testCase, outputDir, repeatIndex, repeatCount)); + } } const summary = { generatedAt: new Date().toISOString(), outputDir, + repeatCount, results, }; await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2)); diff --git a/test/bench-network/README.md b/test/bench-network/README.md index 61e172c..5b93a64 100644 --- a/test/bench-network/README.md +++ b/test/bench-network/README.md @@ -47,6 +47,10 @@ Available local cases: - `p2p-smartphone-vpn-direct` - `p2p-user-turn` +Set `BENCH_REPEAT_COUNT` to run each selected case more than once. Repeated +results are written with suffixes such as `-r01`, `-r02`, and `-r03`, and the +summary records the repeat index for each run. + `p2p-smartphone-vpn-direct` is a structural case name. When it is run inside this Compose package it is not a real smartphone tethering/VPN measurement; it uses the local Compose network. Use it only for wiring checks unless the runner @@ -193,6 +197,41 @@ a unique `BENCH_SPLIT_RUN_ID`: docker compose -f test/bench-network/compose.yml --profile p2p-split down --volumes ``` +## P2P Signalling-Only Emulation + +The optional `signalling-shim` profile shapes only the Nostr signalling relay +path. The P2P host and client run in the benchmark runner as usual, and the +configured relay URL points at a TCP netem shim in front of `nostr-relay`. +This is the preferred fixture when evaluating the hypothesis that P2P avoids a +constrained remote database data path while still depending on a signalling +server for rendezvous. + +```bash +BENCH_CASES=p2p-signalling-netem-home-wifi \ +docker compose -f test/bench-network/compose.yml --profile signalling-shim run --rm \ + bench-runner-signalling-shim +``` + +For a stricter signalling path: + +```bash +NETEM_PROFILE=tethering-vpn \ +NETEM_DELAY_MS=140 \ +NETEM_JITTER_MS=50 \ +NETEM_LOSS_PERCENT=1.0 \ +NETEM_BANDWIDTH_MBIT=10 \ +NETEM_MTU=1380 \ +BENCH_CASES=p2p-signalling-netem-tethering-vpn \ +docker compose -f test/bench-network/compose.yml --profile signalling-shim run --rm \ + bench-runner-signalling-shim +``` + +Use this separately from `p2p-split`. The `p2p-split` profile shapes each peer's +egress path, so it constrains both signalling and the selected WebRTC data +path. The `signalling-shim` profile constrains only relay access, which keeps +it focused on peer-to-signalling-server reachability rather than peer-to-peer +note-data transfer. + ## Shimmed CouchDB benchmark The optional `shim` profile runs a CouchDB benchmark through a TCP forwarding diff --git a/test/bench-network/compose.yml b/test/bench-network/compose.yml index e444ab5..3f3167e 100644 --- a/test/bench-network/compose.yml +++ b/test/bench-network/compose.yml @@ -73,6 +73,7 @@ services: environment: BENCH_COMMAND: ${BENCH_COMMAND:-cases} BENCH_CASES: ${BENCH_CASES:-couchdb-baseline,p2p-direct-local} + BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1} BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results BENCH_SWEEP_RTT_MS: ${BENCH_SWEEP_RTT_MS:-20,50,100,150,300} @@ -141,6 +142,7 @@ services: environment: BENCH_COMMAND: ${BENCH_COMMAND:-cases} BENCH_CASES: ${BENCH_CASES:-couchdb-netem-home-wifi} + BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1} BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results BENCH_COUCHDB_MANAGED: "false" @@ -160,6 +162,64 @@ services: volumes: - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results + p2p-signalling-shim: + build: + context: ../.. + dockerfile: test/bench-network/Dockerfile.shim + profiles: + - signalling-shim + depends_on: + nostr-relay: + condition: service_healthy + cap_add: + - NET_ADMIN + environment: + NETEM_PROFILE: ${NETEM_PROFILE:-home-wifi} + NETEM_INTERFACE: ${NETEM_INTERFACE:-eth0} + NETEM_DELAY_MS: ${NETEM_DELAY_MS:-20} + NETEM_JITTER_MS: ${NETEM_JITTER_MS:-5} + NETEM_LOSS_PERCENT: ${NETEM_LOSS_PERCENT:-0.1} + NETEM_BANDWIDTH_MBIT: ${NETEM_BANDWIDTH_MBIT:-100} + NETEM_MTU: ${NETEM_MTU:-1500} + NETEM_RESULT_ROOT: /bench-results + SHIM_LISTEN_PORT: 7777 + SHIM_TARGET_HOST: nostr-relay + SHIM_TARGET_PORT: 7777 + volumes: + - ./bench-results:/bench-results + healthcheck: + test: ["CMD-SHELL", "nc -z 127.0.0.1 7777"] + interval: 2s + timeout: 5s + retries: 30 + + bench-runner-signalling-shim: + build: + context: ../.. + dockerfile: test/bench-network/Dockerfile.runner + profiles: + - signalling-shim + depends_on: + p2p-signalling-shim: + condition: service_healthy + environment: + BENCH_COMMAND: ${BENCH_COMMAND:-cases} + BENCH_CASES: ${BENCH_CASES:-p2p-signalling-netem-home-wifi} + BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1} + BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_SIGNAL_SHIM_RELAY: ws://p2p-signalling-shim:7777/ + BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20} + BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512} + BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048} + BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5} + BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192} + BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300} + BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60} + LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000} + BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0} + volumes: + - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results + p2p-split-host: build: context: ../..