From fd251346e3df065f1482ecf823a9a65e4c1e73ef Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 8 Jul 2026 06:30:12 +0000 Subject: [PATCH] Add Compose-based CLI network benchmarks --- .dockerignore | 4 +- .github/workflows/cli-p2p-compose-smoke.yml | 70 +++++++ src/apps/cli/testdeno/bench-couchdb.ts | 24 ++- src/apps/cli/testdeno/bench-latency-sweep.ts | 133 +++++++++++++ src/apps/cli/testdeno/bench-network-cases.ts | 198 +++++++++++++++++++ src/apps/cli/testdeno/bench-p2p.ts | 37 +++- src/apps/cli/testdeno/deno.json | 2 + src/apps/cli/testdeno/helpers/net.ts | 2 +- src/apps/cli/testdeno/helpers/p2p.ts | 12 +- test/bench-network/.gitignore | 1 + test/bench-network/Dockerfile.runner | 34 ++++ test/bench-network/README.md | 90 +++++++++ test/bench-network/compose.yml | 93 +++++++++ test/bench-network/run-bench.sh | 16 ++ 14 files changed, 704 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/cli-p2p-compose-smoke.yml create mode 100644 src/apps/cli/testdeno/bench-latency-sweep.ts create mode 100644 src/apps/cli/testdeno/bench-network-cases.ts create mode 100644 test/bench-network/.gitignore create mode 100644 test/bench-network/Dockerfile.runner create mode 100644 test/bench-network/README.md create mode 100644 test/bench-network/compose.yml create mode 100644 test/bench-network/run-bench.sh diff --git a/.dockerignore b/.dockerignore index 76fcffa..ec80feb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,7 +16,9 @@ pouchdb-browser.js production/ # Test coverage and reports -coverage/ +coverage/ +test/bench-network/bench-results/ +src/apps/cli/testdeno/bench-results/ # Local environment / secrets .env diff --git a/.github/workflows/cli-p2p-compose-smoke.yml b/.github/workflows/cli-p2p-compose-smoke.yml new file mode 100644 index 0000000..60455d5 --- /dev/null +++ b/.github/workflows/cli-p2p-compose-smoke.yml @@ -0,0 +1,70 @@ +# Run the Compose-packaged CLI P2P smoke benchmark. +# +# This workflow is intentionally manual/non-required at first. It exercises the +# local Compose package for CouchDB + Nostr relay + CLI runner, and uploads the +# benchmark JSON results for inspection. +name: cli-p2p-compose-smoke + +on: + workflow_dispatch: + inputs: + cases: + description: 'Comma-separated benchmark cases' + required: false + default: 'couchdb-baseline,p2p-direct-local' + md_files: + description: 'Markdown file count' + required: false + default: '2' + bin_files: + description: 'Binary file count' + required: false + default: '1' + couchdb_rtt_ms: + description: 'Requested CouchDB RTT in milliseconds' + required: false + default: '20' + +permissions: + contents: read + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Show Docker versions + run: | + docker --version + docker compose version + + - name: Run Compose P2P smoke benchmark + env: + BENCH_CASES: ${{ inputs.cases }} + BENCH_MD_FILE_COUNT: ${{ inputs.md_files }} + BENCH_MD_MIN_SIZE_BYTES: '128' + BENCH_MD_MAX_SIZE_BYTES: '256' + BENCH_BIN_FILE_COUNT: ${{ inputs.bin_files }} + BENCH_BIN_SIZE_BYTES: '512' + BENCH_COUCHDB_RTT_MS: ${{ inputs.couchdb_rtt_ms }} + BENCH_SYNC_TIMEOUT: '180' + BENCH_PEERS_TIMEOUT: '90' + BENCH_LIVESYNC_TEST_TEE: '0' + run: docker compose -f test/bench-network/compose.yml run --rm bench-runner + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: cli-p2p-compose-smoke-results + path: test/bench-network/bench-results/** + if-no-files-found: warn + + - name: Stop Compose services + if: always() + run: docker compose -f test/bench-network/compose.yml down -v --remove-orphans diff --git a/src/apps/cli/testdeno/bench-couchdb.ts b/src/apps/cli/testdeno/bench-couchdb.ts index 6ed6df7..255aaaf 100644 --- a/src/apps/cli/testdeno/bench-couchdb.ts +++ b/src/apps/cli/testdeno/bench-couchdb.ts @@ -1,10 +1,11 @@ import { TempDir } from "./helpers/temp.ts"; import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; -import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; +import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts"; import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts"; type BenchmarkConfig = { + caseName: string; couchdbBackendUri: string; couchdbProxyUri: string; couchdbUser: string; @@ -21,6 +22,7 @@ type BenchmarkConfig = { requestedRttMs: number; passphrase: string; encrypt: boolean; + managedCouchdb: boolean; }; function readEnvString(name: string, fallback: string): string { @@ -70,6 +72,7 @@ function formatBytes(value: number): string { function buildConfig(): BenchmarkConfig { return { + caseName: readEnvString("BENCH_CASE", "couchdb-baseline"), couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"), couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"), couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")), @@ -86,6 +89,7 @@ function buildConfig(): BenchmarkConfig { requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)), passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), encrypt: readEnvBool("BENCH_ENCRYPT", true), + managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true), }; } @@ -200,7 +204,17 @@ async function main(): Promise { await initSettingsFile(settingsA); await initSettingsFile(settingsB); - await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname); + if (config.managedCouchdb) { + await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname); + } else { + console.log(`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`); + await createCouchdbDatabase( + config.couchdbBackendUri, + config.couchdbUser, + config.couchdbPassword, + config.couchdbDbname + ); + } const proxy = startCouchdbProxy({ backendUri: config.couchdbBackendUri, @@ -265,10 +279,12 @@ async function main(): Promise { } const result = { + caseName: config.caseName, mode: "couchdb-cli-benchmark", couchdbBackendUri: config.couchdbBackendUri, couchdbProxyUri: config.couchdbProxyUri, couchdbDbname: config.couchdbDbname, + managedCouchdb: config.managedCouchdb, rttRequestedMs: config.requestedRttMs, proxyApplied: proxy.applied, proxyNote: proxy.note, @@ -300,7 +316,9 @@ async function main(): Promise { ); } finally { await proxy.stop(); - await stopCouchdb().catch(() => {}); + if (config.managedCouchdb) { + await stopCouchdb().catch(() => {}); + } } } diff --git a/src/apps/cli/testdeno/bench-latency-sweep.ts b/src/apps/cli/testdeno/bench-latency-sweep.ts new file mode 100644 index 0000000..d69de28 --- /dev/null +++ b/src/apps/cli/testdeno/bench-latency-sweep.ts @@ -0,0 +1,133 @@ +type SweepResult = { + name: string; + runner: "p2p" | "couchdb"; + rttMs?: number; + result: Record; +}; + +function readEnvString(name: string, fallback: string): string { + const value = Deno.env.get(name)?.trim(); + return value && value.length > 0 ? value : fallback; +} + +function timestamp(): string { + const d = new Date(); + const pad = (n: number) => String(n).padStart(2, "0"); + return ( + `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` + + `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}` + ); +} + +function parseRttList(raw: string): number[] { + const values = raw + .split(",") + .map((value) => Number(value.trim())) + .filter((value) => Number.isFinite(value) && value > 0) + .map((value) => Math.floor(value)); + if (values.length === 0) { + throw new Error(`BENCH_SWEEP_RTT_MS must contain at least one positive number, got '${raw}'`); + } + return values; +} + +function buildBaseEnv(): Record { + return { + BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"), + BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"), + BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"), + BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"), + BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"), + BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"), + BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"), + BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), + LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"), + }; +} + +async function runBenchmark(options: { + taskName: "bench:p2p" | "bench:couchdb"; + name: string; + outputDir: string; + env: Record; +}): Promise> { + const resultPath = `${options.outputDir}/${options.name}.json`; + const env = { + ...Deno.env.toObject(), + ...options.env, + BENCH_RESULT_JSON: resultPath, + }; + + console.log(`[latency-sweep] running ${options.name}`); + const child = new Deno.Command("deno", { + args: ["task", options.taskName], + cwd: import.meta.dirname, + env, + stdin: "null", + stdout: "inherit", + stderr: "inherit", + }).spawn(); + const status = await child.status; + if (status.code !== 0) { + throw new Error(`benchmark failed: ${options.name} (exit ${status.code})`); + } + return JSON.parse(await Deno.readTextFile(resultPath)) as Record; +} + +async function main(): Promise { + const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`); + const outputDir = `${outRoot}/latency-sweep-${timestamp()}`; + const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300")); + const base = buildBaseEnv(); + + await Deno.mkdir(outputDir, { recursive: true }); + + const results: SweepResult[] = []; + if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") { + const p2pResult = await runBenchmark({ + taskName: "bench:p2p", + name: "p2p-direct-local", + outputDir, + env: { + ...base, + BENCH_CASE: "p2p-direct-local", + BENCH_TURN_SERVERS: "", + }, + }); + results.push({ name: "p2p-direct-local", runner: "p2p", result: p2pResult }); + } + + for (const rtt of rtts) { + const name = `couchdb-rtt-${rtt}ms`; + const couchdbResult = await runBenchmark({ + taskName: "bench:couchdb", + name, + outputDir, + env: { + ...base, + BENCH_CASE: name, + BENCH_COUCHDB_RTT_MS: String(rtt), + }, + }); + results.push({ name, runner: "couchdb", rttMs: rtt, result: couchdbResult }); + } + + const summary = { + generatedAt: new Date().toISOString(), + outputDir, + note: + "This sweep models additional remote CouchDB request latency through the existing HTTP proxy. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.", + rtts, + results, + }; + await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2)); + console.log(JSON.stringify(summary, null, 2)); + console.log(`[latency-sweep] result directory: ${outputDir}`); +} + +if (import.meta.main) { + main().catch((error) => { + console.error("[Fatal Error]", error); + Deno.exit(1); + }); +} diff --git a/src/apps/cli/testdeno/bench-network-cases.ts b/src/apps/cli/testdeno/bench-network-cases.ts new file mode 100644 index 0000000..7997845 --- /dev/null +++ b/src/apps/cli/testdeno/bench-network-cases.ts @@ -0,0 +1,198 @@ +type BenchmarkCase = { + name: string; + runner: "p2p" | "couchdb"; + description: string; + dataPath: string; + trustBoundary: string; + env: Record; +}; + +function readEnvString(name: string, fallback: string): string { + const value = Deno.env.get(name)?.trim(); + return value && value.length > 0 ? value : fallback; +} + +function timestamp(): string { + const d = new Date(); + const pad = (n: number) => String(n).padStart(2, "0"); + return ( + `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` + + `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}` + ); +} + +function buildBaseEnv(): Record { + return { + BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"), + BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"), + BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"), + BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"), + BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"), + BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"), + BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"), + BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), + LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"), + }; +} + +function buildCases(): BenchmarkCase[] { + const base = buildBaseEnv(); + const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20"); + const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120"); + const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478"); + + return [ + { + name: "couchdb-baseline", + runner: "couchdb", + description: "Standard self-hosted CouchDB path through a local latency proxy.", + dataPath: "Device A -> CouchDB -> Device B", + trustBoundary: "CouchDB operator and network path", + env: { + ...base, + BENCH_CASE: "couchdb-baseline", + BENCH_COUCHDB_RTT_MS: couchdbRtt, + }, + }, + { + name: "p2p-direct-local", + runner: "p2p", + description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.", + dataPath: "Device A -> Device B", + trustBoundary: "Nostr relay for signalling metadata; no TURN relay", + env: { + ...base, + BENCH_CASE: "p2p-direct-local", + BENCH_TURN_SERVERS: "", + }, + }, + { + name: "couchdb-tethering-vpn-proxy", + runner: "couchdb", + description: + "Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.", + dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B", + trustBoundary: "VPN/network path and CouchDB operator", + env: { + ...base, + BENCH_CASE: "couchdb-tethering-vpn-proxy", + BENCH_COUCHDB_RTT_MS: tetheringVpnRtt, + }, + }, + { + name: "p2p-smartphone-vpn-direct", + runner: "p2p", + description: + "Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.", + dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds", + trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata", + env: { + ...base, + BENCH_CASE: "p2p-smartphone-vpn-direct", + BENCH_TURN_SERVERS: "", + }, + }, + { + name: "p2p-user-turn", + runner: "p2p", + description: "Optional fallback path through a local user-controlled TURN server.", + dataPath: "Device A -> user-controlled TURN -> Device B", + trustBoundary: "User-controlled TURN server", + env: { + ...base, + BENCH_CASE: "p2p-user-turn", + BENCH_TURN_SERVERS: localTurnServers, + }, + }, + ]; +} + +async function runCase(testCase: BenchmarkCase, outputDir: string): Promise> { + const resultPath = `${outputDir}/${testCase.name}.json`; + const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb"; + const env = { + ...Deno.env.toObject(), + ...testCase.env, + BENCH_RESULT_JSON: resultPath, + }; + + console.log(`[bench-cases] running ${testCase.name}: ${testCase.description}`); + const command = new Deno.Command("deno", { + args: ["task", taskName], + cwd: import.meta.dirname, + env, + stdin: "null", + stdout: "inherit", + stderr: "inherit", + }); + + const child = command.spawn(); + const status = await child.status; + if (status.code !== 0) { + throw new Error(`case failed: ${testCase.name} (exit ${status.code})`); + } + + const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record; + return { + ...testCase, + result, + }; +} + +function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] { + const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local"); + const names = requested + .split(",") + .map((v) => v.trim()) + .filter((v) => v.length > 0); + const byName = new Map(allCases.map((c) => [c.name, c])); + return names.map((name) => { + const found = byName.get(name); + if (!found) { + throw new Error(`Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`); + } + return found; + }); +} + +async function main(): Promise { + const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`); + const outputDir = `${outRoot}/cases-${timestamp()}`; + await Deno.mkdir(outputDir, { recursive: true }); + + const allCases = buildCases(); + const cases = selectCases(allCases); + await Deno.writeTextFile( + `${outputDir}/case-manifest.json`, + JSON.stringify( + { + generatedAt: new Date().toISOString(), + selectedCases: cases, + availableCases: allCases, + }, + null, + 2 + ) + ); + + const results: Record[] = []; + for (const testCase of cases) { + results.push(await runCase(testCase, outputDir)); + } + + const summary = { + generatedAt: new Date().toISOString(), + outputDir, + results, + }; + await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2)); + console.log(JSON.stringify(summary, null, 2)); + console.log(`[bench-cases] result directory: ${outputDir}`); +} + +if (import.meta.main) { + main().catch((error) => { + console.error("[Fatal Error]", error); + Deno.exit(1); + }); +} diff --git a/src/apps/cli/testdeno/bench-p2p.ts b/src/apps/cli/testdeno/bench-p2p.ts index cbc920f..e249428 100644 --- a/src/apps/cli/testdeno/bench-p2p.ts +++ b/src/apps/cli/testdeno/bench-p2p.ts @@ -1,15 +1,23 @@ import { TempDir } from "./helpers/temp.ts"; import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; -import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts"; +import { + discoverPeer, + maybeStartCoturn, + maybeStartLocalRelay, + stopCoturnIfStarted, + stopLocalRelayIfStarted, +} from "./helpers/p2p.ts"; import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts"; type BenchmarkConfig = { + caseName: string; relay: string; appId: string; roomId: string; passphrase: string; + turnServers: string; datasetDirName: string; datasetSeed: string; mdFileCount: number; @@ -61,10 +69,12 @@ function formatBytes(value: number): string { function buildConfig(): BenchmarkConfig { return { + caseName: readEnvString("BENCH_CASE", "p2p-direct-local"), relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"), appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"), roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`), passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), + turnServers: readEnvString("BENCH_TURN_SERVERS", ""), datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"), datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)), @@ -107,6 +117,7 @@ async function main(): Promise { const resultPath = readOptionalResultPath(); const relayStarted = await maybeStartLocalRelay(config.relay); + const coturnStarted = await maybeStartCoturn(config.turnServers); await using workDir = await TempDir.create("livesync-cli-p2p-bench"); const hostVault = workDir.join("vault-host"); @@ -122,8 +133,24 @@ async function main(): Promise { ]); await Promise.all([ - applyP2pSettings(hostSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"), - applyP2pSettings(clientSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"), + applyP2pSettings( + hostSettings, + config.roomId, + config.passphrase, + config.appId, + config.relay, + "~.*", + config.turnServers + ), + applyP2pSettings( + clientSettings, + config.roomId, + config.passphrase, + config.appId, + config.relay, + "~.*", + config.turnServers + ), ]); await Promise.all([ @@ -179,8 +206,11 @@ async function main(): Promise { } const result = { + caseName: config.caseName, mode: "p2p-cli-benchmark", relay: config.relay, + turnServers: config.turnServers, + turnEnabled: config.turnServers.trim().length > 0, appId: config.appId, roomId: config.roomId, datasetSeed: config.datasetSeed, @@ -211,6 +241,7 @@ async function main(): Promise { ); } finally { await host.stop(); + await stopCoturnIfStarted(coturnStarted); await stopLocalRelayIfStarted(relayStarted); } } diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json index 2fd5183..095ab2a 100644 --- a/src/apps/cli/testdeno/deno.json +++ b/src/apps/cli/testdeno/deno.json @@ -17,6 +17,8 @@ "test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts", "bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts", "bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts", + "bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts", + "bench:latency-sweep": "deno run --env-file=.test.env -A --no-check bench-latency-sweep.ts", "bench:item1": "bash ./bench-run-item1.sh", "bench:item1:full": "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh", "test:e2e-couchdb": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-couchdb.ts", diff --git a/src/apps/cli/testdeno/helpers/net.ts b/src/apps/cli/testdeno/helpers/net.ts index fa5debd..35a9d47 100644 --- a/src/apps/cli/testdeno/helpers/net.ts +++ b/src/apps/cli/testdeno/helpers/net.ts @@ -9,7 +9,7 @@ function sleep(ms: number): Promise { } async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise { - let timer: number | undefined; + let timer: ReturnType | undefined; try { const connPromise = Deno.connect({ hostname, port }); const timeoutPromise = new Promise((_, reject) => { diff --git a/src/apps/cli/testdeno/helpers/p2p.ts b/src/apps/cli/testdeno/helpers/p2p.ts index a04b080..6be7ebc 100644 --- a/src/apps/cli/testdeno/helpers/p2p.ts +++ b/src/apps/cli/testdeno/helpers/p2p.ts @@ -76,8 +76,10 @@ export async function discoverPeer( } export async function maybeStartLocalRelay(relay: string): Promise { - if (!isLocalP2pRelay(relay)) return false; - await startP2pRelay(); + const shouldStart = isLocalP2pRelay(relay); + if (shouldStart) { + await startP2pRelay(); + } const endpoint = parseRelayEndpoint(relay); await waitForPort(endpoint.hostname, endpoint.port, { timeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000"), @@ -86,8 +88,10 @@ export async function maybeStartLocalRelay(relay: string): Promise { }); // Docker proxy accepts TCP connections instantly before the container's internal process is fully ready. // Wait an additional few seconds to ensure strfry is actually accepting WebSockets. - await sleep(3000); - return true; + if (shouldStart) { + await sleep(3000); + } + return shouldStart; } export async function stopLocalRelayIfStarted(started: boolean): Promise { diff --git a/test/bench-network/.gitignore b/test/bench-network/.gitignore new file mode 100644 index 0000000..21f92d7 --- /dev/null +++ b/test/bench-network/.gitignore @@ -0,0 +1 @@ +bench-results/ diff --git a/test/bench-network/Dockerfile.runner b/test/bench-network/Dockerfile.runner new file mode 100644 index 0000000..759be55 --- /dev/null +++ b/test/bench-network/Dockerfile.runner @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1 + +FROM node:24-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl unzip python3 make g++ \ + && rm -rf /var/lib/apt/lists/* + +ENV DENO_INSTALL=/usr/local +RUN curl -fsSL https://deno.land/install.sh | sh + +WORKDIR /workspace + +COPY package.json package-lock.json ./ +COPY src/apps/cli/package.json ./src/apps/cli/package.json +COPY src/apps/webapp/package.json ./src/apps/webapp/package.json +COPY src/apps/webpeer/package.json ./src/apps/webpeer/package.json +RUN npm ci + +COPY . . +RUN npm run build -w self-hosted-livesync-cli + +WORKDIR /workspace/src/apps/cli/testdeno + +RUN deno cache --lock=deno.lock \ + bench-network-cases.ts \ + bench-latency-sweep.ts \ + bench-p2p.ts \ + bench-couchdb.ts + +COPY test/bench-network/run-bench.sh /usr/local/bin/run-livesync-bench +RUN chmod +x /usr/local/bin/run-livesync-bench + +CMD ["run-livesync-bench"] diff --git a/test/bench-network/README.md b/test/bench-network/README.md new file mode 100644 index 0000000..7ccb166 --- /dev/null +++ b/test/bench-network/README.md @@ -0,0 +1,90 @@ +# Network benchmark package + +This directory packages the CLI benchmark cases with Docker Compose. It is +intended for reproducible local benchmark runs where CouchDB, the Nostr +signalling relay, optional TURN, and the benchmark runner are fixed by the +Compose file. + +## Quick smoke run + +From the repository root: + +```bash +docker compose -f test/bench-network/compose.yml run --rm bench-runner +``` + +By default this runs: + +- `couchdb-baseline` +- `p2p-direct-local` + +The dataset is intentionally small by default. Results are written to +`test/bench-network/bench-results/`. + +## GitHub Actions smoke run + +`.github/workflows/cli-p2p-compose-smoke.yml` provides a manual +`workflow_dispatch` smoke run for the same Compose package. It is intentionally +not a required check yet, because WebRTC peer discovery can still be slow or +environment-sensitive on GitHub-hosted runners. Keep the dataset small and use +the uploaded JSON artefact to inspect whether failures are caused by peer +discovery, synchronisation, CouchDB startup, or Docker networking. + +## Select cases + +```bash +BENCH_CASES=couchdb-baseline,p2p-direct-local,p2p-user-turn \ +docker compose -f test/bench-network/compose.yml --profile turn run --rm bench-runner +``` + +Available local cases: + +- `couchdb-baseline` +- `p2p-direct-local` +- `couchdb-tethering-vpn-proxy` +- `p2p-smartphone-vpn-direct` +- `p2p-user-turn` + +`p2p-smartphone-vpn-direct` is a structural case name. When it is run inside +this Compose package it is not a real smartphone tethering/VPN measurement; it +uses the local Compose network. Use it only for wiring checks unless the runner +is executed in an actual tethered/VPN environment. + +## Dataset and latency controls + +```bash +BENCH_MD_FILE_COUNT=100 \ +BENCH_MD_MIN_SIZE_BYTES=512 \ +BENCH_MD_MAX_SIZE_BYTES=2048 \ +BENCH_BIN_FILE_COUNT=25 \ +BENCH_BIN_SIZE_BYTES=8192 \ +BENCH_COUCHDB_RTT_MS=20 \ +BENCH_PEERS_TIMEOUT=60 \ +docker compose -f test/bench-network/compose.yml run --rm bench-runner +``` + +The current CouchDB latency model is the existing HTTP proxy inside +`bench-couchdb.ts`. It models a remote database path with additional request +latency, but it does not model packet loss, jitter, MTU, bandwidth limits, +bufferbloat, or VPN encapsulation. + +## Latency sweep + +To run P2P once and CouchDB at several requested RTT values: + +```bash +BENCH_COMMAND=latency-sweep \ +BENCH_SWEEP_RTT_MS=20,50,100,150,300 \ +BENCH_MD_FILE_COUNT=100 \ +BENCH_MD_MIN_SIZE_BYTES=512 \ +BENCH_MD_MAX_SIZE_BYTES=2048 \ +BENCH_BIN_FILE_COUNT=25 \ +BENCH_BIN_SIZE_BYTES=8192 \ +BENCH_SYNC_TIMEOUT=300 \ +BENCH_PEERS_TIMEOUT=60 \ +docker compose -f test/bench-network/compose.yml run --rm bench-runner +``` + +This sweep is useful for finding where the remote CouchDB path falls behind the +local direct P2P path in the current HTTP-proxy latency model. It should not be +presented as a full smartphone/VPN model. diff --git a/test/bench-network/compose.yml b/test/bench-network/compose.yml new file mode 100644 index 0000000..e54ec0e --- /dev/null +++ b/test/bench-network/compose.yml @@ -0,0 +1,93 @@ +services: + couchdb: + image: couchdb:3.5.0 + environment: + COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin} + COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword} + COUCHDB_SINGLE_NODE: "true" + healthcheck: + test: + [ + "CMD-SHELL", + "curl -fsS -u ${BENCH_COUCHDB_USER:-admin}:${BENCH_COUCHDB_PASSWORD:-testpassword} http://127.0.0.1:5984/_up >/dev/null", + ] + interval: 2s + timeout: 5s + retries: 30 + + nostr-relay: + image: ghcr.io/hoytech/strfry:latest + entrypoint: sh + command: + - -lc + - | + cat > /tmp/strfry.conf <<'EOF' + db = "./strfry-db/" + + relay { + bind = "0.0.0.0" + port = 7777 + nofiles = 100000 + + info { + name = "livesync bench relay" + description = "local relay for livesync compose benchmarks" + } + + maxWebsocketPayloadSize = 131072 + autoPingSeconds = 55 + + writePolicy { + plugin = "" + } + } + EOF + exec /app/strfry --config /tmp/strfry.conf relay + tmpfs: + - /app/strfry-db:rw,size=256m + + coturn: + image: coturn/coturn:latest + command: + - --log-file=stdout + - --listening-port=3478 + - --user=${BENCH_TURN_USERNAME:-testuser}:${BENCH_TURN_CREDENTIAL:-testpass} + - --realm=${BENCH_TURN_REALM:-livesync.test} + profiles: + - turn + + bench-runner: + build: + context: ../.. + dockerfile: test/bench-network/Dockerfile.runner + depends_on: + couchdb: + condition: service_healthy + nostr-relay: + condition: service_started + environment: + BENCH_COMMAND: ${BENCH_COMMAND:-cases} + BENCH_CASES: ${BENCH_CASES:-couchdb-baseline,p2p-direct-local} + BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_SWEEP_RTT_MS: ${BENCH_SWEEP_RTT_MS:-20,50,100,150,300} + BENCH_SWEEP_INCLUDE_P2P: ${BENCH_SWEEP_INCLUDE_P2P:-true} + BENCH_COUCHDB_MANAGED: "false" + BENCH_COUCHDB_BACKEND_URI: http://couchdb:5984 + BENCH_COUCHDB_URI: http://127.0.0.1:15989 + BENCH_COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin} + BENCH_COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword} + BENCH_RELAY: ws://nostr-relay:7777/ + BENCH_LOCAL_TURN_SERVERS: turn:coturn:3478 + BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20} + BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512} + BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048} + BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5} + BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192} + BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-20} + BENCH_TETHERING_VPN_RTT_MS: ${BENCH_TETHERING_VPN_RTT_MS:-120} + BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300} + BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60} + BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0} + volumes: + - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results diff --git a/test/bench-network/run-bench.sh b/test/bench-network/run-bench.sh new file mode 100644 index 0000000..4ca9eee --- /dev/null +++ b/test/bench-network/run-bench.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh +set -eu + +case "${BENCH_COMMAND:-cases}" in + cases) + exec deno task bench:cases + ;; + latency-sweep) + exec deno task bench:latency-sweep + ;; + *) + echo "Unknown BENCH_COMMAND: ${BENCH_COMMAND}" >&2 + echo "Expected one of: cases, latency-sweep" >&2 + exit 2 + ;; +esac