Add Compose-based CLI network benchmarks

.dockerignore

@@ -16,7 +16,9 @@ pouchdb-browser.js
 production/
 
 # Test coverage and reports
-coverage/
+coverage/
+test/bench-network/bench-results/
+src/apps/cli/testdeno/bench-results/
 
 # Local environment / secrets
 .env

.github/workflows/cli-p2p-compose-smoke.yml

@@ -0,0 +1,70 @@
+# Run the Compose-packaged CLI P2P smoke benchmark.
+#
+# This workflow is intentionally manual/non-required at first. It exercises the
+# local Compose package for CouchDB + Nostr relay + CLI runner, and uploads the
+# benchmark JSON results for inspection.
+name: cli-p2p-compose-smoke
+
+on:
+  workflow_dispatch:
+    inputs:
+      cases:
+        description: 'Comma-separated benchmark cases'
+        required: false
+        default: 'couchdb-baseline,p2p-direct-local'
+      md_files:
+        description: 'Markdown file count'
+        required: false
+        default: '2'
+      bin_files:
+        description: 'Binary file count'
+        required: false
+        default: '1'
+      couchdb_rtt_ms:
+        description: 'Requested CouchDB RTT in milliseconds'
+        required: false
+        default: '20'
+
+permissions:
+  contents: read
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Show Docker versions
+        run: |
+          docker --version
+          docker compose version
+
+      - name: Run Compose P2P smoke benchmark
+        env:
+          BENCH_CASES: ${{ inputs.cases }}
+          BENCH_MD_FILE_COUNT: ${{ inputs.md_files }}
+          BENCH_MD_MIN_SIZE_BYTES: '128'
+          BENCH_MD_MAX_SIZE_BYTES: '256'
+          BENCH_BIN_FILE_COUNT: ${{ inputs.bin_files }}
+          BENCH_BIN_SIZE_BYTES: '512'
+          BENCH_COUCHDB_RTT_MS: ${{ inputs.couchdb_rtt_ms }}
+          BENCH_SYNC_TIMEOUT: '180'
+          BENCH_PEERS_TIMEOUT: '90'
+          BENCH_LIVESYNC_TEST_TEE: '0'
+        run: docker compose -f test/bench-network/compose.yml run --rm bench-runner
+
+      - name: Upload benchmark results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cli-p2p-compose-smoke-results
+          path: test/bench-network/bench-results/**
+          if-no-files-found: warn
+
+      - name: Stop Compose services
+        if: always()
+        run: docker compose -f test/bench-network/compose.yml down -v --remove-orphans

src/apps/cli/testdeno/bench-couchdb.ts

@@ -1,10 +1,11 @@
 import { TempDir } from "./helpers/temp.ts";
 import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
 import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
-import { startCouchdb, stopCouchdb } from "./helpers/docker.ts";
+import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts";
 import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
 
 type BenchmarkConfig = {
+    caseName: string;
     couchdbBackendUri: string;
     couchdbProxyUri: string;
     couchdbUser: string;
@@ -21,6 +22,7 @@ type BenchmarkConfig = {
     requestedRttMs: number;
     passphrase: string;
     encrypt: boolean;
+    managedCouchdb: boolean;
 };
 
 function readEnvString(name: string, fallback: string): string {
@@ -70,6 +72,7 @@ function formatBytes(value: number): string {
 
 function buildConfig(): BenchmarkConfig {
     return {
+        caseName: readEnvString("BENCH_CASE", "couchdb-baseline"),
         couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"),
         couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"),
         couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")),
@@ -86,6 +89,7 @@ function buildConfig(): BenchmarkConfig {
         requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)),
         passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
         encrypt: readEnvBool("BENCH_ENCRYPT", true),
+        managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true),
     };
 }
 
@@ -200,7 +204,17 @@ async function main(): Promise<void> {
     await initSettingsFile(settingsA);
     await initSettingsFile(settingsB);
 
-    await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
+    if (config.managedCouchdb) {
+        await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
+    } else {
+        console.log(`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`);
+        await createCouchdbDatabase(
+            config.couchdbBackendUri,
+            config.couchdbUser,
+            config.couchdbPassword,
+            config.couchdbDbname
+        );
+    }
 
     const proxy = startCouchdbProxy({
         backendUri: config.couchdbBackendUri,
@@ -265,10 +279,12 @@ async function main(): Promise<void> {
         }
 
         const result = {
+            caseName: config.caseName,
             mode: "couchdb-cli-benchmark",
             couchdbBackendUri: config.couchdbBackendUri,
             couchdbProxyUri: config.couchdbProxyUri,
             couchdbDbname: config.couchdbDbname,
+            managedCouchdb: config.managedCouchdb,
             rttRequestedMs: config.requestedRttMs,
             proxyApplied: proxy.applied,
             proxyNote: proxy.note,
@@ -300,7 +316,9 @@ async function main(): Promise<void> {
         );
     } finally {
         await proxy.stop();
-        await stopCouchdb().catch(() => {});
+        if (config.managedCouchdb) {
+            await stopCouchdb().catch(() => {});
+        }
     }
 }
 

src/apps/cli/testdeno/bench-latency-sweep.ts

@@ -0,0 +1,133 @@
+type SweepResult = {
+    name: string;
+    runner: "p2p" | "couchdb";
+    rttMs?: number;
+    result: Record<string, unknown>;
+};
+
+function readEnvString(name: string, fallback: string): string {
+    const value = Deno.env.get(name)?.trim();
+    return value && value.length > 0 ? value : fallback;
+}
+
+function timestamp(): string {
+    const d = new Date();
+    const pad = (n: number) => String(n).padStart(2, "0");
+    return (
+        `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
+        `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
+    );
+}
+
+function parseRttList(raw: string): number[] {
+    const values = raw
+        .split(",")
+        .map((value) => Number(value.trim()))
+        .filter((value) => Number.isFinite(value) && value > 0)
+        .map((value) => Math.floor(value));
+    if (values.length === 0) {
+        throw new Error(`BENCH_SWEEP_RTT_MS must contain at least one positive number, got '${raw}'`);
+    }
+    return values;
+}
+
+function buildBaseEnv(): Record<string, string> {
+    return {
+        BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
+        BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
+        BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
+        BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
+        BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
+        BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
+        BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
+        BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
+        LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
+    };
+}
+
+async function runBenchmark(options: {
+    taskName: "bench:p2p" | "bench:couchdb";
+    name: string;
+    outputDir: string;
+    env: Record<string, string>;
+}): Promise<Record<string, unknown>> {
+    const resultPath = `${options.outputDir}/${options.name}.json`;
+    const env = {
+        ...Deno.env.toObject(),
+        ...options.env,
+        BENCH_RESULT_JSON: resultPath,
+    };
+
+    console.log(`[latency-sweep] running ${options.name}`);
+    const child = new Deno.Command("deno", {
+        args: ["task", options.taskName],
+        cwd: import.meta.dirname,
+        env,
+        stdin: "null",
+        stdout: "inherit",
+        stderr: "inherit",
+    }).spawn();
+    const status = await child.status;
+    if (status.code !== 0) {
+        throw new Error(`benchmark failed: ${options.name} (exit ${status.code})`);
+    }
+    return JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
+}
+
+async function main(): Promise<void> {
+    const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`);
+    const outputDir = `${outRoot}/latency-sweep-${timestamp()}`;
+    const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300"));
+    const base = buildBaseEnv();
+
+    await Deno.mkdir(outputDir, { recursive: true });
+
+    const results: SweepResult[] = [];
+    if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") {
+        const p2pResult = await runBenchmark({
+            taskName: "bench:p2p",
+            name: "p2p-direct-local",
+            outputDir,
+            env: {
+                ...base,
+                BENCH_CASE: "p2p-direct-local",
+                BENCH_TURN_SERVERS: "",
+            },
+        });
+        results.push({ name: "p2p-direct-local", runner: "p2p", result: p2pResult });
+    }
+
+    for (const rtt of rtts) {
+        const name = `couchdb-rtt-${rtt}ms`;
+        const couchdbResult = await runBenchmark({
+            taskName: "bench:couchdb",
+            name,
+            outputDir,
+            env: {
+                ...base,
+                BENCH_CASE: name,
+                BENCH_COUCHDB_RTT_MS: String(rtt),
+            },
+        });
+        results.push({ name, runner: "couchdb", rttMs: rtt, result: couchdbResult });
+    }
+
+    const summary = {
+        generatedAt: new Date().toISOString(),
+        outputDir,
+        note:
+            "This sweep models additional remote CouchDB request latency through the existing HTTP proxy. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
+        rtts,
+        results,
+    };
+    await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
+    console.log(JSON.stringify(summary, null, 2));
+    console.log(`[latency-sweep] result directory: ${outputDir}`);
+}
+
+if (import.meta.main) {
+    main().catch((error) => {
+        console.error("[Fatal Error]", error);
+        Deno.exit(1);
+    });
+}

src/apps/cli/testdeno/bench-network-cases.ts

@@ -0,0 +1,198 @@
+type BenchmarkCase = {
+    name: string;
+    runner: "p2p" | "couchdb";
+    description: string;
+    dataPath: string;
+    trustBoundary: string;
+    env: Record<string, string>;
+};
+
+function readEnvString(name: string, fallback: string): string {
+    const value = Deno.env.get(name)?.trim();
+    return value && value.length > 0 ? value : fallback;
+}
+
+function timestamp(): string {
+    const d = new Date();
+    const pad = (n: number) => String(n).padStart(2, "0");
+    return (
+        `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
+        `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
+    );
+}
+
+function buildBaseEnv(): Record<string, string> {
+    return {
+        BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
+        BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
+        BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
+        BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
+        BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
+        BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
+        BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
+        BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
+        LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
+    };
+}
+
+function buildCases(): BenchmarkCase[] {
+    const base = buildBaseEnv();
+    const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20");
+    const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120");
+    const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478");
+
+    return [
+        {
+            name: "couchdb-baseline",
+            runner: "couchdb",
+            description: "Standard self-hosted CouchDB path through a local latency proxy.",
+            dataPath: "Device A -> CouchDB -> Device B",
+            trustBoundary: "CouchDB operator and network path",
+            env: {
+                ...base,
+                BENCH_CASE: "couchdb-baseline",
+                BENCH_COUCHDB_RTT_MS: couchdbRtt,
+            },
+        },
+        {
+            name: "p2p-direct-local",
+            runner: "p2p",
+            description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
+            dataPath: "Device A -> Device B",
+            trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
+            env: {
+                ...base,
+                BENCH_CASE: "p2p-direct-local",
+                BENCH_TURN_SERVERS: "",
+            },
+        },
+        {
+            name: "couchdb-tethering-vpn-proxy",
+            runner: "couchdb",
+            description:
+                "Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.",
+            dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
+            trustBoundary: "VPN/network path and CouchDB operator",
+            env: {
+                ...base,
+                BENCH_CASE: "couchdb-tethering-vpn-proxy",
+                BENCH_COUCHDB_RTT_MS: tetheringVpnRtt,
+            },
+        },
+        {
+            name: "p2p-smartphone-vpn-direct",
+            runner: "p2p",
+            description:
+                "Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.",
+            dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds",
+            trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata",
+            env: {
+                ...base,
+                BENCH_CASE: "p2p-smartphone-vpn-direct",
+                BENCH_TURN_SERVERS: "",
+            },
+        },
+        {
+            name: "p2p-user-turn",
+            runner: "p2p",
+            description: "Optional fallback path through a local user-controlled TURN server.",
+            dataPath: "Device A -> user-controlled TURN -> Device B",
+            trustBoundary: "User-controlled TURN server",
+            env: {
+                ...base,
+                BENCH_CASE: "p2p-user-turn",
+                BENCH_TURN_SERVERS: localTurnServers,
+            },
+        },
+    ];
+}
+
+async function runCase(testCase: BenchmarkCase, outputDir: string): Promise<Record<string, unknown>> {
+    const resultPath = `${outputDir}/${testCase.name}.json`;
+    const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb";
+    const env = {
+        ...Deno.env.toObject(),
+        ...testCase.env,
+        BENCH_RESULT_JSON: resultPath,
+    };
+
+    console.log(`[bench-cases] running ${testCase.name}: ${testCase.description}`);
+    const command = new Deno.Command("deno", {
+        args: ["task", taskName],
+        cwd: import.meta.dirname,
+        env,
+        stdin: "null",
+        stdout: "inherit",
+        stderr: "inherit",
+    });
+
+    const child = command.spawn();
+    const status = await child.status;
+    if (status.code !== 0) {
+        throw new Error(`case failed: ${testCase.name} (exit ${status.code})`);
+    }
+
+    const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
+    return {
+        ...testCase,
+        result,
+    };
+}
+
+function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
+    const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local");
+    const names = requested
+        .split(",")
+        .map((v) => v.trim())
+        .filter((v) => v.length > 0);
+    const byName = new Map(allCases.map((c) => [c.name, c]));
+    return names.map((name) => {
+        const found = byName.get(name);
+        if (!found) {
+            throw new Error(`Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`);
+        }
+        return found;
+    });
+}
+
+async function main(): Promise<void> {
+    const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`);
+    const outputDir = `${outRoot}/cases-${timestamp()}`;
+    await Deno.mkdir(outputDir, { recursive: true });
+
+    const allCases = buildCases();
+    const cases = selectCases(allCases);
+    await Deno.writeTextFile(
+        `${outputDir}/case-manifest.json`,
+        JSON.stringify(
+            {
+                generatedAt: new Date().toISOString(),
+                selectedCases: cases,
+                availableCases: allCases,
+            },
+            null,
+            2
+        )
+    );
+
+    const results: Record<string, unknown>[] = [];
+    for (const testCase of cases) {
+        results.push(await runCase(testCase, outputDir));
+    }
+
+    const summary = {
+        generatedAt: new Date().toISOString(),
+        outputDir,
+        results,
+    };
+    await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
+    console.log(JSON.stringify(summary, null, 2));
+    console.log(`[bench-cases] result directory: ${outputDir}`);
+}
+
+if (import.meta.main) {
+    main().catch((error) => {
+        console.error("[Fatal Error]", error);
+        Deno.exit(1);
+    });
+}

src/apps/cli/testdeno/bench-p2p.ts

@@ -1,15 +1,23 @@
 import { TempDir } from "./helpers/temp.ts";
 import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
 import { startCliInBackground } from "./helpers/backgroundCli.ts";
-import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts";
+import {
+    discoverPeer,
+    maybeStartCoturn,
+    maybeStartLocalRelay,
+    stopCoturnIfStarted,
+    stopLocalRelayIfStarted,
+} from "./helpers/p2p.ts";
 import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
 import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
 
 type BenchmarkConfig = {
+    caseName: string;
     relay: string;
     appId: string;
     roomId: string;
     passphrase: string;
+    turnServers: string;
     datasetDirName: string;
     datasetSeed: string;
     mdFileCount: number;
@@ -61,10 +69,12 @@ function formatBytes(value: number): string {
 
 function buildConfig(): BenchmarkConfig {
     return {
+        caseName: readEnvString("BENCH_CASE", "p2p-direct-local"),
         relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"),
         appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
         roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`),
         passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
+        turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
         datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
         datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
         mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
@@ -107,6 +117,7 @@ async function main(): Promise<void> {
     const resultPath = readOptionalResultPath();
 
     const relayStarted = await maybeStartLocalRelay(config.relay);
+    const coturnStarted = await maybeStartCoturn(config.turnServers);
     await using workDir = await TempDir.create("livesync-cli-p2p-bench");
 
     const hostVault = workDir.join("vault-host");
@@ -122,8 +133,24 @@ async function main(): Promise<void> {
     ]);
 
     await Promise.all([
-        applyP2pSettings(hostSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
+        applyP2pSettings(
-        applyP2pSettings(clientSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
+            hostSettings,
+            config.roomId,
+            config.passphrase,
+            config.appId,
+            config.relay,
+            "~.*",
+            config.turnServers
+        ),
+        applyP2pSettings(
+            clientSettings,
+            config.roomId,
+            config.passphrase,
+            config.appId,
+            config.relay,
+            "~.*",
+            config.turnServers
+        ),
     ]);
 
     await Promise.all([
@@ -179,8 +206,11 @@ async function main(): Promise<void> {
         }
 
         const result = {
+            caseName: config.caseName,
             mode: "p2p-cli-benchmark",
             relay: config.relay,
+            turnServers: config.turnServers,
+            turnEnabled: config.turnServers.trim().length > 0,
             appId: config.appId,
             roomId: config.roomId,
             datasetSeed: config.datasetSeed,
@@ -211,6 +241,7 @@ async function main(): Promise<void> {
         );
     } finally {
         await host.stop();
+        await stopCoturnIfStarted(coturnStarted);
         await stopLocalRelayIfStarted(relayStarted);
     }
 }

src/apps/cli/testdeno/deno.json

@@ -17,6 +17,8 @@
         "test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts",
         "bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts",
         "bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts",
+        "bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts",
+        "bench:latency-sweep": "deno run --env-file=.test.env -A --no-check bench-latency-sweep.ts",
         "bench:item1": "bash ./bench-run-item1.sh",
         "bench:item1:full": "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh",
         "test:e2e-couchdb": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-couchdb.ts",

src/apps/cli/testdeno/helpers/net.ts

@@ -9,7 +9,7 @@ function sleep(ms: number): Promise<void> {
 }
 
 async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise<void> {
-    let timer: number | undefined;
+    let timer: ReturnType<typeof setTimeout> | undefined;
     try {
         const connPromise = Deno.connect({ hostname, port });
         const timeoutPromise = new Promise<never>((_, reject) => {

src/apps/cli/testdeno/helpers/p2p.ts

@@ -76,8 +76,10 @@ export async function discoverPeer(
 }
 
 export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
-    if (!isLocalP2pRelay(relay)) return false;
+    const shouldStart = isLocalP2pRelay(relay);
-    await startP2pRelay();
+    if (shouldStart) {
+        await startP2pRelay();
+    }
     const endpoint = parseRelayEndpoint(relay);
     await waitForPort(endpoint.hostname, endpoint.port, {
         timeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000"),
@@ -86,8 +88,10 @@ export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
     });
     // Docker proxy accepts TCP connections instantly before the container's internal process is fully ready.
     // Wait an additional few seconds to ensure strfry is actually accepting WebSockets.
-    await sleep(3000);
+    if (shouldStart) {
-    return true;
+        await sleep(3000);
+    }
+    return shouldStart;
 }
 
 export async function stopLocalRelayIfStarted(started: boolean): Promise<void> {

test/bench-network/.gitignore

@@ -0,0 +1 @@
+bench-results/

test/bench-network/Dockerfile.runner

@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1
+
+FROM node:24-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl unzip python3 make g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV DENO_INSTALL=/usr/local
+RUN curl -fsSL https://deno.land/install.sh | sh
+
+WORKDIR /workspace
+
+COPY package.json package-lock.json ./
+COPY src/apps/cli/package.json ./src/apps/cli/package.json
+COPY src/apps/webapp/package.json ./src/apps/webapp/package.json
+COPY src/apps/webpeer/package.json ./src/apps/webpeer/package.json
+RUN npm ci
+
+COPY . .
+RUN npm run build -w self-hosted-livesync-cli
+
+WORKDIR /workspace/src/apps/cli/testdeno
+
+RUN deno cache --lock=deno.lock \
+    bench-network-cases.ts \
+    bench-latency-sweep.ts \
+    bench-p2p.ts \
+    bench-couchdb.ts
+
+COPY test/bench-network/run-bench.sh /usr/local/bin/run-livesync-bench
+RUN chmod +x /usr/local/bin/run-livesync-bench
+
+CMD ["run-livesync-bench"]

test/bench-network/README.md

@@ -0,0 +1,90 @@
+# Network benchmark package
+
+This directory packages the CLI benchmark cases with Docker Compose. It is
+intended for reproducible local benchmark runs where CouchDB, the Nostr
+signalling relay, optional TURN, and the benchmark runner are fixed by the
+Compose file.
+
+## Quick smoke run
+
+From the repository root:
+
+```bash
+docker compose -f test/bench-network/compose.yml run --rm bench-runner
+```
+
+By default this runs:
+
+- `couchdb-baseline`
+- `p2p-direct-local`
+
+The dataset is intentionally small by default. Results are written to
+`test/bench-network/bench-results/`.
+
+## GitHub Actions smoke run
+
+`.github/workflows/cli-p2p-compose-smoke.yml` provides a manual
+`workflow_dispatch` smoke run for the same Compose package. It is intentionally
+not a required check yet, because WebRTC peer discovery can still be slow or
+environment-sensitive on GitHub-hosted runners. Keep the dataset small and use
+the uploaded JSON artefact to inspect whether failures are caused by peer
+discovery, synchronisation, CouchDB startup, or Docker networking.
+
+## Select cases
+
+```bash
+BENCH_CASES=couchdb-baseline,p2p-direct-local,p2p-user-turn \
+docker compose -f test/bench-network/compose.yml --profile turn run --rm bench-runner
+```
+
+Available local cases:
+
+- `couchdb-baseline`
+- `p2p-direct-local`
+- `couchdb-tethering-vpn-proxy`
+- `p2p-smartphone-vpn-direct`
+- `p2p-user-turn`
+
+`p2p-smartphone-vpn-direct` is a structural case name. When it is run inside
+this Compose package it is not a real smartphone tethering/VPN measurement; it
+uses the local Compose network. Use it only for wiring checks unless the runner
+is executed in an actual tethered/VPN environment.
+
+## Dataset and latency controls
+
+```bash
+BENCH_MD_FILE_COUNT=100 \
+BENCH_MD_MIN_SIZE_BYTES=512 \
+BENCH_MD_MAX_SIZE_BYTES=2048 \
+BENCH_BIN_FILE_COUNT=25 \
+BENCH_BIN_SIZE_BYTES=8192 \
+BENCH_COUCHDB_RTT_MS=20 \
+BENCH_PEERS_TIMEOUT=60 \
+docker compose -f test/bench-network/compose.yml run --rm bench-runner
+```
+
+The current CouchDB latency model is the existing HTTP proxy inside
+`bench-couchdb.ts`. It models a remote database path with additional request
+latency, but it does not model packet loss, jitter, MTU, bandwidth limits,
+bufferbloat, or VPN encapsulation.
+
+## Latency sweep
+
+To run P2P once and CouchDB at several requested RTT values:
+
+```bash
+BENCH_COMMAND=latency-sweep \
+BENCH_SWEEP_RTT_MS=20,50,100,150,300 \
+BENCH_MD_FILE_COUNT=100 \
+BENCH_MD_MIN_SIZE_BYTES=512 \
+BENCH_MD_MAX_SIZE_BYTES=2048 \
+BENCH_BIN_FILE_COUNT=25 \
+BENCH_BIN_SIZE_BYTES=8192 \
+BENCH_SYNC_TIMEOUT=300 \
+BENCH_PEERS_TIMEOUT=60 \
+docker compose -f test/bench-network/compose.yml run --rm bench-runner
+```
+
+This sweep is useful for finding where the remote CouchDB path falls behind the
+local direct P2P path in the current HTTP-proxy latency model. It should not be
+presented as a full smartphone/VPN model.

test/bench-network/compose.yml

@@ -0,0 +1,93 @@
+services:
+  couchdb:
+    image: couchdb:3.5.0
+    environment:
+      COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin}
+      COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword}
+      COUCHDB_SINGLE_NODE: "true"
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -fsS -u ${BENCH_COUCHDB_USER:-admin}:${BENCH_COUCHDB_PASSWORD:-testpassword} http://127.0.0.1:5984/_up >/dev/null",
+        ]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+
+  nostr-relay:
+    image: ghcr.io/hoytech/strfry:latest
+    entrypoint: sh
+    command:
+      - -lc
+      - |
+        cat > /tmp/strfry.conf <<'EOF'
+        db = "./strfry-db/"
+
+        relay {
+          bind = "0.0.0.0"
+          port = 7777
+          nofiles = 100000
+
+          info {
+            name = "livesync bench relay"
+            description = "local relay for livesync compose benchmarks"
+          }
+
+          maxWebsocketPayloadSize = 131072
+          autoPingSeconds = 55
+
+          writePolicy {
+            plugin = ""
+          }
+        }
+        EOF
+        exec /app/strfry --config /tmp/strfry.conf relay
+    tmpfs:
+      - /app/strfry-db:rw,size=256m
+
+  coturn:
+    image: coturn/coturn:latest
+    command:
+      - --log-file=stdout
+      - --listening-port=3478
+      - --user=${BENCH_TURN_USERNAME:-testuser}:${BENCH_TURN_CREDENTIAL:-testpass}
+      - --realm=${BENCH_TURN_REALM:-livesync.test}
+    profiles:
+      - turn
+
+  bench-runner:
+    build:
+      context: ../..
+      dockerfile: test/bench-network/Dockerfile.runner
+    depends_on:
+      couchdb:
+        condition: service_healthy
+      nostr-relay:
+        condition: service_started
+    environment:
+      BENCH_COMMAND: ${BENCH_COMMAND:-cases}
+      BENCH_CASES: ${BENCH_CASES:-couchdb-baseline,p2p-direct-local}
+      BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results
+      BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results
+      BENCH_SWEEP_RTT_MS: ${BENCH_SWEEP_RTT_MS:-20,50,100,150,300}
+      BENCH_SWEEP_INCLUDE_P2P: ${BENCH_SWEEP_INCLUDE_P2P:-true}
+      BENCH_COUCHDB_MANAGED: "false"
+      BENCH_COUCHDB_BACKEND_URI: http://couchdb:5984
+      BENCH_COUCHDB_URI: http://127.0.0.1:15989
+      BENCH_COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin}
+      BENCH_COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword}
+      BENCH_RELAY: ws://nostr-relay:7777/
+      BENCH_LOCAL_TURN_SERVERS: turn:coturn:3478
+      BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20}
+      BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512}
+      BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
+      BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
+      BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
+      BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-20}
+      BENCH_TETHERING_VPN_RTT_MS: ${BENCH_TETHERING_VPN_RTT_MS:-120}
+      BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
+      BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
+      BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0}
+    volumes:
+      - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results

test/bench-network/run-bench.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+set -eu
+
+case "${BENCH_COMMAND:-cases}" in
+  cases)
+    exec deno task bench:cases
+    ;;
+  latency-sweep)
+    exec deno task bench:latency-sweep
+    ;;
+  *)
+    echo "Unknown BENCH_COMMAND: ${BENCH_COMMAND}" >&2
+    echo "Expected one of: cases, latency-sweep" >&2
+    exit 2
+    ;;
+esac