Strengthen CLI benchmark verification

src/apps/cli/testdeno/bench-couchdb.ts

@@ -11,8 +11,12 @@ import {
 } from "./helpers/docker.ts";
 import {
     createDeterministicDataset,
-    type DatasetEntry,
 } from "./helpers/dataset.ts";
+import {
+    type BenchmarkVerificationMode,
+    parseBenchmarkVerificationMode,
+    verifyBenchmarkDataset,
+} from "./helpers/benchmarkVerification.ts";
 
 type BenchmarkConfig = {
     caseName: string;
@@ -38,6 +42,9 @@ type BenchmarkConfig = {
     networkModel: string;
     measurementScope: string;
     limitations: string[];
+    verificationMode: BenchmarkVerificationMode;
+    repeatIndex: number;
+    repeatCount: number;
 };
 
 function readEnvString(name: string, fallback: string): string {
@@ -163,6 +170,11 @@ function buildConfig(): BenchmarkConfig {
         limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
             "This benchmark result is scoped to the configured dataset, remote store, and network model.",
         ]),
+        verificationMode: parseBenchmarkVerificationMode(
+            Deno.env.get("BENCH_VERIFY_MODE"),
+        ),
+        repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
+        repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
     };
 }
 
@@ -174,35 +186,27 @@ function readOptionalResultPath(): string | undefined {
     return raw;
 }
 
-function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
+export type CouchdbProxyHandle = {
-    if (entries.length === 0) {
-        return [];
-    }
-    const md = entries.find((e) => e.kind === "md");
-    const bin = entries.find((e) => e.kind === "bin");
-    const middle = entries[Math.floor(entries.length / 2)];
-    const last = entries[entries.length - 1];
-    const unique = new Map<string, DatasetEntry>();
-    for (const entry of [md, bin, middle, last]) {
-        if (entry) {
-            unique.set(entry.relativePath, entry);
-        }
-    }
-    return [...unique.values()];
-}
-
-type ProxyHandle = {
     stop: () => Promise<void>;
     applied: boolean;
     note: string;
+    directionalDelayMs: number;
 };
 
-function startCouchdbProxy(
+export function startCouchdbProxy(
-    options: { backendUri: string; proxyUri: string; requestedRttMs: number },
+    options: {
-): ProxyHandle {
+        backendUri: string;
+        proxyUri: string;
+        requestedRttMs: number;
+        delay?: (milliseconds: number) => Promise<void>;
+    },
+): CouchdbProxyHandle {
     const backend = new URL(options.backendUri);
     const proxy = new URL(options.proxyUri);
-    const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2));
+    const halfDelayMs = options.requestedRttMs / 2;
+    const delay = options.delay ??
+        ((milliseconds: number) =>
+            new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
     const controller = new AbortController();
 
     const listener = Deno.serve(
@@ -216,7 +220,7 @@ function startCouchdbProxy(
             },
         },
         async (request) => {
-            await new Promise((resolve) => setTimeout(resolve, halfDelayMs));
+            await delay(halfDelayMs);
 
             const targetUrl = new URL(request.url);
             targetUrl.protocol = backend.protocol;
@@ -245,6 +249,7 @@ function startCouchdbProxy(
             const responseHeaders = new Headers(upstream.headers);
             responseHeaders.delete("content-length");
             const responseBody = await upstream.arrayBuffer();
+            await delay(halfDelayMs);
 
             return new Response(responseBody, {
                 status: upstream.status,
@@ -256,8 +261,9 @@ function startCouchdbProxy(
 
     return {
         applied: true,
+        directionalDelayMs: halfDelayMs,
         note:
-            `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`,
+            `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms request-path and ${halfDelayMs}ms response-path delay`,
         stop: async () => {
             controller.abort();
             await listener.finished.catch(() => {});
@@ -350,25 +356,28 @@ async function main(): Promise<void> {
         await runCliOrFail(vaultB, "--settings", settingsB, "sync");
         const syncBElapsed = nowMs() - syncBStart;
 
-        const sampleFiles = pickSampleFiles(seedFiles.entries);
+        const verification = await verifyBenchmarkDataset(
-        for (const sample of sampleFiles) {
+            seedFiles.entries,
-            const pulledPath = workDir.join(
+            config.verificationMode,
-                `pulled-${sample.relativePath.split("/").join("_")}`,
+            async (entry) => {
-            );
+                const pulledPath = workDir.join(
-            await runCliOrFail(
+                    `pulled-${entry.relativePath.split("/").join("_")}`,
-                vaultB,
+                );
-                "--settings",
+                await runCliOrFail(
-                settingsB,
+                    vaultB,
-                "pull",
+                    "--settings",
-                sample.relativePath,
+                    settingsB,
-                pulledPath,
+                    "pull",
-            );
+                    entry.relativePath,
-            await assertFilesEqual(
+                    pulledPath,
-                sample.absolutePath,
+                );
-                pulledPath,
+                await assertFilesEqual(
-                `sample file mismatch after CouchDB sync: ${sample.relativePath}`,
+                    entry.absolutePath,
-            );
+                    pulledPath,
-        }
+                    `file mismatch after CouchDB sync: ${entry.relativePath}`,
+                );
+            },
+        );
 
         const result = {
             caseName: config.caseName,
@@ -382,15 +391,21 @@ async function main(): Promise<void> {
             networkModel: config.networkModel,
             measurementScope: config.measurementScope,
             limitations: config.limitations,
+            repeatIndex: config.repeatIndex,
+            repeatCount: config.repeatCount,
             rttRequestedMs: config.requestedRttMs,
             proxyApplied: proxy.applied,
             proxyNote: proxy.note,
+            proxyDirectionalDelayMs: proxy.directionalDelayMs,
+            proxyConfiguredRttMs: proxy.directionalDelayMs * 2,
+            proxyDelayApplication: "request-and-response",
             datasetSeed: config.datasetSeed,
             datasetDirName: config.datasetDirName,
             totalFiles: seedFiles.totalFiles,
             totalBytes: seedFiles.totalBytes,
             mdFileCount: seedFiles.mdCount,
             binFileCount: seedFiles.binCount,
+            ...verification,
             mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
             syncAElapsedMs: Number(syncAElapsed.toFixed(1)),
             syncBElapsedMs: Number(syncBElapsed.toFixed(1)),

src/apps/cli/testdeno/bench-latency-sweep.ts

@@ -2,6 +2,8 @@ type SweepResult = {
     name: string;
     runner: "p2p" | "couchdb";
     rttMs?: number;
+    repeatIndex: number;
+    repeatCount: number;
     result: Record<string, unknown>;
 };
 
@@ -19,6 +21,15 @@ function timestamp(): string {
     );
 }
 
+function readEnvInteger(name: string, fallback: number): number {
+    const raw = readEnvString(name, String(fallback));
+    const parsed = Number(raw);
+    if (!Number.isInteger(parsed) || parsed < 1) {
+        throw new Error(`${name} must be a positive integer, got '${raw}'`);
+    }
+    return parsed;
+}
+
 function parseRttList(raw: string): number[] {
     const values = raw
         .split(",")
@@ -41,6 +52,7 @@ function buildBaseEnv(): Record<string, string> {
         BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
         BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
         BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
+        BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"),
         LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
     };
 }
@@ -50,12 +62,19 @@ async function runBenchmark(options: {
     name: string;
     outputDir: string;
     env: Record<string, string>;
+    repeatIndex: number;
+    repeatCount: number;
 }): Promise<Record<string, unknown>> {
-    const resultPath = `${options.outputDir}/${options.name}.json`;
+    const suffix = options.repeatCount > 1
+        ? `-r${String(options.repeatIndex).padStart(2, "0")}`
+        : "";
+    const resultPath = `${options.outputDir}/${options.name}${suffix}.json`;
     const env = {
         ...Deno.env.toObject(),
         ...options.env,
         BENCH_RESULT_JSON: resultPath,
+        BENCH_REPEAT_INDEX: String(options.repeatIndex),
+        BENCH_REPEAT_COUNT: String(options.repeatCount),
     };
 
     console.log(`[latency-sweep] running ${options.name}`);
@@ -78,46 +97,69 @@ async function main(): Promise<void> {
     const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`);
     const outputDir = `${outRoot}/latency-sweep-${timestamp()}`;
     const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300"));
+    const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1);
     const base = buildBaseEnv();
 
     await Deno.mkdir(outputDir, { recursive: true });
 
     const results: SweepResult[] = [];
     if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") {
-        const p2pResult = await runBenchmark({
+        for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
-            taskName: "bench:p2p",
+            const p2pResult = await runBenchmark({
-            name: "p2p-direct-local",
+                taskName: "bench:p2p",
-            outputDir,
+                name: "p2p-direct-local",
-            env: {
+                outputDir,
-                ...base,
+                repeatIndex,
-                BENCH_CASE: "p2p-direct-local",
+                repeatCount,
-                BENCH_TURN_SERVERS: "",
+                env: {
-            },
+                    ...base,
-        });
+                    BENCH_CASE: "p2p-direct-local",
-        results.push({ name: "p2p-direct-local", runner: "p2p", result: p2pResult });
+                    BENCH_TURN_SERVERS: "",
+                },
+            });
+            results.push({
+                name: "p2p-direct-local",
+                runner: "p2p",
+                repeatIndex,
+                repeatCount,
+                result: p2pResult,
+            });
+        }
     }
 
     for (const rtt of rtts) {
         const name = `couchdb-rtt-${rtt}ms`;
-        const couchdbResult = await runBenchmark({
+        for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
-            taskName: "bench:couchdb",
+            const couchdbResult = await runBenchmark({
-            name,
+                taskName: "bench:couchdb",
-            outputDir,
+                name,
-            env: {
+                outputDir,
-                ...base,
+                repeatIndex,
-                BENCH_CASE: name,
+                repeatCount,
-                BENCH_COUCHDB_RTT_MS: String(rtt),
+                env: {
-            },
+                    ...base,
-        });
+                    BENCH_CASE: name,
-        results.push({ name, runner: "couchdb", rttMs: rtt, result: couchdbResult });
+                    BENCH_COUCHDB_RTT_MS: String(rtt),
+                },
+            });
+            results.push({
+                name,
+                runner: "couchdb",
+                rttMs: rtt,
+                repeatIndex,
+                repeatCount,
+                result: couchdbResult,
+            });
+        }
     }
 
     const summary = {
         generatedAt: new Date().toISOString(),
         outputDir,
         note:
-            "This sweep models additional remote CouchDB request latency through the existing HTTP proxy. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
+            "This sweep applies half of each requested CouchDB RTT before forwarding requests and half before returning responses. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
         rtts,
+        repeatCount,
         results,
     };
     await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));

src/apps/cli/testdeno/bench-network-cases.ts

@@ -52,6 +52,7 @@ function buildBaseEnv(): Record<string, string> {
         BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
         BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
         BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
+        BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"),
         LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
     };
 }
@@ -119,9 +120,10 @@ export function buildCases(): BenchmarkCase[] {
             dataPath: "Device A -> Device B",
             trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
             measurementScope:
-                "One CLI P2P synchronisation phase over a local WebRTC DataChannel after Nostr signalling, with TURN disabled.",
+                "One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment, with TURN disabled; the earlier peer-list observation command is excluded.",
             limitations: [
-                "This does not measure first-peer discovery latency, public relay operation, mobile carrier behaviour, or TURN-relayed throughput.",
+                "The timed command includes its own signalling and connection establishment, but not the earlier peer-list observation window.",
+                "This does not measure public relay operation, mobile carrier behaviour, or TURN-relayed throughput.",
                 "This small-dataset run should not be treated as a WAN, VPN, or large binary initial synchronisation measurement.",
             ],
             env: {
@@ -240,8 +242,9 @@ export function buildCases(): BenchmarkCase[] {
             trustBoundary:
                 "Nostr signalling metadata through constrained network shim; no TURN relay",
             measurementScope:
-                "Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the home-wifi netem profile.",
+                "One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the home-wifi netem profile; the selected WebRTC note-data path is unshaped.",
             limitations: [
+                "The timed p2p-sync command includes signalling and WebRTC connection establishment.",
                 "This does not shape the selected WebRTC DataChannel note-data path.",
                 "This supports only the claim that constrained signalling access does not place note data on the relay path when a non-relayed ICE path is selected.",
             ],
@@ -267,8 +270,9 @@ export function buildCases(): BenchmarkCase[] {
             trustBoundary:
                 "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
             measurementScope:
-                "Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the tethering-vpn netem profile.",
+                "One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the tethering-vpn netem profile; the selected WebRTC note-data path is unshaped.",
             limitations: [
+                "The timed p2p-sync command includes signalling and WebRTC connection establishment.",
                 "This does not shape the selected WebRTC DataChannel note-data path.",
                 "The profile approximates constrained relay access and is not a field measurement on a real tethered VPN connection.",
             ],

src/apps/cli/testdeno/bench-p2p.ts

@@ -15,8 +15,12 @@ import {
 import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
 import {
     createDeterministicDataset,
-    type DatasetEntry,
 } from "./helpers/dataset.ts";
+import {
+    type BenchmarkVerificationMode,
+    parseBenchmarkVerificationMode,
+    verifyBenchmarkDataset,
+} from "./helpers/benchmarkVerification.ts";
 
 type BenchmarkConfig = {
     caseName: string;
@@ -40,6 +44,9 @@ type BenchmarkConfig = {
     candidatePathVerification: string;
     measurementScope: string;
     limitations: string[];
+    verificationMode: BenchmarkVerificationMode;
+    repeatIndex: number;
+    repeatCount: number;
 };
 
 type P2PConnectionStats = {
@@ -174,11 +181,16 @@ function buildConfig(): BenchmarkConfig {
         ),
         measurementScope: readEnvString(
             "BENCH_MEASUREMENT_SCOPE",
-            "One CLI P2P synchronisation phase over WebRTC DataChannel after signalling.",
+            "One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment; the earlier peer-list observation command is excluded.",
         ),
         limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
             "This benchmark result is scoped to the configured dataset, network model, and selected ICE path.",
         ]),
+        verificationMode: parseBenchmarkVerificationMode(
+            Deno.env.get("BENCH_VERIFY_MODE"),
+        ),
+        repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
+        repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
     };
 }
 
@@ -190,23 +202,6 @@ function readOptionalResultPath(): string | undefined {
     return raw;
 }
 
-function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
-    if (entries.length === 0) {
-        return [];
-    }
-    const md = entries.find((e) => e.kind === "md");
-    const bin = entries.find((e) => e.kind === "bin");
-    const middle = entries[Math.floor(entries.length / 2)];
-    const last = entries[entries.length - 1];
-    const unique = new Map<string, DatasetEntry>();
-    for (const entry of [md, bin, middle, last]) {
-        if (entry) {
-            unique.set(entry.relativePath, entry);
-        }
-    }
-    return [...unique.values()];
-}
-
 async function readLatestP2PConnectionStats(
     statsPath: string,
 ): Promise<P2PConnectionStats | undefined> {
@@ -329,25 +324,28 @@ async function main(): Promise<void> {
             );
             const syncElapsed = nowMs() - syncStart;
 
-            const sampleFiles = pickSampleFiles(seedFiles.entries);
+            const verification = await verifyBenchmarkDataset(
-            for (const sample of sampleFiles) {
+                seedFiles.entries,
-                const pulledPath = workDir.join(
+                config.verificationMode,
-                    `pulled-${sample.relativePath.replaceAll("/", "_")}`,
+                async (entry) => {
-                );
+                    const pulledPath = workDir.join(
-                await runCliOrFail(
+                        `pulled-${entry.relativePath.replaceAll("/", "_")}`,
-                    clientVault,
+                    );
-                    "--settings",
+                    await runCliOrFail(
-                    clientSettings,
+                        clientVault,
-                    "pull",
+                        "--settings",
-                    sample.relativePath,
+                        clientSettings,
-                    pulledPath,
+                        "pull",
-                );
+                        entry.relativePath,
-                await assertFilesEqual(
+                        pulledPath,
-                    sample.absolutePath,
+                    );
-                    pulledPath,
+                    await assertFilesEqual(
-                    `sample file mismatch after sync: ${sample.relativePath}`,
+                        entry.absolutePath,
-                );
+                        pulledPath,
-            }
+                        `file mismatch after P2P sync: ${entry.relativePath}`,
+                    );
+                },
+            );
 
             const p2pConnectionStats = await readLatestP2PConnectionStats(
                 p2pStatsPath,
@@ -363,6 +361,8 @@ async function main(): Promise<void> {
                 networkModel: config.networkModel,
                 measurementScope: config.measurementScope,
                 limitations: config.limitations,
+                repeatIndex: config.repeatIndex,
+                repeatCount: config.repeatCount,
                 p2pCandidatePathVerified:
                     p2pConnectionStats?.candidatePathCollected === true,
                 p2pCandidatePathVerification:
@@ -385,6 +385,7 @@ async function main(): Promise<void> {
                 totalBytes: seedFiles.totalBytes,
                 mdFileCount: seedFiles.mdCount,
                 binFileCount: seedFiles.binCount,
+                ...verification,
                 mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
                 hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
                 peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,

src/apps/cli/testdeno/helpers/benchmarkVerification.ts

@@ -0,0 +1,80 @@
+import type { DatasetEntry } from "./dataset.ts";
+
+export type BenchmarkVerificationMode = "all" | "sample";
+
+export type BenchmarkVerificationResult = {
+    verificationMode: BenchmarkVerificationMode;
+    verifiedFiles: number;
+    verificationComplete: boolean;
+    datasetDigestSha256: string;
+};
+
+function toHex(bytes: ArrayBuffer): string {
+    return [...new Uint8Array(bytes)]
+        .map((value) => value.toString(16).padStart(2, "0"))
+        .join("");
+}
+
+async function sha256(bytes: Uint8Array): Promise<string> {
+    const input = new ArrayBuffer(bytes.byteLength);
+    new Uint8Array(input).set(bytes);
+    return toHex(await crypto.subtle.digest("SHA-256", input));
+}
+
+export function parseBenchmarkVerificationMode(
+    raw: string | undefined,
+    fallback: BenchmarkVerificationMode = "sample",
+): BenchmarkVerificationMode {
+    const value = raw?.trim().toLowerCase();
+    if (!value) return fallback;
+    if (value === "all" || value === "sample") return value;
+    throw new Error(`BENCH_VERIFY_MODE must be 'all' or 'sample', got '${raw}'`);
+}
+
+export function selectVerificationEntries(
+    entries: DatasetEntry[],
+    mode: BenchmarkVerificationMode,
+): DatasetEntry[] {
+    if (mode === "all" || entries.length === 0) return [...entries];
+
+    const md = entries.find((entry) => entry.kind === "md");
+    const bin = entries.find((entry) => entry.kind === "bin");
+    const middle = entries[Math.floor(entries.length / 2)];
+    const last = entries[entries.length - 1];
+    const selected = new Map<string, DatasetEntry>();
+    for (const entry of [md, bin, middle, last]) {
+        if (entry) selected.set(entry.relativePath, entry);
+    }
+    return [...selected.values()];
+}
+
+export async function computeDatasetDigestSha256(
+    entries: DatasetEntry[],
+): Promise<string> {
+    const manifest: string[] = [];
+    for (const entry of entries) {
+        const contentDigest = await sha256(await Deno.readFile(entry.absolutePath));
+        manifest.push(
+            `${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`,
+        );
+    }
+    return await sha256(new TextEncoder().encode(manifest.join("\n")));
+}
+
+export async function verifyBenchmarkDataset(
+    entries: DatasetEntry[],
+    mode: BenchmarkVerificationMode,
+    verifyEntry: (entry: DatasetEntry) => Promise<void>,
+): Promise<BenchmarkVerificationResult> {
+    const selected = selectVerificationEntries(entries, mode);
+    for (const entry of selected) {
+        await verifyEntry(entry);
+    }
+
+    return {
+        verificationMode: mode,
+        verifiedFiles: selected.length,
+        verificationComplete: selected.length === entries.length,
+        datasetDigestSha256: await computeDatasetDigestSha256(entries),
+    };
+}

src/apps/cli/testdeno/test-benchmark-contract.ts

@@ -1,5 +1,20 @@
 import { assert, assertEquals, assertStringIncludes } from "@std/assert";
 import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts";
+import { startCouchdbProxy } from "./bench-couchdb.ts";
+import {
+    parseBenchmarkVerificationMode,
+    selectVerificationEntries,
+} from "./helpers/benchmarkVerification.ts";
+import type { DatasetEntry } from "./helpers/dataset.ts";
+
+function getFreePort(): number {
+    const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 });
+    try {
+        return (listener.addr as Deno.NetAddr).port;
+    } finally {
+        listener.close();
+    }
+}
 
 function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase {
     const found = cases.find((testCase) => testCase.name === name);
@@ -56,9 +71,76 @@ Deno.test("benchmark cases record scope and limitations for paper use", () => {
             testCase.measurementScope,
         );
         assertEquals(parsedLimitations(testCase), testCase.limitations);
+        assertEquals(
+            testCase.env.BENCH_VERIFY_MODE,
+            "all",
+            `${testCase.name} must verify the complete dataset`,
+        );
     }
 });
 
+Deno.test("CouchDB latency proxy applies half the requested RTT in each direction", async () => {
+    const backendPort = getFreePort();
+    const proxyPort = getFreePort();
+    const delays: number[] = [];
+    const backend = Deno.serve(
+        {
+            hostname: "127.0.0.1",
+            port: backendPort,
+            onListen() {},
+        },
+        () => new Response("ok"),
+    );
+    const proxy = startCouchdbProxy({
+        backendUri: `http://127.0.0.1:${backendPort}`,
+        proxyUri: `http://127.0.0.1:${proxyPort}`,
+        requestedRttMs: 20,
+        delay: (milliseconds) => {
+            delays.push(milliseconds);
+            return Promise.resolve();
+        },
+    });
+
+    try {
+        const response = await fetch(`http://127.0.0.1:${proxyPort}/probe`);
+        assertEquals(await response.text(), "ok");
+        assertEquals(proxy.directionalDelayMs, 10);
+        assertEquals(delays, [10, 10]);
+    } finally {
+        await proxy.stop();
+        await backend.shutdown();
+    }
+
+    const halfMillisecondProxy = startCouchdbProxy({
+        backendUri: "http://127.0.0.1:1",
+        proxyUri: `http://127.0.0.1:${getFreePort()}`,
+        requestedRttMs: 1,
+        delay: () => Promise.resolve(),
+    });
+    try {
+        assertEquals(halfMillisecondProxy.directionalDelayMs, 0.5);
+    } finally {
+        await halfMillisecondProxy.stop();
+    }
+});
+
+Deno.test("benchmark verification mode selects either all files or a labelled sample", () => {
+    const entries: DatasetEntry[] = [
+        { kind: "md", relativePath: "a.md", absolutePath: "/a", size: 1 },
+        { kind: "md", relativePath: "b.md", absolutePath: "/b", size: 1 },
+        { kind: "bin", relativePath: "c.bin", absolutePath: "/c", size: 1 },
+        { kind: "md", relativePath: "d.md", absolutePath: "/d", size: 1 },
+        { kind: "bin", relativePath: "e.bin", absolutePath: "/e", size: 1 },
+    ];
+
+    assertEquals(parseBenchmarkVerificationMode("ALL"), "all");
+    assertEquals(selectVerificationEntries(entries, "all").length, entries.length);
+    const sample = selectVerificationEntries(entries, "sample");
+    assert(sample.length > 0 && sample.length < entries.length);
+    assert(sample.some((entry) => entry.kind === "md"));
+    assert(sample.some((entry) => entry.kind === "bin"));
+});
+
 Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => {
     const cases = buildCases();
     for (
@@ -77,6 +159,13 @@ Deno.test("P2P signalling-shim cases do not claim to shape the note-data path",
         );
         assertStringIncludes(testCase.dataPath, "WebRTC DataChannel");
         assertStringIncludes(testCase.dataPath, "Nostr signalling");
+        assertStringIncludes(testCase.measurementScope, "fresh CLI p2p-sync");
+        assert(
+            testCase.limitations.some((limitation) =>
+                limitation.includes("connection establishment")
+            ),
+            `${name} must state that connection establishment is timed`,
+        );
         assert(
             testCase.limitations.some((limitation) =>
                 limitation.includes("does not shape the selected WebRTC")

test/bench-network/README.md

@@ -64,16 +64,24 @@ path:
 | Case               | Data path                                   | What is measured                                                               | What is not measured                                                                                      |
 | ------------------ | ------------------------------------------- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------- |
 | `couchdb-baseline` | Device A -> CouchDB -> Device B             | Two one-shot CLI synchronisation commands through a local HTTP latency proxy   | Real WAN jitter, packet loss, bandwidth limits, VPN encapsulation, and server contention                  |
-| `p2p-direct-local` | Device A -> Device B after Nostr signalling | One CLI P2P synchronisation command over WebRTC DataChannel with TURN disabled | Public relay operation, mobile carrier behaviour, TURN relay throughput, and first-peer discovery latency |
+| `p2p-direct-local` | Device A -> Device B using Nostr signalling | One fresh CLI `p2p-sync` command, including process start-up and WebRTC connection establishment, with TURN disabled | Public relay operation, mobile carrier behaviour, and TURN relay throughput |
 
 Use the CouchDB result as the remote-store baseline and the P2P result as the
 direct-transfer comparison. The Nostr relay is used for signalling in the P2P
 case, but synchronised note content is transferred over the WebRTC DataChannel.
-The P2P result JSON records the selected WebRTC ICE candidate pair when the CLI
+The earlier `p2p-peers` observation command is excluded from the P2P timing,
+but the timed `p2p-sync` command performs its own signalling and connection
+establishment. The P2P result JSON records the selected WebRTC ICE candidate pair when the CLI
 can collect it from `RTCPeerConnection.getStats()`. Interpret P2P paths from
 the recorded candidate types rather than from TURN configuration alone. Do not
-report P2P runs as Tier 2 constrained-network measurements until host and
+report a signalling-only Tier 2 run as though the selected note-data path were
-client are captured under an equivalent shaped topology.
+also shaped.
+
+Benchmark cases use `BENCH_VERIFY_MODE=all` by default. After the timed phase,
+the runner retrieves and compares every generated file and records the verified
+file count, whether verification was complete, and a SHA-256 digest of the
+deterministic dataset. Set `BENCH_VERIFY_MODE=sample` only for exploratory
+large-dataset runs where the additional verification time is impractical.
 
 ## Dataset and latency controls
 
@@ -88,10 +96,10 @@ BENCH_PEERS_TIMEOUT=60 \
 docker compose -f test/bench-network/compose.yml run --rm bench-runner
 ```
 
-The current CouchDB latency model is the existing HTTP proxy inside
+The CouchDB latency model is the HTTP proxy inside `bench-couchdb.ts`. It adds
-`bench-couchdb.ts`. It models a remote database path with additional request
+half of the requested RTT before forwarding each request and the other half
-latency, but it does not model packet loss, jitter, MTU, bandwidth limits,
+before returning its response. It does not model packet loss, jitter, MTU,
-bufferbloat, or VPN encapsulation.
+bandwidth limits, bufferbloat, or VPN encapsulation.
 
 For P2P runs, `BENCH_PEERS_TIMEOUT` is passed to `p2p-peers`. That command waits
 for the requested observation window before printing discovered peers, so the
@@ -104,6 +112,7 @@ To run P2P once and CouchDB at several requested RTT values:
 ```bash
 BENCH_COMMAND=latency-sweep \
 BENCH_SWEEP_RTT_MS=20,50,100,150,300 \
+BENCH_REPEAT_COUNT=3 \
 BENCH_MD_FILE_COUNT=100 \
 BENCH_MD_MIN_SIZE_BYTES=512 \
 BENCH_MD_MAX_SIZE_BYTES=2048 \

test/bench-network/compose.yml

@@ -90,6 +90,7 @@ services:
       BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
       BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
       BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
+      BENCH_VERIFY_MODE: ${BENCH_VERIFY_MODE:-all}
       BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-20}
       BENCH_TETHERING_VPN_RTT_MS: ${BENCH_TETHERING_VPN_RTT_MS:-120}
       BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
@@ -220,6 +221,7 @@ services:
       BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
       BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
       BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
+      BENCH_VERIFY_MODE: ${BENCH_VERIFY_MODE:-all}
       BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
       BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
       LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000}