From a60932d9e4c54c64fc3369c8ac0080ab71d460f7 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 12 Jul 2026 10:54:16 +0000 Subject: [PATCH] Strengthen CLI benchmark verification --- src/apps/cli/testdeno/bench-couchdb.ts | 103 ++++++++++-------- src/apps/cli/testdeno/bench-latency-sweep.ts | 90 +++++++++++---- src/apps/cli/testdeno/bench-network-cases.ts | 12 +- src/apps/cli/testdeno/bench-p2p.ts | 77 ++++++------- .../testdeno/helpers/benchmarkVerification.ts | 80 ++++++++++++++ .../cli/testdeno/test-benchmark-contract.ts | 89 +++++++++++++++ test/bench-network/README.md | 25 +++-- test/bench-network/compose.yml | 2 + 8 files changed, 360 insertions(+), 118 deletions(-) create mode 100644 src/apps/cli/testdeno/helpers/benchmarkVerification.ts diff --git a/src/apps/cli/testdeno/bench-couchdb.ts b/src/apps/cli/testdeno/bench-couchdb.ts index cdf50d8..dfe78aa 100644 --- a/src/apps/cli/testdeno/bench-couchdb.ts +++ b/src/apps/cli/testdeno/bench-couchdb.ts @@ -11,8 +11,12 @@ import { } from "./helpers/docker.ts"; import { createDeterministicDataset, - type DatasetEntry, } from "./helpers/dataset.ts"; +import { + type BenchmarkVerificationMode, + parseBenchmarkVerificationMode, + verifyBenchmarkDataset, +} from "./helpers/benchmarkVerification.ts"; type BenchmarkConfig = { caseName: string; @@ -38,6 +42,9 @@ type BenchmarkConfig = { networkModel: string; measurementScope: string; limitations: string[]; + verificationMode: BenchmarkVerificationMode; + repeatIndex: number; + repeatCount: number; }; function readEnvString(name: string, fallback: string): string { @@ -163,6 +170,11 @@ function buildConfig(): BenchmarkConfig { limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [ "This benchmark result is scoped to the configured dataset, remote store, and network model.", ]), + verificationMode: parseBenchmarkVerificationMode( + Deno.env.get("BENCH_VERIFY_MODE"), + ), + repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)), + repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)), }; } @@ -174,35 +186,27 @@ function readOptionalResultPath(): string | undefined { return raw; } -function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] { - if (entries.length === 0) { - return []; - } - const md = entries.find((e) => e.kind === "md"); - const bin = entries.find((e) => e.kind === "bin"); - const middle = entries[Math.floor(entries.length / 2)]; - const last = entries[entries.length - 1]; - const unique = new Map(); - for (const entry of [md, bin, middle, last]) { - if (entry) { - unique.set(entry.relativePath, entry); - } - } - return [...unique.values()]; -} - -type ProxyHandle = { +export type CouchdbProxyHandle = { stop: () => Promise; applied: boolean; note: string; + directionalDelayMs: number; }; -function startCouchdbProxy( - options: { backendUri: string; proxyUri: string; requestedRttMs: number }, -): ProxyHandle { +export function startCouchdbProxy( + options: { + backendUri: string; + proxyUri: string; + requestedRttMs: number; + delay?: (milliseconds: number) => Promise; + }, +): CouchdbProxyHandle { const backend = new URL(options.backendUri); const proxy = new URL(options.proxyUri); - const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2)); + const halfDelayMs = options.requestedRttMs / 2; + const delay = options.delay ?? + ((milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds))); const controller = new AbortController(); const listener = Deno.serve( @@ -216,7 +220,7 @@ function startCouchdbProxy( }, }, async (request) => { - await new Promise((resolve) => setTimeout(resolve, halfDelayMs)); + await delay(halfDelayMs); const targetUrl = new URL(request.url); targetUrl.protocol = backend.protocol; @@ -245,6 +249,7 @@ function startCouchdbProxy( const responseHeaders = new Headers(upstream.headers); responseHeaders.delete("content-length"); const responseBody = await upstream.arrayBuffer(); + await delay(halfDelayMs); return new Response(responseBody, { status: upstream.status, @@ -256,8 +261,9 @@ function startCouchdbProxy( return { applied: true, + directionalDelayMs: halfDelayMs, note: - `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`, + `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms request-path and ${halfDelayMs}ms response-path delay`, stop: async () => { controller.abort(); await listener.finished.catch(() => {}); @@ -350,25 +356,28 @@ async function main(): Promise { await runCliOrFail(vaultB, "--settings", settingsB, "sync"); const syncBElapsed = nowMs() - syncBStart; - const sampleFiles = pickSampleFiles(seedFiles.entries); - for (const sample of sampleFiles) { - const pulledPath = workDir.join( - `pulled-${sample.relativePath.split("/").join("_")}`, - ); - await runCliOrFail( - vaultB, - "--settings", - settingsB, - "pull", - sample.relativePath, - pulledPath, - ); - await assertFilesEqual( - sample.absolutePath, - pulledPath, - `sample file mismatch after CouchDB sync: ${sample.relativePath}`, - ); - } + const verification = await verifyBenchmarkDataset( + seedFiles.entries, + config.verificationMode, + async (entry) => { + const pulledPath = workDir.join( + `pulled-${entry.relativePath.split("/").join("_")}`, + ); + await runCliOrFail( + vaultB, + "--settings", + settingsB, + "pull", + entry.relativePath, + pulledPath, + ); + await assertFilesEqual( + entry.absolutePath, + pulledPath, + `file mismatch after CouchDB sync: ${entry.relativePath}`, + ); + }, + ); const result = { caseName: config.caseName, @@ -382,15 +391,21 @@ async function main(): Promise { networkModel: config.networkModel, measurementScope: config.measurementScope, limitations: config.limitations, + repeatIndex: config.repeatIndex, + repeatCount: config.repeatCount, rttRequestedMs: config.requestedRttMs, proxyApplied: proxy.applied, proxyNote: proxy.note, + proxyDirectionalDelayMs: proxy.directionalDelayMs, + proxyConfiguredRttMs: proxy.directionalDelayMs * 2, + proxyDelayApplication: "request-and-response", datasetSeed: config.datasetSeed, datasetDirName: config.datasetDirName, totalFiles: seedFiles.totalFiles, totalBytes: seedFiles.totalBytes, mdFileCount: seedFiles.mdCount, binFileCount: seedFiles.binCount, + ...verification, mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), syncAElapsedMs: Number(syncAElapsed.toFixed(1)), syncBElapsedMs: Number(syncBElapsed.toFixed(1)), diff --git a/src/apps/cli/testdeno/bench-latency-sweep.ts b/src/apps/cli/testdeno/bench-latency-sweep.ts index d69de28..3624b1d 100644 --- a/src/apps/cli/testdeno/bench-latency-sweep.ts +++ b/src/apps/cli/testdeno/bench-latency-sweep.ts @@ -2,6 +2,8 @@ type SweepResult = { name: string; runner: "p2p" | "couchdb"; rttMs?: number; + repeatIndex: number; + repeatCount: number; result: Record; }; @@ -19,6 +21,15 @@ function timestamp(): string { ); } +function readEnvInteger(name: string, fallback: number): number { + const raw = readEnvString(name, String(fallback)); + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${name} must be a positive integer, got '${raw}'`); + } + return parsed; +} + function parseRttList(raw: string): number[] { const values = raw .split(",") @@ -41,6 +52,7 @@ function buildBaseEnv(): Record { BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"), BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"), BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), + BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"), LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"), }; } @@ -50,12 +62,19 @@ async function runBenchmark(options: { name: string; outputDir: string; env: Record; + repeatIndex: number; + repeatCount: number; }): Promise> { - const resultPath = `${options.outputDir}/${options.name}.json`; + const suffix = options.repeatCount > 1 + ? `-r${String(options.repeatIndex).padStart(2, "0")}` + : ""; + const resultPath = `${options.outputDir}/${options.name}${suffix}.json`; const env = { ...Deno.env.toObject(), ...options.env, BENCH_RESULT_JSON: resultPath, + BENCH_REPEAT_INDEX: String(options.repeatIndex), + BENCH_REPEAT_COUNT: String(options.repeatCount), }; console.log(`[latency-sweep] running ${options.name}`); @@ -78,46 +97,69 @@ async function main(): Promise { const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`); const outputDir = `${outRoot}/latency-sweep-${timestamp()}`; const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300")); + const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1); const base = buildBaseEnv(); await Deno.mkdir(outputDir, { recursive: true }); const results: SweepResult[] = []; if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") { - const p2pResult = await runBenchmark({ - taskName: "bench:p2p", - name: "p2p-direct-local", - outputDir, - env: { - ...base, - BENCH_CASE: "p2p-direct-local", - BENCH_TURN_SERVERS: "", - }, - }); - results.push({ name: "p2p-direct-local", runner: "p2p", result: p2pResult }); + for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) { + const p2pResult = await runBenchmark({ + taskName: "bench:p2p", + name: "p2p-direct-local", + outputDir, + repeatIndex, + repeatCount, + env: { + ...base, + BENCH_CASE: "p2p-direct-local", + BENCH_TURN_SERVERS: "", + }, + }); + results.push({ + name: "p2p-direct-local", + runner: "p2p", + repeatIndex, + repeatCount, + result: p2pResult, + }); + } } for (const rtt of rtts) { const name = `couchdb-rtt-${rtt}ms`; - const couchdbResult = await runBenchmark({ - taskName: "bench:couchdb", - name, - outputDir, - env: { - ...base, - BENCH_CASE: name, - BENCH_COUCHDB_RTT_MS: String(rtt), - }, - }); - results.push({ name, runner: "couchdb", rttMs: rtt, result: couchdbResult }); + for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) { + const couchdbResult = await runBenchmark({ + taskName: "bench:couchdb", + name, + outputDir, + repeatIndex, + repeatCount, + env: { + ...base, + BENCH_CASE: name, + BENCH_COUCHDB_RTT_MS: String(rtt), + }, + }); + results.push({ + name, + runner: "couchdb", + rttMs: rtt, + repeatIndex, + repeatCount, + result: couchdbResult, + }); + } } const summary = { generatedAt: new Date().toISOString(), outputDir, note: - "This sweep models additional remote CouchDB request latency through the existing HTTP proxy. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.", + "This sweep applies half of each requested CouchDB RTT before forwarding requests and half before returning responses. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.", rtts, + repeatCount, results, }; await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2)); diff --git a/src/apps/cli/testdeno/bench-network-cases.ts b/src/apps/cli/testdeno/bench-network-cases.ts index 689939a..5795800 100644 --- a/src/apps/cli/testdeno/bench-network-cases.ts +++ b/src/apps/cli/testdeno/bench-network-cases.ts @@ -52,6 +52,7 @@ function buildBaseEnv(): Record { BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"), BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"), BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), + BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"), LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"), }; } @@ -119,9 +120,10 @@ export function buildCases(): BenchmarkCase[] { dataPath: "Device A -> Device B", trustBoundary: "Nostr relay for signalling metadata; no TURN relay", measurementScope: - "One CLI P2P synchronisation phase over a local WebRTC DataChannel after Nostr signalling, with TURN disabled.", + "One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment, with TURN disabled; the earlier peer-list observation command is excluded.", limitations: [ - "This does not measure first-peer discovery latency, public relay operation, mobile carrier behaviour, or TURN-relayed throughput.", + "The timed command includes its own signalling and connection establishment, but not the earlier peer-list observation window.", + "This does not measure public relay operation, mobile carrier behaviour, or TURN-relayed throughput.", "This small-dataset run should not be treated as a WAN, VPN, or large binary initial synchronisation measurement.", ], env: { @@ -240,8 +242,9 @@ export function buildCases(): BenchmarkCase[] { trustBoundary: "Nostr signalling metadata through constrained network shim; no TURN relay", measurementScope: - "Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the home-wifi netem profile.", + "One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the home-wifi netem profile; the selected WebRTC note-data path is unshaped.", limitations: [ + "The timed p2p-sync command includes signalling and WebRTC connection establishment.", "This does not shape the selected WebRTC DataChannel note-data path.", "This supports only the claim that constrained signalling access does not place note data on the relay path when a non-relayed ICE path is selected.", ], @@ -267,8 +270,9 @@ export function buildCases(): BenchmarkCase[] { trustBoundary: "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay", measurementScope: - "Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the tethering-vpn netem profile.", + "One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the tethering-vpn netem profile; the selected WebRTC note-data path is unshaped.", limitations: [ + "The timed p2p-sync command includes signalling and WebRTC connection establishment.", "This does not shape the selected WebRTC DataChannel note-data path.", "The profile approximates constrained relay access and is not a field measurement on a real tethered VPN connection.", ], diff --git a/src/apps/cli/testdeno/bench-p2p.ts b/src/apps/cli/testdeno/bench-p2p.ts index e763e9d..0551269 100644 --- a/src/apps/cli/testdeno/bench-p2p.ts +++ b/src/apps/cli/testdeno/bench-p2p.ts @@ -15,8 +15,12 @@ import { import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; import { createDeterministicDataset, - type DatasetEntry, } from "./helpers/dataset.ts"; +import { + type BenchmarkVerificationMode, + parseBenchmarkVerificationMode, + verifyBenchmarkDataset, +} from "./helpers/benchmarkVerification.ts"; type BenchmarkConfig = { caseName: string; @@ -40,6 +44,9 @@ type BenchmarkConfig = { candidatePathVerification: string; measurementScope: string; limitations: string[]; + verificationMode: BenchmarkVerificationMode; + repeatIndex: number; + repeatCount: number; }; type P2PConnectionStats = { @@ -174,11 +181,16 @@ function buildConfig(): BenchmarkConfig { ), measurementScope: readEnvString( "BENCH_MEASUREMENT_SCOPE", - "One CLI P2P synchronisation phase over WebRTC DataChannel after signalling.", + "One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment; the earlier peer-list observation command is excluded.", ), limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [ "This benchmark result is scoped to the configured dataset, network model, and selected ICE path.", ]), + verificationMode: parseBenchmarkVerificationMode( + Deno.env.get("BENCH_VERIFY_MODE"), + ), + repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)), + repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)), }; } @@ -190,23 +202,6 @@ function readOptionalResultPath(): string | undefined { return raw; } -function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] { - if (entries.length === 0) { - return []; - } - const md = entries.find((e) => e.kind === "md"); - const bin = entries.find((e) => e.kind === "bin"); - const middle = entries[Math.floor(entries.length / 2)]; - const last = entries[entries.length - 1]; - const unique = new Map(); - for (const entry of [md, bin, middle, last]) { - if (entry) { - unique.set(entry.relativePath, entry); - } - } - return [...unique.values()]; -} - async function readLatestP2PConnectionStats( statsPath: string, ): Promise { @@ -329,25 +324,28 @@ async function main(): Promise { ); const syncElapsed = nowMs() - syncStart; - const sampleFiles = pickSampleFiles(seedFiles.entries); - for (const sample of sampleFiles) { - const pulledPath = workDir.join( - `pulled-${sample.relativePath.replaceAll("/", "_")}`, - ); - await runCliOrFail( - clientVault, - "--settings", - clientSettings, - "pull", - sample.relativePath, - pulledPath, - ); - await assertFilesEqual( - sample.absolutePath, - pulledPath, - `sample file mismatch after sync: ${sample.relativePath}`, - ); - } + const verification = await verifyBenchmarkDataset( + seedFiles.entries, + config.verificationMode, + async (entry) => { + const pulledPath = workDir.join( + `pulled-${entry.relativePath.replaceAll("/", "_")}`, + ); + await runCliOrFail( + clientVault, + "--settings", + clientSettings, + "pull", + entry.relativePath, + pulledPath, + ); + await assertFilesEqual( + entry.absolutePath, + pulledPath, + `file mismatch after P2P sync: ${entry.relativePath}`, + ); + }, + ); const p2pConnectionStats = await readLatestP2PConnectionStats( p2pStatsPath, @@ -363,6 +361,8 @@ async function main(): Promise { networkModel: config.networkModel, measurementScope: config.measurementScope, limitations: config.limitations, + repeatIndex: config.repeatIndex, + repeatCount: config.repeatCount, p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true, p2pCandidatePathVerification: @@ -385,6 +385,7 @@ async function main(): Promise { totalBytes: seedFiles.totalBytes, mdFileCount: seedFiles.mdCount, binFileCount: seedFiles.binCount, + ...verification, mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)), peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds, diff --git a/src/apps/cli/testdeno/helpers/benchmarkVerification.ts b/src/apps/cli/testdeno/helpers/benchmarkVerification.ts new file mode 100644 index 0000000..ace22ed --- /dev/null +++ b/src/apps/cli/testdeno/helpers/benchmarkVerification.ts @@ -0,0 +1,80 @@ +import type { DatasetEntry } from "./dataset.ts"; + +export type BenchmarkVerificationMode = "all" | "sample"; + +export type BenchmarkVerificationResult = { + verificationMode: BenchmarkVerificationMode; + verifiedFiles: number; + verificationComplete: boolean; + datasetDigestSha256: string; +}; + +function toHex(bytes: ArrayBuffer): string { + return [...new Uint8Array(bytes)] + .map((value) => value.toString(16).padStart(2, "0")) + .join(""); +} + +async function sha256(bytes: Uint8Array): Promise { + const input = new ArrayBuffer(bytes.byteLength); + new Uint8Array(input).set(bytes); + return toHex(await crypto.subtle.digest("SHA-256", input)); +} + +export function parseBenchmarkVerificationMode( + raw: string | undefined, + fallback: BenchmarkVerificationMode = "sample", +): BenchmarkVerificationMode { + const value = raw?.trim().toLowerCase(); + if (!value) return fallback; + if (value === "all" || value === "sample") return value; + throw new Error(`BENCH_VERIFY_MODE must be 'all' or 'sample', got '${raw}'`); +} + +export function selectVerificationEntries( + entries: DatasetEntry[], + mode: BenchmarkVerificationMode, +): DatasetEntry[] { + if (mode === "all" || entries.length === 0) return [...entries]; + + const md = entries.find((entry) => entry.kind === "md"); + const bin = entries.find((entry) => entry.kind === "bin"); + const middle = entries[Math.floor(entries.length / 2)]; + const last = entries[entries.length - 1]; + const selected = new Map(); + for (const entry of [md, bin, middle, last]) { + if (entry) selected.set(entry.relativePath, entry); + } + return [...selected.values()]; +} + +export async function computeDatasetDigestSha256( + entries: DatasetEntry[], +): Promise { + const manifest: string[] = []; + for (const entry of entries) { + const contentDigest = await sha256(await Deno.readFile(entry.absolutePath)); + manifest.push( + `${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`, + ); + } + return await sha256(new TextEncoder().encode(manifest.join("\n"))); +} + +export async function verifyBenchmarkDataset( + entries: DatasetEntry[], + mode: BenchmarkVerificationMode, + verifyEntry: (entry: DatasetEntry) => Promise, +): Promise { + const selected = selectVerificationEntries(entries, mode); + for (const entry of selected) { + await verifyEntry(entry); + } + + return { + verificationMode: mode, + verifiedFiles: selected.length, + verificationComplete: selected.length === entries.length, + datasetDigestSha256: await computeDatasetDigestSha256(entries), + }; +} diff --git a/src/apps/cli/testdeno/test-benchmark-contract.ts b/src/apps/cli/testdeno/test-benchmark-contract.ts index a2ad593..6b18921 100644 --- a/src/apps/cli/testdeno/test-benchmark-contract.ts +++ b/src/apps/cli/testdeno/test-benchmark-contract.ts @@ -1,5 +1,20 @@ import { assert, assertEquals, assertStringIncludes } from "@std/assert"; import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts"; +import { startCouchdbProxy } from "./bench-couchdb.ts"; +import { + parseBenchmarkVerificationMode, + selectVerificationEntries, +} from "./helpers/benchmarkVerification.ts"; +import type { DatasetEntry } from "./helpers/dataset.ts"; + +function getFreePort(): number { + const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); + try { + return (listener.addr as Deno.NetAddr).port; + } finally { + listener.close(); + } +} function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase { const found = cases.find((testCase) => testCase.name === name); @@ -56,9 +71,76 @@ Deno.test("benchmark cases record scope and limitations for paper use", () => { testCase.measurementScope, ); assertEquals(parsedLimitations(testCase), testCase.limitations); + assertEquals( + testCase.env.BENCH_VERIFY_MODE, + "all", + `${testCase.name} must verify the complete dataset`, + ); } }); +Deno.test("CouchDB latency proxy applies half the requested RTT in each direction", async () => { + const backendPort = getFreePort(); + const proxyPort = getFreePort(); + const delays: number[] = []; + const backend = Deno.serve( + { + hostname: "127.0.0.1", + port: backendPort, + onListen() {}, + }, + () => new Response("ok"), + ); + const proxy = startCouchdbProxy({ + backendUri: `http://127.0.0.1:${backendPort}`, + proxyUri: `http://127.0.0.1:${proxyPort}`, + requestedRttMs: 20, + delay: (milliseconds) => { + delays.push(milliseconds); + return Promise.resolve(); + }, + }); + + try { + const response = await fetch(`http://127.0.0.1:${proxyPort}/probe`); + assertEquals(await response.text(), "ok"); + assertEquals(proxy.directionalDelayMs, 10); + assertEquals(delays, [10, 10]); + } finally { + await proxy.stop(); + await backend.shutdown(); + } + + const halfMillisecondProxy = startCouchdbProxy({ + backendUri: "http://127.0.0.1:1", + proxyUri: `http://127.0.0.1:${getFreePort()}`, + requestedRttMs: 1, + delay: () => Promise.resolve(), + }); + try { + assertEquals(halfMillisecondProxy.directionalDelayMs, 0.5); + } finally { + await halfMillisecondProxy.stop(); + } +}); + +Deno.test("benchmark verification mode selects either all files or a labelled sample", () => { + const entries: DatasetEntry[] = [ + { kind: "md", relativePath: "a.md", absolutePath: "/a", size: 1 }, + { kind: "md", relativePath: "b.md", absolutePath: "/b", size: 1 }, + { kind: "bin", relativePath: "c.bin", absolutePath: "/c", size: 1 }, + { kind: "md", relativePath: "d.md", absolutePath: "/d", size: 1 }, + { kind: "bin", relativePath: "e.bin", absolutePath: "/e", size: 1 }, + ]; + + assertEquals(parseBenchmarkVerificationMode("ALL"), "all"); + assertEquals(selectVerificationEntries(entries, "all").length, entries.length); + const sample = selectVerificationEntries(entries, "sample"); + assert(sample.length > 0 && sample.length < entries.length); + assert(sample.some((entry) => entry.kind === "md")); + assert(sample.some((entry) => entry.kind === "bin")); +}); + Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => { const cases = buildCases(); for ( @@ -77,6 +159,13 @@ Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", ); assertStringIncludes(testCase.dataPath, "WebRTC DataChannel"); assertStringIncludes(testCase.dataPath, "Nostr signalling"); + assertStringIncludes(testCase.measurementScope, "fresh CLI p2p-sync"); + assert( + testCase.limitations.some((limitation) => + limitation.includes("connection establishment") + ), + `${name} must state that connection establishment is timed`, + ); assert( testCase.limitations.some((limitation) => limitation.includes("does not shape the selected WebRTC") diff --git a/test/bench-network/README.md b/test/bench-network/README.md index 5b93a64..330b48a 100644 --- a/test/bench-network/README.md +++ b/test/bench-network/README.md @@ -64,16 +64,24 @@ path: | Case | Data path | What is measured | What is not measured | | ------------------ | ------------------------------------------- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------- | | `couchdb-baseline` | Device A -> CouchDB -> Device B | Two one-shot CLI synchronisation commands through a local HTTP latency proxy | Real WAN jitter, packet loss, bandwidth limits, VPN encapsulation, and server contention | -| `p2p-direct-local` | Device A -> Device B after Nostr signalling | One CLI P2P synchronisation command over WebRTC DataChannel with TURN disabled | Public relay operation, mobile carrier behaviour, TURN relay throughput, and first-peer discovery latency | +| `p2p-direct-local` | Device A -> Device B using Nostr signalling | One fresh CLI `p2p-sync` command, including process start-up and WebRTC connection establishment, with TURN disabled | Public relay operation, mobile carrier behaviour, and TURN relay throughput | Use the CouchDB result as the remote-store baseline and the P2P result as the direct-transfer comparison. The Nostr relay is used for signalling in the P2P case, but synchronised note content is transferred over the WebRTC DataChannel. -The P2P result JSON records the selected WebRTC ICE candidate pair when the CLI +The earlier `p2p-peers` observation command is excluded from the P2P timing, +but the timed `p2p-sync` command performs its own signalling and connection +establishment. The P2P result JSON records the selected WebRTC ICE candidate pair when the CLI can collect it from `RTCPeerConnection.getStats()`. Interpret P2P paths from the recorded candidate types rather than from TURN configuration alone. Do not -report P2P runs as Tier 2 constrained-network measurements until host and -client are captured under an equivalent shaped topology. +report a signalling-only Tier 2 run as though the selected note-data path were +also shaped. + +Benchmark cases use `BENCH_VERIFY_MODE=all` by default. After the timed phase, +the runner retrieves and compares every generated file and records the verified +file count, whether verification was complete, and a SHA-256 digest of the +deterministic dataset. Set `BENCH_VERIFY_MODE=sample` only for exploratory +large-dataset runs where the additional verification time is impractical. ## Dataset and latency controls @@ -88,10 +96,10 @@ BENCH_PEERS_TIMEOUT=60 \ docker compose -f test/bench-network/compose.yml run --rm bench-runner ``` -The current CouchDB latency model is the existing HTTP proxy inside -`bench-couchdb.ts`. It models a remote database path with additional request -latency, but it does not model packet loss, jitter, MTU, bandwidth limits, -bufferbloat, or VPN encapsulation. +The CouchDB latency model is the HTTP proxy inside `bench-couchdb.ts`. It adds +half of the requested RTT before forwarding each request and the other half +before returning its response. It does not model packet loss, jitter, MTU, +bandwidth limits, bufferbloat, or VPN encapsulation. For P2P runs, `BENCH_PEERS_TIMEOUT` is passed to `p2p-peers`. That command waits for the requested observation window before printing discovered peers, so the @@ -104,6 +112,7 @@ To run P2P once and CouchDB at several requested RTT values: ```bash BENCH_COMMAND=latency-sweep \ BENCH_SWEEP_RTT_MS=20,50,100,150,300 \ +BENCH_REPEAT_COUNT=3 \ BENCH_MD_FILE_COUNT=100 \ BENCH_MD_MIN_SIZE_BYTES=512 \ BENCH_MD_MAX_SIZE_BYTES=2048 \ diff --git a/test/bench-network/compose.yml b/test/bench-network/compose.yml index db9d621..7add3d3 100644 --- a/test/bench-network/compose.yml +++ b/test/bench-network/compose.yml @@ -90,6 +90,7 @@ services: BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048} BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5} BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192} + BENCH_VERIFY_MODE: ${BENCH_VERIFY_MODE:-all} BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-20} BENCH_TETHERING_VPN_RTT_MS: ${BENCH_TETHERING_VPN_RTT_MS:-120} BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300} @@ -220,6 +221,7 @@ services: BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048} BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5} BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192} + BENCH_VERIFY_MODE: ${BENCH_VERIFY_MODE:-all} BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300} BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60} LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000}