diff --git a/docs/terms.md b/docs/terms.md index ac34ef0..c8809c8 100644 --- a/docs/terms.md +++ b/docs/terms.md @@ -65,7 +65,7 @@ All guidelines and conventions listed below are disclosed and maintained solely - livesync-serverpeer / webpeer - Pseudo-clients that assist in WebRTC peer-to-peer communication. - Metadata (File metadata) - - A database document that stores properties of a file, including its filename, path, size, modification time, conflict history, and references (hashes) of the chunks that comprise the file's content. In Self-hosted LiveSync, metadata is stored separately from the actual file content to enable efficient synchronisation and versioning. + - A database document that stores properties of a file, including its filename, path, size, modification time, and references (hashes) of the chunks that comprise the file's content. Conflict state is carried by the surrounding PouchDB/CouchDB revision metadata rather than by a separate history field inside the file metadata document. In Self-hosted LiveSync, file metadata is stored separately from the actual file content to enable efficient synchronisation and versioning. - OneShot Sync - A single, immediate bidirectional synchronisation (pull then push) triggered on demand or on specific events, as opposed to continuous (live) replication. - Overwrite Server Data with This Device's Files @@ -100,4 +100,3 @@ All guidelines and conventions listed below are disclosed and maintained solely - An optimisation that groups multiple local file edits together over a short delay before committing them to the local database, reducing the number of database write operations. - WebRTC P2P (Peer-to-Peer) - A synchronisation method enabling direct communication between devices without a central server database. - diff --git a/src/apps/cli/testdeno/bench-couchdb.ts b/src/apps/cli/testdeno/bench-couchdb.ts index d337efa..cdf50d8 100644 --- a/src/apps/cli/testdeno/bench-couchdb.ts +++ b/src/apps/cli/testdeno/bench-couchdb.ts @@ -1,8 +1,18 @@ import { TempDir } from "./helpers/temp.ts"; -import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; +import { + applyRemoteSyncSettings, + initSettingsFile, +} from "./helpers/settings.ts"; import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; -import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts"; -import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts"; +import { + createCouchdbDatabase, + startCouchdb, + stopCouchdb, +} from "./helpers/docker.ts"; +import { + createDeterministicDataset, + type DatasetEntry, +} from "./helpers/dataset.ts"; type BenchmarkConfig = { caseName: string; @@ -26,6 +36,8 @@ type BenchmarkConfig = { simulationTier: string; networkProfile: string; networkModel: string; + measurementScope: string; + limitations: string[]; }; function readEnvString(name: string, fallback: string): string { @@ -54,6 +66,30 @@ function readEnvBool(name: string, fallback: boolean): boolean { return /^(1|true|yes|on)$/i.test(raw.trim()); } +function readEnvStringArray(name: string, fallback: string[]): string[] { + const raw = Deno.env.get(name)?.trim(); + if (!raw) { + return fallback; + } + + try { + const parsed = JSON.parse(raw); + if ( + Array.isArray(parsed) && + parsed.every((item) => typeof item === "string") + ) { + return parsed; + } + } catch { + // Fall through to pipe-separated parsing for hand-written invocations. + } + + return raw + .split("|") + .map((item) => item.trim()) + .filter((item) => item.length > 0); +} + function nowMs(): number { return performance.now(); } @@ -76,26 +112,57 @@ function formatBytes(value: number): string { function buildConfig(): BenchmarkConfig { return { caseName: readEnvString("BENCH_CASE", "couchdb-baseline"), - couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"), - couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"), - couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")), - couchdbPassword: readEnvString("BENCH_COUCHDB_PASSWORD", readEnvString("password", "password")), - couchdbDbname: readEnvString("BENCH_COUCHDB_DBNAME", `bench-couchdb-${Date.now()}`), + couchdbBackendUri: readEnvString( + "BENCH_COUCHDB_BACKEND_URI", + "http://127.0.0.1:5989", + ), + couchdbProxyUri: readEnvString( + "BENCH_COUCHDB_URI", + "http://127.0.0.1:15989", + ), + couchdbUser: readEnvString( + "BENCH_COUCHDB_USER", + readEnvString("username", "admin"), + ), + couchdbPassword: readEnvString( + "BENCH_COUCHDB_PASSWORD", + readEnvString("password", "password"), + ), + couchdbDbname: readEnvString( + "BENCH_COUCHDB_DBNAME", + `bench-couchdb-${Date.now()}`, + ), datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"), datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)), - mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)), - mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)), + mdMinSizeBytes: Math.floor( + readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024), + ), + mdMaxSizeBytes: Math.floor( + readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024), + ), binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)), - binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)), + binSizeBytes: Math.floor( + readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024), + ), syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240), requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)), passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), encrypt: readEnvBool("BENCH_ENCRYPT", true), managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true), simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"), - networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "http-latency-proxy"), + networkProfile: readEnvString( + "BENCH_NETWORK_PROFILE", + "http-latency-proxy", + ), networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-http-proxy"), + measurementScope: readEnvString( + "BENCH_MEASUREMENT_SCOPE", + "Two one-shot synchronisation phases through a CouchDB-compatible remote-store path.", + ), + limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [ + "This benchmark result is scoped to the configured dataset, remote store, and network model.", + ]), }; } @@ -130,7 +197,9 @@ type ProxyHandle = { note: string; }; -function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requestedRttMs: number }): ProxyHandle { +function startCouchdbProxy( + options: { backendUri: string; proxyUri: string; requestedRttMs: number }, +): ProxyHandle { const backend = new URL(options.backendUri); const proxy = new URL(options.proxyUri); const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2)); @@ -182,12 +251,13 @@ function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requ statusText: upstream.statusText, headers: responseHeaders, }); - } + }, ); return { applied: true, - note: `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`, + note: + `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`, stop: async () => { controller.abort(); await listener.finished.catch(() => {}); @@ -211,14 +281,21 @@ async function main(): Promise { await initSettingsFile(settingsB); if (config.managedCouchdb) { - await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname); + await startCouchdb( + config.couchdbBackendUri, + config.couchdbUser, + config.couchdbPassword, + config.couchdbDbname, + ); } else { - console.log(`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`); + console.log( + `[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`, + ); await createCouchdbDatabase( config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, - config.couchdbDbname + config.couchdbDbname, ); } @@ -275,12 +352,21 @@ async function main(): Promise { const sampleFiles = pickSampleFiles(seedFiles.entries); for (const sample of sampleFiles) { - const pulledPath = workDir.join(`pulled-${sample.relativePath.split("/").join("_")}`); - await runCliOrFail(vaultB, "--settings", settingsB, "pull", sample.relativePath, pulledPath); + const pulledPath = workDir.join( + `pulled-${sample.relativePath.split("/").join("_")}`, + ); + await runCliOrFail( + vaultB, + "--settings", + settingsB, + "pull", + sample.relativePath, + pulledPath, + ); await assertFilesEqual( sample.absolutePath, pulledPath, - `sample file mismatch after CouchDB sync: ${sample.relativePath}` + `sample file mismatch after CouchDB sync: ${sample.relativePath}`, ); } @@ -294,6 +380,8 @@ async function main(): Promise { simulationTier: config.simulationTier, networkProfile: config.networkProfile, networkModel: config.networkModel, + measurementScope: config.measurementScope, + limitations: config.limitations, rttRequestedMs: config.requestedRttMs, proxyApplied: proxy.applied, proxyNote: proxy.note, @@ -306,22 +394,40 @@ async function main(): Promise { mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), syncAElapsedMs: Number(syncAElapsed.toFixed(1)), syncBElapsedMs: Number(syncBElapsed.toFixed(1)), - totalSyncElapsedMs: Number((syncAElapsed + syncBElapsed).toFixed(1)), - throughputBytesPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000)).toFixed(2)), + totalSyncElapsedMs: Number( + (syncAElapsed + syncBElapsed).toFixed(1), + ), + throughputBytesPerSec: Number( + (seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000)) + .toFixed( + 2, + ), + ), throughputMiBPerSec: Number( - (seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / 1024 / 1024).toFixed(4) + (seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / + 1024 / + 1024).toFixed(4), ), }; if (resultPath) { - await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2)); + await Deno.writeTextFile( + resultPath, + JSON.stringify(result, null, 2), + ); } console.log(JSON.stringify(result, null, 2)); console.error( - `[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${formatBytes(seedFiles.totalBytes)}) in ${formatMs( - mirrorElapsed - )}, synced in ${formatMs(syncAElapsed + syncBElapsed)} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)` + `[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${ + formatBytes(seedFiles.totalBytes) + }) in ${ + formatMs( + mirrorElapsed, + ) + }, synced in ${ + formatMs(syncAElapsed + syncBElapsed) + } (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`, ); } finally { await proxy.stop(); diff --git a/src/apps/cli/testdeno/bench-network-cases.ts b/src/apps/cli/testdeno/bench-network-cases.ts index dfb6e3f..689939a 100644 --- a/src/apps/cli/testdeno/bench-network-cases.ts +++ b/src/apps/cli/testdeno/bench-network-cases.ts @@ -1,9 +1,11 @@ -type BenchmarkCase = { +export type BenchmarkCase = { name: string; runner: "p2p" | "couchdb"; description: string; dataPath: string; trustBoundary: string; + measurementScope: string; + limitations: string[]; env: Record; }; @@ -25,16 +27,26 @@ function timestamp(): string { const d = new Date(); const pad = (n: number) => String(n).padStart(2, "0"); return ( - `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` + - `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}` + `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${ + pad(d.getUTCDate()) + }-` + + `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${ + pad(d.getUTCSeconds()) + }` ); } function buildBaseEnv(): Record { return { BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"), - BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"), - BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"), + BENCH_MD_MIN_SIZE_BYTES: readEnvString( + "BENCH_MD_MIN_SIZE_BYTES", + "512", + ), + BENCH_MD_MAX_SIZE_BYTES: readEnvString( + "BENCH_MD_MAX_SIZE_BYTES", + "2048", + ), BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"), BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"), BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"), @@ -44,33 +56,74 @@ function buildBaseEnv(): Record { }; } -function buildCases(): BenchmarkCase[] { +function withScopeEnv( + env: Record, + options: Pick, +): Record { + return { + ...env, + BENCH_MEASUREMENT_SCOPE: options.measurementScope, + BENCH_LIMITATIONS_JSON: JSON.stringify(options.limitations), + }; +} + +function defineCase(testCase: BenchmarkCase): BenchmarkCase { + return { + ...testCase, + env: withScopeEnv(testCase.env, testCase), + }; +} + +export function buildCases(): BenchmarkCase[] { const base = buildBaseEnv(); const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20"); const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120"); - const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478"); - const shimCouchdbUri = readEnvString("BENCH_SHIM_COUCHDB_URI", "http://couchdb-shim:5984"); - const signallingShimRelay = readEnvString("BENCH_SIGNAL_SHIM_RELAY", "ws://p2p-signalling-shim:7777/"); + const localTurnServers = readEnvString( + "BENCH_LOCAL_TURN_SERVERS", + "turn:127.0.0.1:3478", + ); + const shimCouchdbUri = readEnvString( + "BENCH_SHIM_COUCHDB_URI", + "http://couchdb-shim:5984", + ); + const signallingShimRelay = readEnvString( + "BENCH_SIGNAL_SHIM_RELAY", + "ws://p2p-signalling-shim:7777/", + ); return [ - { + defineCase({ name: "couchdb-baseline", runner: "couchdb", - description: "Standard self-hosted CouchDB path through a local latency proxy.", + description: + "Standard self-hosted CouchDB path through a local latency proxy.", dataPath: "Device A -> CouchDB -> Device B", trustBoundary: "CouchDB operator and network path", + measurementScope: + "Two one-shot synchronisation phases through a CouchDB-compatible remote-store path with a local HTTP latency proxy.", + limitations: [ + "This is not a full netem model of packet loss, jitter, MTU, bandwidth limits, or VPN encapsulation.", + "This result should be compared with P2P only as a remote-store baseline under the same deterministic dataset.", + ], env: { ...base, BENCH_CASE: "couchdb-baseline", BENCH_COUCHDB_RTT_MS: couchdbRtt, }, - }, - { + }), + defineCase({ name: "p2p-direct-local", runner: "p2p", - description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.", + description: + "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.", dataPath: "Device A -> Device B", trustBoundary: "Nostr relay for signalling metadata; no TURN relay", + measurementScope: + "One CLI P2P synchronisation phase over a local WebRTC DataChannel after Nostr signalling, with TURN disabled.", + limitations: [ + "This does not measure first-peer discovery latency, public relay operation, mobile carrier behaviour, or TURN-relayed throughput.", + "This small-dataset run should not be treated as a WAN, VPN, or large binary initial synchronisation measurement.", + ], env: { ...base, BENCH_CASE: "p2p-direct-local", @@ -78,29 +131,44 @@ function buildCases(): BenchmarkCase[] { BENCH_SIMULATION_TIER: "1", BENCH_NETWORK_PROFILE: "local-direct", BENCH_NETWORK_MODEL: "local-runner-webrtc", - BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "turn-disabled-but-selected-ice-pair-not-collected", + BENCH_P2P_CANDIDATE_PATH_VERIFICATION: + "turn-disabled-but-selected-ice-pair-not-collected", }, - }, - { + }), + defineCase({ name: "couchdb-tethering-vpn-proxy", runner: "couchdb", description: "Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.", - dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B", + dataPath: + "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B", trustBoundary: "VPN/network path and CouchDB operator", + measurementScope: + "Two one-shot CouchDB synchronisation phases with additional requested RTT through the local HTTP proxy.", + limitations: [ + "This approximates request latency only and does not model loss, jitter, MTU, bandwidth limits, carrier NAT, or VPN encapsulation.", + "Use the Tier 2 netem shim cases for a stronger constrained-network fixture.", + ], env: { ...base, BENCH_CASE: "couchdb-tethering-vpn-proxy", BENCH_COUCHDB_RTT_MS: tetheringVpnRtt, }, - }, - { + }), + defineCase({ name: "couchdb-netem-home-wifi", runner: "couchdb", description: "Tier 2 CouchDB path through the Compose netem TCP shim using the home-wifi profile.", - dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B", + dataPath: + "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B", trustBoundary: "CouchDB operator and constrained network shim", + measurementScope: + "Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the home-wifi netem profile.", + limitations: [ + "This shapes the CouchDB TCP path, not the WebRTC P2P data path.", + "The fixture remains a reproducible network emulation, not a field measurement on a real home network.", + ], env: { ...base, BENCH_CASE: "couchdb-netem-home-wifi", @@ -110,14 +178,22 @@ function buildCases(): BenchmarkCase[] { BENCH_NETWORK_PROFILE: "home-wifi", BENCH_NETWORK_MODEL: "compose-netem-tcp-shim", }, - }, - { + }), + defineCase({ name: "couchdb-netem-tethering-vpn", runner: "couchdb", description: "Tier 2 CouchDB path through the Compose netem TCP shim using a tethering-vpn profile.", - dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B", - trustBoundary: "CouchDB operator and constrained smartphone/VPN-like network shim", + dataPath: + "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B", + trustBoundary: + "CouchDB operator and constrained smartphone/VPN-like network shim", + measurementScope: + "Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the tethering-vpn netem profile.", + limitations: [ + "This shapes the CouchDB TCP path, not the WebRTC P2P data path.", + "The profile approximates smartphone/VPN constraints but is not a field measurement on a real tethered VPN connection.", + ], env: { ...base, BENCH_CASE: "couchdb-netem-tethering-vpn", @@ -127,14 +203,22 @@ function buildCases(): BenchmarkCase[] { BENCH_NETWORK_PROFILE: "tethering-vpn", BENCH_NETWORK_MODEL: "compose-netem-tcp-shim", }, - }, - { + }), + defineCase({ name: "p2p-smartphone-vpn-direct", runner: "p2p", description: "Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.", - dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds", - trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata", + dataPath: + "Device A -> Device B when WebRTC direct connectivity succeeds", + trustBoundary: + "Smartphone/VPN routing policy plus Nostr signalling metadata", + measurementScope: + "Structural placeholder for direct P2P measurements on a real smartphone tethering/VPN path.", + limitations: [ + "In the local runner this is unshaped and must not be reported as smartphone, VPN, WAN, or Tier 2 evidence.", + "Use only when the command is executed on the intended real network path and the selected ICE candidate pair is recorded.", + ], env: { ...base, BENCH_CASE: "p2p-smartphone-vpn-direct", @@ -145,14 +229,22 @@ function buildCases(): BenchmarkCase[] { BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "structural-placeholder-only; selected ICE pair may be collected, but the path is not shaped", }, - }, - { + }), + defineCase({ name: "p2p-signalling-netem-home-wifi", runner: "p2p", description: "Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.", - dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim", - trustBoundary: "Nostr signalling metadata through constrained network shim; no TURN relay", + dataPath: + "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim", + trustBoundary: + "Nostr signalling metadata through constrained network shim; no TURN relay", + measurementScope: + "Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the home-wifi netem profile.", + limitations: [ + "This does not shape the selected WebRTC DataChannel note-data path.", + "This supports only the claim that constrained signalling access does not place note data on the relay path when a non-relayed ICE path is selected.", + ], env: { ...base, BENCH_CASE: "p2p-signalling-netem-home-wifi", @@ -164,14 +256,22 @@ function buildCases(): BenchmarkCase[] { BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "selected ICE pair collected; only Nostr signalling path is shaped", }, - }, - { + }), + defineCase({ name: "p2p-signalling-netem-tethering-vpn", runner: "p2p", description: "Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.", - dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim", - trustBoundary: "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay", + dataPath: + "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim", + trustBoundary: + "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay", + measurementScope: + "Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the tethering-vpn netem profile.", + limitations: [ + "This does not shape the selected WebRTC DataChannel note-data path.", + "The profile approximates constrained relay access and is not a field measurement on a real tethered VPN connection.", + ], env: { ...base, BENCH_CASE: "p2p-signalling-netem-tethering-vpn", @@ -183,13 +283,20 @@ function buildCases(): BenchmarkCase[] { BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "selected ICE pair collected; only Nostr signalling path is shaped", }, - }, - { + }), + defineCase({ name: "p2p-user-turn", runner: "p2p", - description: "Optional fallback path through a local user-controlled TURN server.", + description: + "Optional fallback path through a local user-controlled TURN server.", dataPath: "Device A -> user-controlled TURN -> Device B", trustBoundary: "User-controlled TURN server", + measurementScope: + "Optional local TURN fallback wiring check with a user-controlled TURN server configured.", + limitations: [ + "TURN configuration does not prove that the selected ICE path was relayed; interpret the recorded candidate pair.", + "This is not evidence for public TURN relay privacy, throughput, or availability.", + ], env: { ...base, BENCH_CASE: "p2p-user-turn", @@ -200,7 +307,7 @@ function buildCases(): BenchmarkCase[] { BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "turn-configured; selected ICE pair may still be direct or relayed, so interpret the recorded candidate types", }, - }, + }), ]; } @@ -208,9 +315,11 @@ async function runCase( testCase: BenchmarkCase, outputDir: string, repeatIndex: number, - repeatCount: number + repeatCount: number, ): Promise> { - const suffix = repeatCount > 1 ? `-r${String(repeatIndex).padStart(2, "0")}` : ""; + const suffix = repeatCount > 1 + ? `-r${String(repeatIndex).padStart(2, "0")}` + : ""; const resultPath = `${outputDir}/${testCase.name}${suffix}.json`; const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb"; const env = { @@ -221,8 +330,12 @@ async function runCase( BENCH_REPEAT_COUNT: String(repeatCount), }; - const repeatLabel = repeatCount > 1 ? ` (${repeatIndex}/${repeatCount})` : ""; - console.log(`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`); + const repeatLabel = repeatCount > 1 + ? ` (${repeatIndex}/${repeatCount})` + : ""; + console.log( + `[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`, + ); const command = new Deno.Command("deno", { args: ["task", taskName], cwd: import.meta.dirname, @@ -238,7 +351,10 @@ async function runCase( throw new Error(`case failed: ${testCase.name} (exit ${status.code})`); } - const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record; + const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record< + string, + unknown + >; return { ...testCase, repeatIndex, @@ -249,7 +365,10 @@ async function runCase( } function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] { - const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local"); + const requested = readEnvString( + "BENCH_CASES", + "couchdb-baseline,p2p-direct-local", + ); const names = requested .split(",") .map((v) => v.trim()) @@ -258,14 +377,21 @@ function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] { return names.map((name) => { const found = byName.get(name); if (!found) { - throw new Error(`Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`); + throw new Error( + `Unknown BENCH_CASES entry '${name}'. Available: ${ + allCases.map((c) => c.name).join(", ") + }`, + ); } return found; }); } async function main(): Promise { - const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`); + const outRoot = readEnvString( + "BENCH_CASES_ROOT", + `${import.meta.dirname}/bench-results`, + ); const outputDir = `${outRoot}/cases-${timestamp()}`; await Deno.mkdir(outputDir, { recursive: true }); @@ -282,14 +408,16 @@ async function main(): Promise { availableCases: allCases, }, null, - 2 - ) + 2, + ), ); const results: Record[] = []; for (const testCase of cases) { for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) { - results.push(await runCase(testCase, outputDir, repeatIndex, repeatCount)); + results.push( + await runCase(testCase, outputDir, repeatIndex, repeatCount), + ); } } @@ -299,7 +427,10 @@ async function main(): Promise { repeatCount, results, }; - await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2)); + await Deno.writeTextFile( + `${outputDir}/summary.json`, + JSON.stringify(summary, null, 2), + ); console.log(JSON.stringify(summary, null, 2)); console.log(`[bench-cases] result directory: ${outputDir}`); } diff --git a/src/apps/cli/testdeno/bench-p2p.ts b/src/apps/cli/testdeno/bench-p2p.ts index 2bbf4e5..e763e9d 100644 --- a/src/apps/cli/testdeno/bench-p2p.ts +++ b/src/apps/cli/testdeno/bench-p2p.ts @@ -1,5 +1,9 @@ import { TempDir } from "./helpers/temp.ts"; -import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; +import { + applyP2pSettings, + applyP2pTestTweaks, + initSettingsFile, +} from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; import { discoverPeer, @@ -9,7 +13,10 @@ import { stopLocalRelayIfStarted, } from "./helpers/p2p.ts"; import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; -import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts"; +import { + createDeterministicDataset, + type DatasetEntry, +} from "./helpers/dataset.ts"; type BenchmarkConfig = { caseName: string; @@ -31,6 +38,8 @@ type BenchmarkConfig = { networkProfile: string; networkModel: string; candidatePathVerification: string; + measurementScope: string; + limitations: string[]; }; type P2PConnectionStats = { @@ -83,6 +92,30 @@ function readEnvNumber(name: string, fallback: number): number { return parsed; } +function readEnvStringArray(name: string, fallback: string[]): string[] { + const raw = Deno.env.get(name)?.trim(); + if (!raw) { + return fallback; + } + + try { + const parsed = JSON.parse(raw); + if ( + Array.isArray(parsed) && + parsed.every((item) => typeof item === "string") + ) { + return parsed; + } + } catch { + // Fall through to comma-separated parsing for hand-written invocations. + } + + return raw + .split("|") + .map((item) => item.trim()) + .filter((item) => item.length > 0); +} + function nowMs(): number { return performance.now(); } @@ -107,23 +140,45 @@ function buildConfig(): BenchmarkConfig { return { caseName: readEnvString("BENCH_CASE", "p2p-direct-local"), relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"), - appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"), + appId: readEnvString( + "BENCH_APP_ID", + "self-hosted-livesync-cli-benchmark", + ), roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`), passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), turnServers: readEnvString("BENCH_TURN_SERVERS", ""), datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"), datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)), - mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)), - mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)), + mdMinSizeBytes: Math.floor( + readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024), + ), + mdMaxSizeBytes: Math.floor( + readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024), + ), binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)), - binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)), + binSizeBytes: Math.floor( + readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024), + ), peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20), syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240), simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"), networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "local-direct"), - networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-runner-webrtc"), - candidatePathVerification: readEnvString("BENCH_P2P_CANDIDATE_PATH_VERIFICATION", "not-collected"), + networkModel: readEnvString( + "BENCH_NETWORK_MODEL", + "local-runner-webrtc", + ), + candidatePathVerification: readEnvString( + "BENCH_P2P_CANDIDATE_PATH_VERIFICATION", + "not-collected", + ), + measurementScope: readEnvString( + "BENCH_MEASUREMENT_SCOPE", + "One CLI P2P synchronisation phase over WebRTC DataChannel after signalling.", + ), + limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [ + "This benchmark result is scoped to the configured dataset, network model, and selected ICE path.", + ]), }; } @@ -152,7 +207,9 @@ function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] { return [...unique.values()]; } -async function readLatestP2PConnectionStats(statsPath: string): Promise { +async function readLatestP2PConnectionStats( + statsPath: string, +): Promise { try { const text = await Deno.readTextFile(statsPath); const lines = text @@ -200,7 +257,7 @@ async function main(): Promise { config.appId, config.relay, "~.*", - config.turnServers + config.turnServers, ), applyP2pSettings( clientSettings, @@ -209,13 +266,21 @@ async function main(): Promise { config.appId, config.relay, "~.*", - config.turnServers + config.turnServers, ), ]); await Promise.all([ - applyP2pTestTweaks(hostSettings, "p2p-bench-host", config.passphrase), - applyP2pTestTweaks(clientSettings, "p2p-bench-client", config.passphrase), + applyP2pTestTweaks( + hostSettings, + "p2p-bench-host", + config.passphrase, + ), + applyP2pTestTweaks( + clientSettings, + "p2p-bench-client", + config.passphrase, + ), ]); const seedFiles = await createDeterministicDataset({ @@ -233,15 +298,25 @@ async function main(): Promise { await runCliOrFail(hostVault, "--settings", hostSettings, "mirror"); const mirrorElapsed = nowMs() - mirrorStart; - const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host"); + const host = startCliInBackground( + hostVault, + "--settings", + hostSettings, + "p2p-host", + ); try { const hostReadyStart = nowMs(); await host.waitUntilContains("P2P host is running", 20000); const hostReadyElapsed = nowMs() - hostReadyStart; const peerDiscoveryCommandStart = nowMs(); - const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds); - const peerDiscoveryCommandElapsed = nowMs() - peerDiscoveryCommandStart; + const peer = await discoverPeer( + clientVault, + clientSettings, + config.peersTimeoutSeconds, + ); + const peerDiscoveryCommandElapsed = nowMs() - + peerDiscoveryCommandStart; const syncStart = nowMs(); await runCliOrFail( @@ -250,22 +325,33 @@ async function main(): Promise { clientSettings, "p2p-sync", peer.id, - String(config.syncTimeoutSeconds) + String(config.syncTimeoutSeconds), ); const syncElapsed = nowMs() - syncStart; const sampleFiles = pickSampleFiles(seedFiles.entries); for (const sample of sampleFiles) { - const pulledPath = workDir.join(`pulled-${sample.relativePath.replaceAll("/", "_")}`); - await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath); + const pulledPath = workDir.join( + `pulled-${sample.relativePath.replaceAll("/", "_")}`, + ); + await runCliOrFail( + clientVault, + "--settings", + clientSettings, + "pull", + sample.relativePath, + pulledPath, + ); await assertFilesEqual( sample.absolutePath, pulledPath, - `sample file mismatch after sync: ${sample.relativePath}` + `sample file mismatch after sync: ${sample.relativePath}`, ); } - const p2pConnectionStats = await readLatestP2PConnectionStats(p2pStatsPath); + const p2pConnectionStats = await readLatestP2PConnectionStats( + p2pStatsPath, + ); const result = { caseName: config.caseName, mode: "p2p-cli-benchmark", @@ -275,15 +361,19 @@ async function main(): Promise { simulationTier: config.simulationTier, networkProfile: config.networkProfile, networkModel: config.networkModel, - p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true, - p2pCandidatePathVerification: p2pConnectionStats?.candidatePathCollected - ? "selected ICE candidate pair collected from RTCPeerConnection.getStats" - : config.candidatePathVerification, + measurementScope: config.measurementScope, + limitations: config.limitations, + p2pCandidatePathVerified: + p2pConnectionStats?.candidatePathCollected === true, + p2pCandidatePathVerification: + p2pConnectionStats?.candidatePathCollected + ? "selected ICE candidate pair collected from RTCPeerConnection.getStats" + : config.candidatePathVerification, p2pCandidatePathNote: p2pConnectionStats?.candidatePathCollected ? "The selected ICE candidate pair was collected by the CLI benchmark. Interpret the path from the candidate types; do not infer TURN use from configuration alone." : config.turnServers.trim().length > 0 - ? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run." - : "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.", + ? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run." + : "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.", p2pConnectionStats, appId: config.appId, roomId: config.roomId, @@ -298,25 +388,39 @@ async function main(): Promise { mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)), peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds, - peerDiscoveryCommandElapsedMs: Number(peerDiscoveryCommandElapsed.toFixed(1)), + peerDiscoveryCommandElapsedMs: Number( + peerDiscoveryCommandElapsed.toFixed(1), + ), peerDiscoveryNote: "p2p-peers waits for the requested timeout before printing discovered peers, so this is command duration, not first-peer latency.", syncElapsedMs: Number(syncElapsed.toFixed(1)), - throughputBytesPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2)), - throughputMiBPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024).toFixed(4)), + throughputBytesPerSec: Number( + (seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2), + ), + throughputMiBPerSec: Number( + (seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024) + .toFixed( + 4, + ), + ), }; if (resultPath) { - await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2)); + await Deno.writeTextFile( + resultPath, + JSON.stringify(result, null, 2), + ); } console.log(JSON.stringify(result, null, 2)); console.error( - `[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes( - seedFiles.totalBytes - )}) in ${formatMs(mirrorElapsed)}, ` + + `[Benchmark] mirrored ${seedFiles.totalFiles} files (${ + formatBytes( + seedFiles.totalBytes, + ) + }) in ${formatMs(mirrorElapsed)}, ` + `synced in ${formatMs(syncElapsed)} ` + - `(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)` + `(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`, ); } finally { await host.stop(); diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json index 3a369e3..7845ba1 100644 --- a/src/apps/cli/testdeno/deno.json +++ b/src/apps/cli/testdeno/deno.json @@ -15,6 +15,7 @@ "test:p2p-sync": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts", "test:p2p-three-nodes": "deno test --env-file=.test.env -A --no-check test-p2p-three-nodes-conflict.ts", "test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts", + "test:benchmark-contract": "deno test --env-file=.test.env -A --no-check test-benchmark-contract.ts", "bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts", "bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts", "bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts", diff --git a/src/apps/cli/testdeno/test-benchmark-contract.ts b/src/apps/cli/testdeno/test-benchmark-contract.ts new file mode 100644 index 0000000..a2ad593 --- /dev/null +++ b/src/apps/cli/testdeno/test-benchmark-contract.ts @@ -0,0 +1,134 @@ +import { assert, assertEquals, assertStringIncludes } from "@std/assert"; +import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts"; + +function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase { + const found = cases.find((testCase) => testCase.name === name); + assert(found, `missing benchmark case: ${name}`); + return found; +} + +function parsedLimitations(testCase: BenchmarkCase): string[] { + const raw = testCase.env.BENCH_LIMITATIONS_JSON; + assert( + raw, + `${testCase.name} must pass BENCH_LIMITATIONS_JSON to benchmark result output`, + ); + const parsed = JSON.parse(raw); + assert( + Array.isArray(parsed), + `${testCase.name} limitations must be an array`, + ); + assert( + parsed.every((item) => + typeof item === "string" && item.trim().length > 0 + ), + ); + return parsed; +} + +Deno.test("benchmark cases record scope and limitations for paper use", () => { + const cases = buildCases(); + assert(cases.length > 0); + + for (const testCase of cases) { + assert( + testCase.description.trim().length > 0, + `${testCase.name} must describe the case`, + ); + assert( + testCase.dataPath.trim().length > 0, + `${testCase.name} must describe the data path`, + ); + assert( + testCase.trustBoundary.trim().length > 0, + `${testCase.name} must describe the trust boundary`, + ); + assert( + testCase.measurementScope.trim().length > 0, + `${testCase.name} must describe the measurement scope`, + ); + assert( + testCase.limitations.length > 0, + `${testCase.name} must list limitations`, + ); + assertEquals( + testCase.env.BENCH_MEASUREMENT_SCOPE, + testCase.measurementScope, + ); + assertEquals(parsedLimitations(testCase), testCase.limitations); + } +}); + +Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => { + const cases = buildCases(); + for ( + const name of [ + "p2p-signalling-netem-home-wifi", + "p2p-signalling-netem-tethering-vpn", + ] + ) { + const testCase = getCase(cases, name); + assertEquals(testCase.runner, "p2p"); + assertEquals(testCase.env.BENCH_TURN_SERVERS, ""); + assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2"); + assertEquals( + testCase.env.BENCH_NETWORK_MODEL, + "compose-netem-signalling-shim", + ); + assertStringIncludes(testCase.dataPath, "WebRTC DataChannel"); + assertStringIncludes(testCase.dataPath, "Nostr signalling"); + assert( + testCase.limitations.some((limitation) => + limitation.includes("does not shape the selected WebRTC") + ), + `${name} must avoid claiming that the P2P note-data path was shaped`, + ); + } +}); + +Deno.test("placeholder and TURN cases are clearly non-evidence for broad P2P performance", () => { + const cases = buildCases(); + + const smartphone = getCase(cases, "p2p-smartphone-vpn-direct"); + assertEquals(smartphone.env.BENCH_SIMULATION_TIER, "unmeasured"); + assertEquals(smartphone.env.BENCH_NETWORK_MODEL, "local-runner-no-netem"); + assert( + smartphone.limitations.some((limitation) => + limitation.includes("must not be reported as smartphone") + ), + "smartphone/VPN placeholder must not be usable as field evidence by accident", + ); + + const turn = getCase(cases, "p2p-user-turn"); + assertStringIncludes(turn.env.BENCH_TURN_SERVERS, "turn:"); + assert( + turn.limitations.some((limitation) => + limitation.includes( + "does not prove that the selected ICE path was relayed", + ) + ), + "TURN case must require selected ICE candidate interpretation", + ); +}); + +Deno.test("CouchDB netem cases are marked as remote-store baselines", () => { + const cases = buildCases(); + for ( + const name of ["couchdb-netem-home-wifi", "couchdb-netem-tethering-vpn"] + ) { + const testCase = getCase(cases, name); + assertEquals(testCase.runner, "couchdb"); + assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2"); + assertEquals( + testCase.env.BENCH_NETWORK_MODEL, + "compose-netem-tcp-shim", + ); + assertStringIncludes(testCase.measurementScope, "CouchDB"); + assert( + testCase.limitations.some((limitation) => + limitation.includes("not the WebRTC P2P data path") + ), + `${name} must remain scoped to the CouchDB remote-store path`, + ); + } +});