public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-07-12 15:53:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Strengthen CLI benchmark verification

Browse Source
This commit is contained in:
vorotamoroz
2026-07-12 10:54:16 +00:00
parent fd3e8416b7
commit a60932d9e4
8 changed files with 360 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/cli/testdeno/bench-couchdb.ts
+59 -44
View File
@@ -11,8 +11,12 @@ import {
} from "./helpers/docker.ts";
import {
createDeterministicDataset,
type DatasetEntry,
} from "./helpers/dataset.ts";
import {
type BenchmarkVerificationMode,
parseBenchmarkVerificationMode,
verifyBenchmarkDataset,
} from "./helpers/benchmarkVerification.ts";
type BenchmarkConfig = {
caseName: string;
@@ -38,6 +42,9 @@ type BenchmarkConfig = {
networkModel: string;
measurementScope: string;
limitations: string[];
verificationMode: BenchmarkVerificationMode;
repeatIndex: number;
repeatCount: number;
};
function readEnvString(name: string, fallback: string): string {
@@ -163,6 +170,11 @@ function buildConfig(): BenchmarkConfig {
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
"This benchmark result is scoped to the configured dataset, remote store, and network model.",
]),
verificationMode: parseBenchmarkVerificationMode(
Deno.env.get("BENCH_VERIFY_MODE"),
),
repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
};
}
@@ -174,35 +186,27 @@ function readOptionalResultPath(): string | undefined {
return raw;
}
function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
if (entries.length === 0) {
return [];
}
const md = entries.find((e) => e.kind === "md");
const bin = entries.find((e) => e.kind === "bin");
const middle = entries[Math.floor(entries.length / 2)];
const last = entries[entries.length - 1];
const unique = new Map<string, DatasetEntry>();
for (const entry of [md, bin, middle, last]) {
if (entry) {
unique.set(entry.relativePath, entry);
}
}
return [...unique.values()];
}
type ProxyHandle = {
export type CouchdbProxyHandle = {
stop: () => Promise<void>;
applied: boolean;
note: string;
directionalDelayMs: number;
};
function startCouchdbProxy(
options: { backendUri: string; proxyUri: string; requestedRttMs: number },
): ProxyHandle {
export function startCouchdbProxy(
options: {
backendUri: string;
proxyUri: string;
requestedRttMs: number;
delay?: (milliseconds: number) => Promise<void>;
},
): CouchdbProxyHandle {
const backend = new URL(options.backendUri);
const proxy = new URL(options.proxyUri);
const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2));
const halfDelayMs = options.requestedRttMs / 2;
const delay = options.delay ??
((milliseconds: number) =>
new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
const controller = new AbortController();
const listener = Deno.serve(
@@ -216,7 +220,7 @@ function startCouchdbProxy(
},
},
async (request) => {
await new Promise((resolve) => setTimeout(resolve, halfDelayMs));
await delay(halfDelayMs);
const targetUrl = new URL(request.url);
targetUrl.protocol = backend.protocol;
@@ -245,6 +249,7 @@ function startCouchdbProxy(
const responseHeaders = new Headers(upstream.headers);
responseHeaders.delete("content-length");
const responseBody = await upstream.arrayBuffer();
await delay(halfDelayMs);
return new Response(responseBody, {
status: upstream.status,
@@ -256,8 +261,9 @@ function startCouchdbProxy(
return {
applied: true,
directionalDelayMs: halfDelayMs,
note:
`local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`,
`local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms request-path and ${halfDelayMs}ms response-path delay`,
stop: async () => {
controller.abort();
await listener.finished.catch(() => {});
@@ -350,25 +356,28 @@ async function main(): Promise<void> {
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
const syncBElapsed = nowMs() - syncBStart;
const sampleFiles = pickSampleFiles(seedFiles.entries);
for (const sample of sampleFiles) {
const pulledPath = workDir.join(
`pulled-${sample.relativePath.split("/").join("_")}`,
);
await runCliOrFail(
vaultB,
"--settings",
settingsB,
"pull",
sample.relativePath,
pulledPath,
);
await assertFilesEqual(
sample.absolutePath,
pulledPath,
`sample file mismatch after CouchDB sync: ${sample.relativePath}`,
);
}
const verification = await verifyBenchmarkDataset(
seedFiles.entries,
config.verificationMode,
async (entry) => {
const pulledPath = workDir.join(
`pulled-${entry.relativePath.split("/").join("_")}`,
);
await runCliOrFail(
vaultB,
"--settings",
settingsB,
"pull",
entry.relativePath,
pulledPath,
);
await assertFilesEqual(
entry.absolutePath,
pulledPath,
`file mismatch after CouchDB sync: ${entry.relativePath}`,
);
},
);
const result = {
caseName: config.caseName,
@@ -382,15 +391,21 @@ async function main(): Promise<void> {
networkModel: config.networkModel,
measurementScope: config.measurementScope,
limitations: config.limitations,
repeatIndex: config.repeatIndex,
repeatCount: config.repeatCount,
rttRequestedMs: config.requestedRttMs,
proxyApplied: proxy.applied,
proxyNote: proxy.note,
proxyDirectionalDelayMs: proxy.directionalDelayMs,
proxyConfiguredRttMs: proxy.directionalDelayMs * 2,
proxyDelayApplication: "request-and-response",
datasetSeed: config.datasetSeed,
datasetDirName: config.datasetDirName,
totalFiles: seedFiles.totalFiles,
totalBytes: seedFiles.totalBytes,
mdFileCount: seedFiles.mdCount,
binFileCount: seedFiles.binCount,
...verification,
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
syncAElapsedMs: Number(syncAElapsed.toFixed(1)),
syncBElapsedMs: Number(syncBElapsed.toFixed(1)),
src/apps/cli/testdeno/bench-latency-sweep.ts
+66 -24
View File
@@ -2,6 +2,8 @@ type SweepResult = {
name: string;
runner: "p2p" | "couchdb";
rttMs?: number;
repeatIndex: number;
repeatCount: number;
result: Record<string, unknown>;
};
@@ -19,6 +21,15 @@ function timestamp(): string {
);
}
function readEnvInteger(name: string, fallback: number): number {
const raw = readEnvString(name, String(fallback));
const parsed = Number(raw);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${name} must be a positive integer, got '${raw}'`);
}
return parsed;
}
function parseRttList(raw: string): number[] {
const values = raw
.split(",")
@@ -41,6 +52,7 @@ function buildBaseEnv(): Record<string, string> {
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"),
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
};
}
@@ -50,12 +62,19 @@ async function runBenchmark(options: {
name: string;
outputDir: string;
env: Record<string, string>;
repeatIndex: number;
repeatCount: number;
}): Promise<Record<string, unknown>> {
const resultPath = `${options.outputDir}/${options.name}.json`;
const suffix = options.repeatCount > 1
? `-r${String(options.repeatIndex).padStart(2, "0")}`
: "";
const resultPath = `${options.outputDir}/${options.name}${suffix}.json`;
const env = {
...Deno.env.toObject(),
...options.env,
BENCH_RESULT_JSON: resultPath,
BENCH_REPEAT_INDEX: String(options.repeatIndex),
BENCH_REPEAT_COUNT: String(options.repeatCount),
};
console.log(`[latency-sweep] running ${options.name}`);
@@ -78,46 +97,69 @@ async function main(): Promise<void> {
const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`);
const outputDir = `${outRoot}/latency-sweep-${timestamp()}`;
const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300"));
const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1);
const base = buildBaseEnv();
await Deno.mkdir(outputDir, { recursive: true });
const results: SweepResult[] = [];
if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") {
const p2pResult = await runBenchmark({
taskName: "bench:p2p",
name: "p2p-direct-local",
outputDir,
env: {
...base,
BENCH_CASE: "p2p-direct-local",
BENCH_TURN_SERVERS: "",
},
});
results.push({ name: "p2p-direct-local", runner: "p2p", result: p2pResult });
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
const p2pResult = await runBenchmark({
taskName: "bench:p2p",
name: "p2p-direct-local",
outputDir,
repeatIndex,
repeatCount,
env: {
...base,
BENCH_CASE: "p2p-direct-local",
BENCH_TURN_SERVERS: "",
},
});
results.push({
name: "p2p-direct-local",
runner: "p2p",
repeatIndex,
repeatCount,
result: p2pResult,
});
}
}
for (const rtt of rtts) {
const name = `couchdb-rtt-${rtt}ms`;
const couchdbResult = await runBenchmark({
taskName: "bench:couchdb",
name,
outputDir,
env: {
...base,
BENCH_CASE: name,
BENCH_COUCHDB_RTT_MS: String(rtt),
},
});
results.push({ name, runner: "couchdb", rttMs: rtt, result: couchdbResult });
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
const couchdbResult = await runBenchmark({
taskName: "bench:couchdb",
name,
outputDir,
repeatIndex,
repeatCount,
env: {
...base,
BENCH_CASE: name,
BENCH_COUCHDB_RTT_MS: String(rtt),
},
});
results.push({
name,
runner: "couchdb",
rttMs: rtt,
repeatIndex,
repeatCount,
result: couchdbResult,
});
}
}
const summary = {
generatedAt: new Date().toISOString(),
outputDir,
note:
"This sweep models additional remote CouchDB request latency through the existing HTTP proxy. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
"This sweep applies half of each requested CouchDB RTT before forwarding requests and half before returning responses. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
rtts,
repeatCount,
results,
};
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
src/apps/cli/testdeno/bench-network-cases.ts
+8 -4
View File
@@ -52,6 +52,7 @@ function buildBaseEnv(): Record<string, string> {
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"),
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
};
}
@@ -119,9 +120,10 @@ export function buildCases(): BenchmarkCase[] {
dataPath: "Device A -> Device B",
trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
measurementScope:
"One CLI P2P synchronisation phase over a local WebRTC DataChannel after Nostr signalling, with TURN disabled.",
"One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment, with TURN disabled; the earlier peer-list observation command is excluded.",
limitations: [
"This does not measure first-peer discovery latency, public relay operation, mobile carrier behaviour, or TURN-relayed throughput.",
"The timed command includes its own signalling and connection establishment, but not the earlier peer-list observation window.",
"This does not measure public relay operation, mobile carrier behaviour, or TURN-relayed throughput.",
"This small-dataset run should not be treated as a WAN, VPN, or large binary initial synchronisation measurement.",
],
env: {
@@ -240,8 +242,9 @@ export function buildCases(): BenchmarkCase[] {
trustBoundary:
"Nostr signalling metadata through constrained network shim; no TURN relay",
measurementScope:
"Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the home-wifi netem profile.",
"One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the home-wifi netem profile; the selected WebRTC note-data path is unshaped.",
limitations: [
"The timed p2p-sync command includes signalling and WebRTC connection establishment.",
"This does not shape the selected WebRTC DataChannel note-data path.",
"This supports only the claim that constrained signalling access does not place note data on the relay path when a non-relayed ICE path is selected.",
],
@@ -267,8 +270,9 @@ export function buildCases(): BenchmarkCase[] {
trustBoundary:
"Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
measurementScope:
"Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the tethering-vpn netem profile.",
"One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the tethering-vpn netem profile; the selected WebRTC note-data path is unshaped.",
limitations: [
"The timed p2p-sync command includes signalling and WebRTC connection establishment.",
"This does not shape the selected WebRTC DataChannel note-data path.",
"The profile approximates constrained relay access and is not a field measurement on a real tethered VPN connection.",
],
src/apps/cli/testdeno/bench-p2p.ts
+39 -38
View File
@@ -15,8 +15,12 @@ import {
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
import {
createDeterministicDataset,
type DatasetEntry,
} from "./helpers/dataset.ts";
import {
type BenchmarkVerificationMode,
parseBenchmarkVerificationMode,
verifyBenchmarkDataset,
} from "./helpers/benchmarkVerification.ts";
type BenchmarkConfig = {
caseName: string;
@@ -40,6 +44,9 @@ type BenchmarkConfig = {
candidatePathVerification: string;
measurementScope: string;
limitations: string[];
verificationMode: BenchmarkVerificationMode;
repeatIndex: number;
repeatCount: number;
};
type P2PConnectionStats = {
@@ -174,11 +181,16 @@ function buildConfig(): BenchmarkConfig {
),
measurementScope: readEnvString(
"BENCH_MEASUREMENT_SCOPE",
"One CLI P2P synchronisation phase over WebRTC DataChannel after signalling.",
"One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment; the earlier peer-list observation command is excluded.",
),
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
"This benchmark result is scoped to the configured dataset, network model, and selected ICE path.",
]),
verificationMode: parseBenchmarkVerificationMode(
Deno.env.get("BENCH_VERIFY_MODE"),
),
repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
};
}
@@ -190,23 +202,6 @@ function readOptionalResultPath(): string | undefined {
return raw;
}
function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
if (entries.length === 0) {
return [];
}
const md = entries.find((e) => e.kind === "md");
const bin = entries.find((e) => e.kind === "bin");
const middle = entries[Math.floor(entries.length / 2)];
const last = entries[entries.length - 1];
const unique = new Map<string, DatasetEntry>();
for (const entry of [md, bin, middle, last]) {
if (entry) {
unique.set(entry.relativePath, entry);
}
}
return [...unique.values()];
}
async function readLatestP2PConnectionStats(
statsPath: string,
): Promise<P2PConnectionStats | undefined> {
@@ -329,25 +324,28 @@ async function main(): Promise<void> {
);
const syncElapsed = nowMs() - syncStart;
const sampleFiles = pickSampleFiles(seedFiles.entries);
for (const sample of sampleFiles) {
const pulledPath = workDir.join(
`pulled-${sample.relativePath.replaceAll("/", "_")}`,
);
await runCliOrFail(
clientVault,
"--settings",
clientSettings,
"pull",
sample.relativePath,
pulledPath,
);
await assertFilesEqual(
sample.absolutePath,
pulledPath,
`sample file mismatch after sync: ${sample.relativePath}`,
);
}
const verification = await verifyBenchmarkDataset(
seedFiles.entries,
config.verificationMode,
async (entry) => {
const pulledPath = workDir.join(
`pulled-${entry.relativePath.replaceAll("/", "_")}`,
);
await runCliOrFail(
clientVault,
"--settings",
clientSettings,
"pull",
entry.relativePath,
pulledPath,
);
await assertFilesEqual(
entry.absolutePath,
pulledPath,
`file mismatch after P2P sync: ${entry.relativePath}`,
);
},
);
const p2pConnectionStats = await readLatestP2PConnectionStats(
p2pStatsPath,
@@ -363,6 +361,8 @@ async function main(): Promise<void> {
networkModel: config.networkModel,
measurementScope: config.measurementScope,
limitations: config.limitations,
repeatIndex: config.repeatIndex,
repeatCount: config.repeatCount,
p2pCandidatePathVerified:
p2pConnectionStats?.candidatePathCollected === true,
p2pCandidatePathVerification:
@@ -385,6 +385,7 @@ async function main(): Promise<void> {
totalBytes: seedFiles.totalBytes,
mdFileCount: seedFiles.mdCount,
binFileCount: seedFiles.binCount,
...verification,
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
src/apps/cli/testdeno/helpers/benchmarkVerification.ts
+80
View File
@@ -0,0 +1,80 @@
import type { DatasetEntry } from "./dataset.ts";
export type BenchmarkVerificationMode = "all" | "sample";
export type BenchmarkVerificationResult = {
verificationMode: BenchmarkVerificationMode;
verifiedFiles: number;
verificationComplete: boolean;
datasetDigestSha256: string;
};
function toHex(bytes: ArrayBuffer): string {
return [...new Uint8Array(bytes)]
.map((value) => value.toString(16).padStart(2, "0"))
.join("");
}
async function sha256(bytes: Uint8Array): Promise<string> {
const input = new ArrayBuffer(bytes.byteLength);
new Uint8Array(input).set(bytes);
return toHex(await crypto.subtle.digest("SHA-256", input));
}
export function parseBenchmarkVerificationMode(
raw: string | undefined,
fallback: BenchmarkVerificationMode = "sample",
): BenchmarkVerificationMode {
const value = raw?.trim().toLowerCase();
if (!value) return fallback;
if (value === "all" || value === "sample") return value;
throw new Error(`BENCH_VERIFY_MODE must be 'all' or 'sample', got '${raw}'`);
}
export function selectVerificationEntries(
entries: DatasetEntry[],
mode: BenchmarkVerificationMode,
): DatasetEntry[] {
if (mode === "all" || entries.length === 0) return [...entries];
const md = entries.find((entry) => entry.kind === "md");
const bin = entries.find((entry) => entry.kind === "bin");
const middle = entries[Math.floor(entries.length / 2)];
const last = entries[entries.length - 1];
const selected = new Map<string, DatasetEntry>();
for (const entry of [md, bin, middle, last]) {
if (entry) selected.set(entry.relativePath, entry);
}
return [...selected.values()];
}
export async function computeDatasetDigestSha256(
entries: DatasetEntry[],
): Promise<string> {
const manifest: string[] = [];
for (const entry of entries) {
const contentDigest = await sha256(await Deno.readFile(entry.absolutePath));
manifest.push(
`${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`,
);
}
return await sha256(new TextEncoder().encode(manifest.join("\n")));
}
export async function verifyBenchmarkDataset(
entries: DatasetEntry[],
mode: BenchmarkVerificationMode,
verifyEntry: (entry: DatasetEntry) => Promise<void>,
): Promise<BenchmarkVerificationResult> {
const selected = selectVerificationEntries(entries, mode);
for (const entry of selected) {
await verifyEntry(entry);
}
return {
verificationMode: mode,
verifiedFiles: selected.length,
verificationComplete: selected.length === entries.length,
datasetDigestSha256: await computeDatasetDigestSha256(entries),
};
}
src/apps/cli/testdeno/test-benchmark-contract.ts
+89
View File
@@ -1,5 +1,20 @@
import { assert, assertEquals, assertStringIncludes } from "@std/assert";
import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts";
import { startCouchdbProxy } from "./bench-couchdb.ts";
import {
parseBenchmarkVerificationMode,
selectVerificationEntries,
} from "./helpers/benchmarkVerification.ts";
import type { DatasetEntry } from "./helpers/dataset.ts";
function getFreePort(): number {
const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 });
try {
return (listener.addr as Deno.NetAddr).port;
} finally {
listener.close();
}
}
function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase {
const found = cases.find((testCase) => testCase.name === name);
@@ -56,9 +71,76 @@ Deno.test("benchmark cases record scope and limitations for paper use", () => {
testCase.measurementScope,
);
assertEquals(parsedLimitations(testCase), testCase.limitations);
assertEquals(
testCase.env.BENCH_VERIFY_MODE,
"all",
`${testCase.name} must verify the complete dataset`,
);
}
});
Deno.test("CouchDB latency proxy applies half the requested RTT in each direction", async () => {
const backendPort = getFreePort();
const proxyPort = getFreePort();
const delays: number[] = [];
const backend = Deno.serve(
{
hostname: "127.0.0.1",
port: backendPort,
onListen() {},
},
() => new Response("ok"),
);
const proxy = startCouchdbProxy({
backendUri: `http://127.0.0.1:${backendPort}`,
proxyUri: `http://127.0.0.1:${proxyPort}`,
requestedRttMs: 20,
delay: (milliseconds) => {
delays.push(milliseconds);
return Promise.resolve();
},
});
try {
const response = await fetch(`http://127.0.0.1:${proxyPort}/probe`);
assertEquals(await response.text(), "ok");
assertEquals(proxy.directionalDelayMs, 10);
assertEquals(delays, [10, 10]);
} finally {
await proxy.stop();
await backend.shutdown();
}
const halfMillisecondProxy = startCouchdbProxy({
backendUri: "http://127.0.0.1:1",
proxyUri: `http://127.0.0.1:${getFreePort()}`,
requestedRttMs: 1,
delay: () => Promise.resolve(),
});
try {
assertEquals(halfMillisecondProxy.directionalDelayMs, 0.5);
} finally {
await halfMillisecondProxy.stop();
}
});
Deno.test("benchmark verification mode selects either all files or a labelled sample", () => {
const entries: DatasetEntry[] = [
{ kind: "md", relativePath: "a.md", absolutePath: "/a", size: 1 },
{ kind: "md", relativePath: "b.md", absolutePath: "/b", size: 1 },
{ kind: "bin", relativePath: "c.bin", absolutePath: "/c", size: 1 },
{ kind: "md", relativePath: "d.md", absolutePath: "/d", size: 1 },
{ kind: "bin", relativePath: "e.bin", absolutePath: "/e", size: 1 },
];
assertEquals(parseBenchmarkVerificationMode("ALL"), "all");
assertEquals(selectVerificationEntries(entries, "all").length, entries.length);
const sample = selectVerificationEntries(entries, "sample");
assert(sample.length > 0 && sample.length < entries.length);
assert(sample.some((entry) => entry.kind === "md"));
assert(sample.some((entry) => entry.kind === "bin"));
});
Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => {
const cases = buildCases();
for (
@@ -77,6 +159,13 @@ Deno.test("P2P signalling-shim cases do not claim to shape the note-data path",
);
assertStringIncludes(testCase.dataPath, "WebRTC DataChannel");
assertStringIncludes(testCase.dataPath, "Nostr signalling");
assertStringIncludes(testCase.measurementScope, "fresh CLI p2p-sync");
assert(
testCase.limitations.some((limitation) =>
limitation.includes("connection establishment")
),
`${name} must state that connection establishment is timed`,
);
assert(
testCase.limitations.some((limitation) =>
limitation.includes("does not shape the selected WebRTC")
test/bench-network/README.md
+17 -8
View File
@@ -64,16 +64,24 @@ path:
| Case | Data path | What is measured | What is not measured |
| ------------------ | ------------------------------------------- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------- |
| `couchdb-baseline` | Device A -> CouchDB -> Device B | Two one-shot CLI synchronisation commands through a local HTTP latency proxy | Real WAN jitter, packet loss, bandwidth limits, VPN encapsulation, and server contention |
| `p2p-direct-local` | Device A -> Device B after Nostr signalling | One CLI P2P synchronisation command over WebRTC DataChannel with TURN disabled | Public relay operation, mobile carrier behaviour, TURN relay throughput, and first-peer discovery latency |
| `p2p-direct-local` | Device A -> Device B using Nostr signalling | One fresh CLI `p2p-sync` command, including process start-up and WebRTC connection establishment, with TURN disabled | Public relay operation, mobile carrier behaviour, and TURN relay throughput |
Use the CouchDB result as the remote-store baseline and the P2P result as the
direct-transfer comparison. The Nostr relay is used for signalling in the P2P
case, but synchronised note content is transferred over the WebRTC DataChannel.
The P2P result JSON records the selected WebRTC ICE candidate pair when the CLI
The earlier `p2p-peers` observation command is excluded from the P2P timing,
but the timed `p2p-sync` command performs its own signalling and connection
establishment. The P2P result JSON records the selected WebRTC ICE candidate pair when the CLI
can collect it from `RTCPeerConnection.getStats()`. Interpret P2P paths from
the recorded candidate types rather than from TURN configuration alone. Do not
report P2P runs as Tier 2 constrained-network measurements until host and
client are captured under an equivalent shaped topology.
report a signalling-only Tier 2 run as though the selected note-data path were
also shaped.
Benchmark cases use `BENCH_VERIFY_MODE=all` by default. After the timed phase,
the runner retrieves and compares every generated file and records the verified
file count, whether verification was complete, and a SHA-256 digest of the
deterministic dataset. Set `BENCH_VERIFY_MODE=sample` only for exploratory
large-dataset runs where the additional verification time is impractical.
## Dataset and latency controls
@@ -88,10 +96,10 @@ BENCH_PEERS_TIMEOUT=60 \
docker compose -f test/bench-network/compose.yml run --rm bench-runner
```
The current CouchDB latency model is the existing HTTP proxy inside
`bench-couchdb.ts`. It models a remote database path with additional request
latency, but it does not model packet loss, jitter, MTU, bandwidth limits,
bufferbloat, or VPN encapsulation.
The CouchDB latency model is the HTTP proxy inside `bench-couchdb.ts`. It adds
half of the requested RTT before forwarding each request and the other half
before returning its response. It does not model packet loss, jitter, MTU,
bandwidth limits, bufferbloat, or VPN encapsulation.
For P2P runs, `BENCH_PEERS_TIMEOUT` is passed to `p2p-peers`. That command waits
for the requested observation window before printing discovered peers, so the
@@ -104,6 +112,7 @@ To run P2P once and CouchDB at several requested RTT values:
```bash
BENCH_COMMAND=latency-sweep \
BENCH_SWEEP_RTT_MS=20,50,100,150,300 \
BENCH_REPEAT_COUNT=3 \
BENCH_MD_FILE_COUNT=100 \
BENCH_MD_MIN_SIZE_BYTES=512 \
BENCH_MD_MAX_SIZE_BYTES=2048 \
test/bench-network/compose.yml
+2
View File
@@ -90,6 +90,7 @@ services:
BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
BENCH_VERIFY_MODE: ${BENCH_VERIFY_MODE:-all}
BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-20}
BENCH_TETHERING_VPN_RTT_MS: ${BENCH_TETHERING_VPN_RTT_MS:-120}
BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
@@ -220,6 +221,7 @@ services:
BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
BENCH_VERIFY_MODE: ${BENCH_VERIFY_MODE:-all}
BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000}