public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-07-09 06:13:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add scoped CLI benchmark metadata

Browse Source
This commit is contained in:
vorotamoroz
2026-07-09 01:47:34 +00:00
parent 236f9c434e
commit dbb905f0d5
6 changed files with 593 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/terms.md
+1 -2
View File
@@ -65,7 +65,7 @@ All guidelines and conventions listed below are disclosed and maintained solely
- livesync-serverpeer / webpeer
- Pseudo-clients that assist in WebRTC peer-to-peer communication.
- Metadata (File metadata)
- A database document that stores properties of a file, including its filename, path, size, modification time, conflict history, and references (hashes) of the chunks that comprise the file's content. In Self-hosted LiveSync, metadata is stored separately from the actual file content to enable efficient synchronisation and versioning.
- A database document that stores properties of a file, including its filename, path, size, modification time, and references (hashes) of the chunks that comprise the file's content. Conflict state is carried by the surrounding PouchDB/CouchDB revision metadata rather than by a separate history field inside the file metadata document. In Self-hosted LiveSync, file metadata is stored separately from the actual file content to enable efficient synchronisation and versioning.
- OneShot Sync
- A single, immediate bidirectional synchronisation (pull then push) triggered on demand or on specific events, as opposed to continuous (live) replication.
- Overwrite Server Data with This Device's Files
@@ -100,4 +100,3 @@ All guidelines and conventions listed below are disclosed and maintained solely
- An optimisation that groups multiple local file edits together over a short delay before committing them to the local database, reducing the number of database write operations.
- WebRTC P2P (Peer-to-Peer)
- A synchronisation method enabling direct communication between devices without a central server database.
src/apps/cli/testdeno/bench-couchdb.ts
+134 -28
View File
@@ -1,8 +1,18 @@
import { TempDir } from "./helpers/temp.ts";
import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
import {
applyRemoteSyncSettings,
initSettingsFile,
} from "./helpers/settings.ts";
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts";
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
import {
createCouchdbDatabase,
startCouchdb,
stopCouchdb,
} from "./helpers/docker.ts";
import {
createDeterministicDataset,
type DatasetEntry,
} from "./helpers/dataset.ts";
type BenchmarkConfig = {
caseName: string;
@@ -26,6 +36,8 @@ type BenchmarkConfig = {
simulationTier: string;
networkProfile: string;
networkModel: string;
measurementScope: string;
limitations: string[];
};
function readEnvString(name: string, fallback: string): string {
@@ -54,6 +66,30 @@ function readEnvBool(name: string, fallback: boolean): boolean {
return /^(1|true|yes|on)$/i.test(raw.trim());
}
function readEnvStringArray(name: string, fallback: string[]): string[] {
const raw = Deno.env.get(name)?.trim();
if (!raw) {
return fallback;
}
try {
const parsed = JSON.parse(raw);
if (
Array.isArray(parsed) &&
parsed.every((item) => typeof item === "string")
) {
return parsed;
}
} catch {
// Fall through to pipe-separated parsing for hand-written invocations.
}
return raw
.split("|")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
function nowMs(): number {
return performance.now();
}
@@ -76,26 +112,57 @@ function formatBytes(value: number): string {
function buildConfig(): BenchmarkConfig {
return {
caseName: readEnvString("BENCH_CASE", "couchdb-baseline"),
couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"),
couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"),
couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")),
couchdbPassword: readEnvString("BENCH_COUCHDB_PASSWORD", readEnvString("password", "password")),
couchdbDbname: readEnvString("BENCH_COUCHDB_DBNAME", `bench-couchdb-${Date.now()}`),
couchdbBackendUri: readEnvString(
"BENCH_COUCHDB_BACKEND_URI",
"http://127.0.0.1:5989",
),
couchdbProxyUri: readEnvString(
"BENCH_COUCHDB_URI",
"http://127.0.0.1:15989",
),
couchdbUser: readEnvString(
"BENCH_COUCHDB_USER",
readEnvString("username", "admin"),
),
couchdbPassword: readEnvString(
"BENCH_COUCHDB_PASSWORD",
readEnvString("password", "password"),
),
couchdbDbname: readEnvString(
"BENCH_COUCHDB_DBNAME",
`bench-couchdb-${Date.now()}`,
),
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)),
mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)),
mdMinSizeBytes: Math.floor(
readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024),
),
mdMaxSizeBytes: Math.floor(
readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024),
),
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)),
binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
binSizeBytes: Math.floor(
readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024),
),
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)),
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
encrypt: readEnvBool("BENCH_ENCRYPT", true),
managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true),
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "http-latency-proxy"),
networkProfile: readEnvString(
"BENCH_NETWORK_PROFILE",
"http-latency-proxy",
),
networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-http-proxy"),
measurementScope: readEnvString(
"BENCH_MEASUREMENT_SCOPE",
"Two one-shot synchronisation phases through a CouchDB-compatible remote-store path.",
),
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
"This benchmark result is scoped to the configured dataset, remote store, and network model.",
]),
};
}
@@ -130,7 +197,9 @@ type ProxyHandle = {
note: string;
};
function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requestedRttMs: number }): ProxyHandle {
function startCouchdbProxy(
options: { backendUri: string; proxyUri: string; requestedRttMs: number },
): ProxyHandle {
const backend = new URL(options.backendUri);
const proxy = new URL(options.proxyUri);
const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2));
@@ -182,12 +251,13 @@ function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requ
statusText: upstream.statusText,
headers: responseHeaders,
});
}
},
);
return {
applied: true,
note: `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`,
note:
`local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`,
stop: async () => {
controller.abort();
await listener.finished.catch(() => {});
@@ -211,14 +281,21 @@ async function main(): Promise<void> {
await initSettingsFile(settingsB);
if (config.managedCouchdb) {
await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
await startCouchdb(
config.couchdbBackendUri,
config.couchdbUser,
config.couchdbPassword,
config.couchdbDbname,
);
} else {
console.log(`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`);
console.log(
`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`,
);
await createCouchdbDatabase(
config.couchdbBackendUri,
config.couchdbUser,
config.couchdbPassword,
config.couchdbDbname
config.couchdbDbname,
);
}
@@ -275,12 +352,21 @@ async function main(): Promise<void> {
const sampleFiles = pickSampleFiles(seedFiles.entries);
for (const sample of sampleFiles) {
const pulledPath = workDir.join(`pulled-${sample.relativePath.split("/").join("_")}`);
await runCliOrFail(vaultB, "--settings", settingsB, "pull", sample.relativePath, pulledPath);
const pulledPath = workDir.join(
`pulled-${sample.relativePath.split("/").join("_")}`,
);
await runCliOrFail(
vaultB,
"--settings",
settingsB,
"pull",
sample.relativePath,
pulledPath,
);
await assertFilesEqual(
sample.absolutePath,
pulledPath,
`sample file mismatch after CouchDB sync: ${sample.relativePath}`
`sample file mismatch after CouchDB sync: ${sample.relativePath}`,
);
}
@@ -294,6 +380,8 @@ async function main(): Promise<void> {
simulationTier: config.simulationTier,
networkProfile: config.networkProfile,
networkModel: config.networkModel,
measurementScope: config.measurementScope,
limitations: config.limitations,
rttRequestedMs: config.requestedRttMs,
proxyApplied: proxy.applied,
proxyNote: proxy.note,
@@ -306,22 +394,40 @@ async function main(): Promise<void> {
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
syncAElapsedMs: Number(syncAElapsed.toFixed(1)),
syncBElapsedMs: Number(syncBElapsed.toFixed(1)),
totalSyncElapsedMs: Number((syncAElapsed + syncBElapsed).toFixed(1)),
throughputBytesPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000)).toFixed(2)),
totalSyncElapsedMs: Number(
(syncAElapsed + syncBElapsed).toFixed(1),
),
throughputBytesPerSec: Number(
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000))
.toFixed(
2,
),
),
throughputMiBPerSec: Number(
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / 1024 / 1024).toFixed(4)
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) /
1024 /
1024).toFixed(4),
),
};
if (resultPath) {
await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2));
await Deno.writeTextFile(
resultPath,
JSON.stringify(result, null, 2),
);
}
console.log(JSON.stringify(result, null, 2));
console.error(
`[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${formatBytes(seedFiles.totalBytes)}) in ${formatMs(
mirrorElapsed
)}, synced in ${formatMs(syncAElapsed + syncBElapsed)} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
`[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${
formatBytes(seedFiles.totalBytes)
}) in ${
formatMs(
mirrorElapsed,
)
}, synced in ${
formatMs(syncAElapsed + syncBElapsed)
} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`,
);
} finally {
await proxy.stop();
src/apps/cli/testdeno/bench-network-cases.ts
+184 -53
View File
@@ -1,9 +1,11 @@
type BenchmarkCase = {
export type BenchmarkCase = {
name: string;
runner: "p2p" | "couchdb";
description: string;
dataPath: string;
trustBoundary: string;
measurementScope: string;
limitations: string[];
env: Record<string, string>;
};
@@ -25,16 +27,26 @@ function timestamp(): string {
const d = new Date();
const pad = (n: number) => String(n).padStart(2, "0");
return (
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${
pad(d.getUTCDate())
}-` +
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${
pad(d.getUTCSeconds())
}`
);
}
function buildBaseEnv(): Record<string, string> {
return {
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
BENCH_MD_MIN_SIZE_BYTES: readEnvString(
"BENCH_MD_MIN_SIZE_BYTES",
"512",
),
BENCH_MD_MAX_SIZE_BYTES: readEnvString(
"BENCH_MD_MAX_SIZE_BYTES",
"2048",
),
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
@@ -44,33 +56,74 @@ function buildBaseEnv(): Record<string, string> {
};
}
function buildCases(): BenchmarkCase[] {
function withScopeEnv(
env: Record<string, string>,
options: Pick<BenchmarkCase, "measurementScope" | "limitations">,
): Record<string, string> {
return {
...env,
BENCH_MEASUREMENT_SCOPE: options.measurementScope,
BENCH_LIMITATIONS_JSON: JSON.stringify(options.limitations),
};
}
function defineCase(testCase: BenchmarkCase): BenchmarkCase {
return {
...testCase,
env: withScopeEnv(testCase.env, testCase),
};
}
export function buildCases(): BenchmarkCase[] {
const base = buildBaseEnv();
const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20");
const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120");
const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478");
const shimCouchdbUri = readEnvString("BENCH_SHIM_COUCHDB_URI", "http://couchdb-shim:5984");
const signallingShimRelay = readEnvString("BENCH_SIGNAL_SHIM_RELAY", "ws://p2p-signalling-shim:7777/");
const localTurnServers = readEnvString(
"BENCH_LOCAL_TURN_SERVERS",
"turn:127.0.0.1:3478",
);
const shimCouchdbUri = readEnvString(
"BENCH_SHIM_COUCHDB_URI",
"http://couchdb-shim:5984",
);
const signallingShimRelay = readEnvString(
"BENCH_SIGNAL_SHIM_RELAY",
"ws://p2p-signalling-shim:7777/",
);
return [
{
defineCase({
name: "couchdb-baseline",
runner: "couchdb",
description: "Standard self-hosted CouchDB path through a local latency proxy.",
description:
"Standard self-hosted CouchDB path through a local latency proxy.",
dataPath: "Device A -> CouchDB -> Device B",
trustBoundary: "CouchDB operator and network path",
measurementScope:
"Two one-shot synchronisation phases through a CouchDB-compatible remote-store path with a local HTTP latency proxy.",
limitations: [
"This is not a full netem model of packet loss, jitter, MTU, bandwidth limits, or VPN encapsulation.",
"This result should be compared with P2P only as a remote-store baseline under the same deterministic dataset.",
],
env: {
...base,
BENCH_CASE: "couchdb-baseline",
BENCH_COUCHDB_RTT_MS: couchdbRtt,
},
},
{
}),
defineCase({
name: "p2p-direct-local",
runner: "p2p",
description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
description:
"Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
dataPath: "Device A -> Device B",
trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
measurementScope:
"One CLI P2P synchronisation phase over a local WebRTC DataChannel after Nostr signalling, with TURN disabled.",
limitations: [
"This does not measure first-peer discovery latency, public relay operation, mobile carrier behaviour, or TURN-relayed throughput.",
"This small-dataset run should not be treated as a WAN, VPN, or large binary initial synchronisation measurement.",
],
env: {
...base,
BENCH_CASE: "p2p-direct-local",
@@ -78,29 +131,44 @@ function buildCases(): BenchmarkCase[] {
BENCH_SIMULATION_TIER: "1",
BENCH_NETWORK_PROFILE: "local-direct",
BENCH_NETWORK_MODEL: "local-runner-webrtc",
BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "turn-disabled-but-selected-ice-pair-not-collected",
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"turn-disabled-but-selected-ice-pair-not-collected",
},
},
{
}),
defineCase({
name: "couchdb-tethering-vpn-proxy",
runner: "couchdb",
description:
"Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.",
dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
dataPath:
"Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
trustBoundary: "VPN/network path and CouchDB operator",
measurementScope:
"Two one-shot CouchDB synchronisation phases with additional requested RTT through the local HTTP proxy.",
limitations: [
"This approximates request latency only and does not model loss, jitter, MTU, bandwidth limits, carrier NAT, or VPN encapsulation.",
"Use the Tier 2 netem shim cases for a stronger constrained-network fixture.",
],
env: {
...base,
BENCH_CASE: "couchdb-tethering-vpn-proxy",
BENCH_COUCHDB_RTT_MS: tetheringVpnRtt,
},
},
{
}),
defineCase({
name: "couchdb-netem-home-wifi",
runner: "couchdb",
description:
"Tier 2 CouchDB path through the Compose netem TCP shim using the home-wifi profile.",
dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
dataPath:
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary: "CouchDB operator and constrained network shim",
measurementScope:
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the home-wifi netem profile.",
limitations: [
"This shapes the CouchDB TCP path, not the WebRTC P2P data path.",
"The fixture remains a reproducible network emulation, not a field measurement on a real home network.",
],
env: {
...base,
BENCH_CASE: "couchdb-netem-home-wifi",
@@ -110,14 +178,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_NETWORK_PROFILE: "home-wifi",
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
},
},
{
}),
defineCase({
name: "couchdb-netem-tethering-vpn",
runner: "couchdb",
description:
"Tier 2 CouchDB path through the Compose netem TCP shim using a tethering-vpn profile.",
dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary: "CouchDB operator and constrained smartphone/VPN-like network shim",
dataPath:
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary:
"CouchDB operator and constrained smartphone/VPN-like network shim",
measurementScope:
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the tethering-vpn netem profile.",
limitations: [
"This shapes the CouchDB TCP path, not the WebRTC P2P data path.",
"The profile approximates smartphone/VPN constraints but is not a field measurement on a real tethered VPN connection.",
],
env: {
...base,
BENCH_CASE: "couchdb-netem-tethering-vpn",
@@ -127,14 +203,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_NETWORK_PROFILE: "tethering-vpn",
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
},
},
{
}),
defineCase({
name: "p2p-smartphone-vpn-direct",
runner: "p2p",
description:
"Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.",
dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds",
trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata",
dataPath:
"Device A -> Device B when WebRTC direct connectivity succeeds",
trustBoundary:
"Smartphone/VPN routing policy plus Nostr signalling metadata",
measurementScope:
"Structural placeholder for direct P2P measurements on a real smartphone tethering/VPN path.",
limitations: [
"In the local runner this is unshaped and must not be reported as smartphone, VPN, WAN, or Tier 2 evidence.",
"Use only when the command is executed on the intended real network path and the selected ICE candidate pair is recorded.",
],
env: {
...base,
BENCH_CASE: "p2p-smartphone-vpn-direct",
@@ -145,14 +229,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"structural-placeholder-only; selected ICE pair may be collected, but the path is not shaped",
},
},
{
}),
defineCase({
name: "p2p-signalling-netem-home-wifi",
runner: "p2p",
description:
"Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.",
dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary: "Nostr signalling metadata through constrained network shim; no TURN relay",
dataPath:
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary:
"Nostr signalling metadata through constrained network shim; no TURN relay",
measurementScope:
"Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the home-wifi netem profile.",
limitations: [
"This does not shape the selected WebRTC DataChannel note-data path.",
"This supports only the claim that constrained signalling access does not place note data on the relay path when a non-relayed ICE path is selected.",
],
env: {
...base,
BENCH_CASE: "p2p-signalling-netem-home-wifi",
@@ -164,14 +256,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"selected ICE pair collected; only Nostr signalling path is shaped",
},
},
{
}),
defineCase({
name: "p2p-signalling-netem-tethering-vpn",
runner: "p2p",
description:
"Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.",
dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary: "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
dataPath:
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary:
"Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
measurementScope:
"Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the tethering-vpn netem profile.",
limitations: [
"This does not shape the selected WebRTC DataChannel note-data path.",
"The profile approximates constrained relay access and is not a field measurement on a real tethered VPN connection.",
],
env: {
...base,
BENCH_CASE: "p2p-signalling-netem-tethering-vpn",
@@ -183,13 +283,20 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"selected ICE pair collected; only Nostr signalling path is shaped",
},
},
{
}),
defineCase({
name: "p2p-user-turn",
runner: "p2p",
description: "Optional fallback path through a local user-controlled TURN server.",
description:
"Optional fallback path through a local user-controlled TURN server.",
dataPath: "Device A -> user-controlled TURN -> Device B",
trustBoundary: "User-controlled TURN server",
measurementScope:
"Optional local TURN fallback wiring check with a user-controlled TURN server configured.",
limitations: [
"TURN configuration does not prove that the selected ICE path was relayed; interpret the recorded candidate pair.",
"This is not evidence for public TURN relay privacy, throughput, or availability.",
],
env: {
...base,
BENCH_CASE: "p2p-user-turn",
@@ -200,7 +307,7 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"turn-configured; selected ICE pair may still be direct or relayed, so interpret the recorded candidate types",
},
},
}),
];
}
@@ -208,9 +315,11 @@ async function runCase(
testCase: BenchmarkCase,
outputDir: string,
repeatIndex: number,
repeatCount: number
repeatCount: number,
): Promise<Record<string, unknown>> {
const suffix = repeatCount > 1 ? `-r${String(repeatIndex).padStart(2, "0")}` : "";
const suffix = repeatCount > 1
? `-r${String(repeatIndex).padStart(2, "0")}`
: "";
const resultPath = `${outputDir}/${testCase.name}${suffix}.json`;
const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb";
const env = {
@@ -221,8 +330,12 @@ async function runCase(
BENCH_REPEAT_COUNT: String(repeatCount),
};
const repeatLabel = repeatCount > 1 ? ` (${repeatIndex}/${repeatCount})` : "";
console.log(`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`);
const repeatLabel = repeatCount > 1
? ` (${repeatIndex}/${repeatCount})`
: "";
console.log(
`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`,
);
const command = new Deno.Command("deno", {
args: ["task", taskName],
cwd: import.meta.dirname,
@@ -238,7 +351,10 @@ async function runCase(
throw new Error(`case failed: ${testCase.name} (exit ${status.code})`);
}
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<
string,
unknown
>;
return {
...testCase,
repeatIndex,
@@ -249,7 +365,10 @@ async function runCase(
}
function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local");
const requested = readEnvString(
"BENCH_CASES",
"couchdb-baseline,p2p-direct-local",
);
const names = requested
.split(",")
.map((v) => v.trim())
@@ -258,14 +377,21 @@ function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
return names.map((name) => {
const found = byName.get(name);
if (!found) {
throw new Error(`Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`);
throw new Error(
`Unknown BENCH_CASES entry '${name}'. Available: ${
allCases.map((c) => c.name).join(", ")
}`,
);
}
return found;
});
}
async function main(): Promise<void> {
const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`);
const outRoot = readEnvString(
"BENCH_CASES_ROOT",
`${import.meta.dirname}/bench-results`,
);
const outputDir = `${outRoot}/cases-${timestamp()}`;
await Deno.mkdir(outputDir, { recursive: true });
@@ -282,14 +408,16 @@ async function main(): Promise<void> {
availableCases: allCases,
},
null,
2
)
2,
),
);
const results: Record<string, unknown>[] = [];
for (const testCase of cases) {
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
results.push(await runCase(testCase, outputDir, repeatIndex, repeatCount));
results.push(
await runCase(testCase, outputDir, repeatIndex, repeatCount),
);
}
}
@@ -299,7 +427,10 @@ async function main(): Promise<void> {
repeatCount,
results,
};
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
await Deno.writeTextFile(
`${outputDir}/summary.json`,
JSON.stringify(summary, null, 2),
);
console.log(JSON.stringify(summary, null, 2));
console.log(`[bench-cases] result directory: ${outputDir}`);
}
src/apps/cli/testdeno/bench-p2p.ts
+139 -35
View File
@@ -1,5 +1,9 @@
import { TempDir } from "./helpers/temp.ts";
import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
import {
applyP2pSettings,
applyP2pTestTweaks,
initSettingsFile,
} from "./helpers/settings.ts";
import { startCliInBackground } from "./helpers/backgroundCli.ts";
import {
discoverPeer,
@@ -9,7 +13,10 @@ import {
stopLocalRelayIfStarted,
} from "./helpers/p2p.ts";
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
import {
createDeterministicDataset,
type DatasetEntry,
} from "./helpers/dataset.ts";
type BenchmarkConfig = {
caseName: string;
@@ -31,6 +38,8 @@ type BenchmarkConfig = {
networkProfile: string;
networkModel: string;
candidatePathVerification: string;
measurementScope: string;
limitations: string[];
};
type P2PConnectionStats = {
@@ -83,6 +92,30 @@ function readEnvNumber(name: string, fallback: number): number {
return parsed;
}
function readEnvStringArray(name: string, fallback: string[]): string[] {
const raw = Deno.env.get(name)?.trim();
if (!raw) {
return fallback;
}
try {
const parsed = JSON.parse(raw);
if (
Array.isArray(parsed) &&
parsed.every((item) => typeof item === "string")
) {
return parsed;
}
} catch {
// Fall through to comma-separated parsing for hand-written invocations.
}
return raw
.split("|")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
function nowMs(): number {
return performance.now();
}
@@ -107,23 +140,45 @@ function buildConfig(): BenchmarkConfig {
return {
caseName: readEnvString("BENCH_CASE", "p2p-direct-local"),
relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"),
appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
appId: readEnvString(
"BENCH_APP_ID",
"self-hosted-livesync-cli-benchmark",
),
roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`),
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)),
mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)),
mdMinSizeBytes: Math.floor(
readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024),
),
mdMaxSizeBytes: Math.floor(
readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024),
),
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)),
binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
binSizeBytes: Math.floor(
readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024),
),
peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20),
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "local-direct"),
networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-runner-webrtc"),
candidatePathVerification: readEnvString("BENCH_P2P_CANDIDATE_PATH_VERIFICATION", "not-collected"),
networkModel: readEnvString(
"BENCH_NETWORK_MODEL",
"local-runner-webrtc",
),
candidatePathVerification: readEnvString(
"BENCH_P2P_CANDIDATE_PATH_VERIFICATION",
"not-collected",
),
measurementScope: readEnvString(
"BENCH_MEASUREMENT_SCOPE",
"One CLI P2P synchronisation phase over WebRTC DataChannel after signalling.",
),
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
"This benchmark result is scoped to the configured dataset, network model, and selected ICE path.",
]),
};
}
@@ -152,7 +207,9 @@ function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
return [...unique.values()];
}
async function readLatestP2PConnectionStats(statsPath: string): Promise<P2PConnectionStats | undefined> {
async function readLatestP2PConnectionStats(
statsPath: string,
): Promise<P2PConnectionStats | undefined> {
try {
const text = await Deno.readTextFile(statsPath);
const lines = text
@@ -200,7 +257,7 @@ async function main(): Promise<void> {
config.appId,
config.relay,
"~.*",
config.turnServers
config.turnServers,
),
applyP2pSettings(
clientSettings,
@@ -209,13 +266,21 @@ async function main(): Promise<void> {
config.appId,
config.relay,
"~.*",
config.turnServers
config.turnServers,
),
]);
await Promise.all([
applyP2pTestTweaks(hostSettings, "p2p-bench-host", config.passphrase),
applyP2pTestTweaks(clientSettings, "p2p-bench-client", config.passphrase),
applyP2pTestTweaks(
hostSettings,
"p2p-bench-host",
config.passphrase,
),
applyP2pTestTweaks(
clientSettings,
"p2p-bench-client",
config.passphrase,
),
]);
const seedFiles = await createDeterministicDataset({
@@ -233,15 +298,25 @@ async function main(): Promise<void> {
await runCliOrFail(hostVault, "--settings", hostSettings, "mirror");
const mirrorElapsed = nowMs() - mirrorStart;
const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
const host = startCliInBackground(
hostVault,
"--settings",
hostSettings,
"p2p-host",
);
try {
const hostReadyStart = nowMs();
await host.waitUntilContains("P2P host is running", 20000);
const hostReadyElapsed = nowMs() - hostReadyStart;
const peerDiscoveryCommandStart = nowMs();
const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds);
const peerDiscoveryCommandElapsed = nowMs() - peerDiscoveryCommandStart;
const peer = await discoverPeer(
clientVault,
clientSettings,
config.peersTimeoutSeconds,
);
const peerDiscoveryCommandElapsed = nowMs() -
peerDiscoveryCommandStart;
const syncStart = nowMs();
await runCliOrFail(
@@ -250,22 +325,33 @@ async function main(): Promise<void> {
clientSettings,
"p2p-sync",
peer.id,
String(config.syncTimeoutSeconds)
String(config.syncTimeoutSeconds),
);
const syncElapsed = nowMs() - syncStart;
const sampleFiles = pickSampleFiles(seedFiles.entries);
for (const sample of sampleFiles) {
const pulledPath = workDir.join(`pulled-${sample.relativePath.replaceAll("/", "_")}`);
await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath);
const pulledPath = workDir.join(
`pulled-${sample.relativePath.replaceAll("/", "_")}`,
);
await runCliOrFail(
clientVault,
"--settings",
clientSettings,
"pull",
sample.relativePath,
pulledPath,
);
await assertFilesEqual(
sample.absolutePath,
pulledPath,
`sample file mismatch after sync: ${sample.relativePath}`
`sample file mismatch after sync: ${sample.relativePath}`,
);
}
const p2pConnectionStats = await readLatestP2PConnectionStats(p2pStatsPath);
const p2pConnectionStats = await readLatestP2PConnectionStats(
p2pStatsPath,
);
const result = {
caseName: config.caseName,
mode: "p2p-cli-benchmark",
@@ -275,15 +361,19 @@ async function main(): Promise<void> {
simulationTier: config.simulationTier,
networkProfile: config.networkProfile,
networkModel: config.networkModel,
p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
p2pCandidatePathVerification: p2pConnectionStats?.candidatePathCollected
? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
: config.candidatePathVerification,
measurementScope: config.measurementScope,
limitations: config.limitations,
p2pCandidatePathVerified:
p2pConnectionStats?.candidatePathCollected === true,
p2pCandidatePathVerification:
p2pConnectionStats?.candidatePathCollected
? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
: config.candidatePathVerification,
p2pCandidatePathNote: p2pConnectionStats?.candidatePathCollected
? "The selected ICE candidate pair was collected by the CLI benchmark. Interpret the path from the candidate types; do not infer TURN use from configuration alone."
: config.turnServers.trim().length > 0
? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run."
: "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.",
? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run."
: "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.",
p2pConnectionStats,
appId: config.appId,
roomId: config.roomId,
@@ -298,25 +388,39 @@ async function main(): Promise<void> {
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
peerDiscoveryCommandElapsedMs: Number(peerDiscoveryCommandElapsed.toFixed(1)),
peerDiscoveryCommandElapsedMs: Number(
peerDiscoveryCommandElapsed.toFixed(1),
),
peerDiscoveryNote:
"p2p-peers waits for the requested timeout before printing discovered peers, so this is command duration, not first-peer latency.",
syncElapsedMs: Number(syncElapsed.toFixed(1)),
throughputBytesPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2)),
throughputMiBPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024).toFixed(4)),
throughputBytesPerSec: Number(
(seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2),
),
throughputMiBPerSec: Number(
(seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024)
.toFixed(
4,
),
),
};
if (resultPath) {
await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2));
await Deno.writeTextFile(
resultPath,
JSON.stringify(result, null, 2),
);
}
console.log(JSON.stringify(result, null, 2));
console.error(
`[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes(
seedFiles.totalBytes
)}) in ${formatMs(mirrorElapsed)}, ` +
`[Benchmark] mirrored ${seedFiles.totalFiles} files (${
formatBytes(
seedFiles.totalBytes,
)
}) in ${formatMs(mirrorElapsed)}, ` +
`synced in ${formatMs(syncElapsed)} ` +
`(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
`(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`,
);
} finally {
await host.stop();
src/apps/cli/testdeno/deno.json
+1
View File
@@ -15,6 +15,7 @@
"test:p2p-sync": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts",
"test:p2p-three-nodes": "deno test --env-file=.test.env -A --no-check test-p2p-three-nodes-conflict.ts",
"test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts",
"test:benchmark-contract": "deno test --env-file=.test.env -A --no-check test-benchmark-contract.ts",
"bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts",
"bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts",
"bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts",
src/apps/cli/testdeno/test-benchmark-contract.ts
+134
View File
@@ -0,0 +1,134 @@
import { assert, assertEquals, assertStringIncludes } from "@std/assert";
import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts";
function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase {
const found = cases.find((testCase) => testCase.name === name);
assert(found, `missing benchmark case: ${name}`);
return found;
}
function parsedLimitations(testCase: BenchmarkCase): string[] {
const raw = testCase.env.BENCH_LIMITATIONS_JSON;
assert(
raw,
`${testCase.name} must pass BENCH_LIMITATIONS_JSON to benchmark result output`,
);
const parsed = JSON.parse(raw);
assert(
Array.isArray(parsed),
`${testCase.name} limitations must be an array`,
);
assert(
parsed.every((item) =>
typeof item === "string" && item.trim().length > 0
),
);
return parsed;
}
Deno.test("benchmark cases record scope and limitations for paper use", () => {
const cases = buildCases();
assert(cases.length > 0);
for (const testCase of cases) {
assert(
testCase.description.trim().length > 0,
`${testCase.name} must describe the case`,
);
assert(
testCase.dataPath.trim().length > 0,
`${testCase.name} must describe the data path`,
);
assert(
testCase.trustBoundary.trim().length > 0,
`${testCase.name} must describe the trust boundary`,
);
assert(
testCase.measurementScope.trim().length > 0,
`${testCase.name} must describe the measurement scope`,
);
assert(
testCase.limitations.length > 0,
`${testCase.name} must list limitations`,
);
assertEquals(
testCase.env.BENCH_MEASUREMENT_SCOPE,
testCase.measurementScope,
);
assertEquals(parsedLimitations(testCase), testCase.limitations);
}
});
Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => {
const cases = buildCases();
for (
const name of [
"p2p-signalling-netem-home-wifi",
"p2p-signalling-netem-tethering-vpn",
]
) {
const testCase = getCase(cases, name);
assertEquals(testCase.runner, "p2p");
assertEquals(testCase.env.BENCH_TURN_SERVERS, "");
assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2");
assertEquals(
testCase.env.BENCH_NETWORK_MODEL,
"compose-netem-signalling-shim",
);
assertStringIncludes(testCase.dataPath, "WebRTC DataChannel");
assertStringIncludes(testCase.dataPath, "Nostr signalling");
assert(
testCase.limitations.some((limitation) =>
limitation.includes("does not shape the selected WebRTC")
),
`${name} must avoid claiming that the P2P note-data path was shaped`,
);
}
});
Deno.test("placeholder and TURN cases are clearly non-evidence for broad P2P performance", () => {
const cases = buildCases();
const smartphone = getCase(cases, "p2p-smartphone-vpn-direct");
assertEquals(smartphone.env.BENCH_SIMULATION_TIER, "unmeasured");
assertEquals(smartphone.env.BENCH_NETWORK_MODEL, "local-runner-no-netem");
assert(
smartphone.limitations.some((limitation) =>
limitation.includes("must not be reported as smartphone")
),
"smartphone/VPN placeholder must not be usable as field evidence by accident",
);
const turn = getCase(cases, "p2p-user-turn");
assertStringIncludes(turn.env.BENCH_TURN_SERVERS, "turn:");
assert(
turn.limitations.some((limitation) =>
limitation.includes(
"does not prove that the selected ICE path was relayed",
)
),
"TURN case must require selected ICE candidate interpretation",
);
});
Deno.test("CouchDB netem cases are marked as remote-store baselines", () => {
const cases = buildCases();
for (
const name of ["couchdb-netem-home-wifi", "couchdb-netem-tethering-vpn"]
) {
const testCase = getCase(cases, name);
assertEquals(testCase.runner, "couchdb");
assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2");
assertEquals(
testCase.env.BENCH_NETWORK_MODEL,
"compose-netem-tcp-shim",
);
assertStringIncludes(testCase.measurementScope, "CouchDB");
assert(
testCase.limitations.some((limitation) =>
limitation.includes("not the WebRTC P2P data path")
),
`${name} must remain scoped to the CouchDB remote-store path`,
);
}
});