public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-07-17 10:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Normalise tracked text files and formatter scope

Browse Source
This commit is contained in:
vorotamoroz
2026-07-17 01:05:29 +00:00
parent e114f66fb2
commit 3a1a8948d2
13 changed files with 474 additions and 730 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+5 -1
View File
@@ -13,6 +13,11 @@
*.mjs text eol=lf *.mjs text eol=lf
*.css text eol=lf *.css text eol=lf
# Extensionless text files
.gitignore text eol=lf
.dockerignore text eol=lf
Dockerfile text eol=lf
# Binary files — no line ending conversion # Binary files — no line ending conversion
*.png binary *.png binary
*.jpg binary *.jpg binary
@@ -21,4 +26,3 @@
*.ico binary *.ico binary
*.woff2 binary *.woff2 binary
*.woff binary *.woff binary
*.sh text eol=lf
.prettierignore
+1
View File
@@ -2,3 +2,4 @@ pouchdb-browser.js
main_org.js main_org.js
main.js main.js
_types/** _types/**
src/lib/**
src/apps/_test/storageAdapterContract.ts
+12 -3
View File
@@ -87,7 +87,10 @@ export const storageAdapterContractCases: readonly StorageAdapterContractCase[]
name: "keeps operations inside the configured root", name: "keeps operations inside the configured root",
async run(adapter) { async run(adapter) {
await assertRejects(() => adapter.exists("../outside"), "parent traversal should be rejected"); await assertRejects(() => adapter.exists("../outside"), "parent traversal should be rejected");
await assertRejects(() => adapter.write("nested/../outside", "content"), "nested traversal should be rejected"); await assertRejects(
() => adapter.write("nested/../outside", "content"),
"nested traversal should be rejected"
);
await assertRejects(() => adapter.read("/absolute"), "absolute paths should be rejected"); await assertRejects(() => adapter.read("/absolute"), "absolute paths should be rejected");
await assertRejects(() => adapter.read("C:\\absolute"), "drive-qualified paths should be rejected"); await assertRejects(() => adapter.read("C:\\absolute"), "drive-qualified paths should be rejected");
await assertRejects(() => adapter.read("nested\\outside"), "backslash-separated paths should be rejected"); await assertRejects(() => adapter.read("nested\\outside"), "backslash-separated paths should be rejected");
@@ -101,8 +104,14 @@ export const storageAdapterContractCases: readonly StorageAdapterContractCase[]
assertEqual(await adapter.exists(""), true, "the configured root should exist"); assertEqual(await adapter.exists(""), true, "the configured root should exist");
assertEqual((await adapter.stat(""))?.type, "folder", "the configured root should be a folder"); assertEqual((await adapter.stat(""))?.type, "folder", "the configured root should be a folder");
assertEqual(await adapter.list(""), { files: [], folders: [] }, "the configured root should be listable"); assertEqual(await adapter.list(""), { files: [], folders: [] }, "the configured root should be listable");
await assertRejects(() => adapter.write("", "content"), "writing over the configured root should be rejected"); await assertRejects(
await assertRejects(() => adapter.append("", "content"), "appending to the configured root should be rejected"); () => adapter.write("", "content"),
"writing over the configured root should be rejected"
);
await assertRejects(
() => adapter.append("", "content"),
"appending to the configured root should be rejected"
);
}, },
}, },
]; ];
src/apps/cli/testdeno/bench-couchdb.ts
+46 -124
View File
@@ -1,17 +1,8 @@
import { TempDir } from "./helpers/temp.ts"; import { TempDir } from "./helpers/temp.ts";
import { import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
applyRemoteSyncSettings,
initSettingsFile,
} from "./helpers/settings.ts";
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
import { import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts";
createCouchdbDatabase, import { createDeterministicDataset } from "./helpers/dataset.ts";
startCouchdb,
stopCouchdb,
} from "./helpers/docker.ts";
import {
createDeterministicDataset,
} from "./helpers/dataset.ts";
import { import {
type BenchmarkVerificationMode, type BenchmarkVerificationMode,
parseBenchmarkVerificationMode, parseBenchmarkVerificationMode,
@@ -81,10 +72,7 @@ function readEnvStringArray(name: string, fallback: string[]): string[] {
try { try {
const parsed = JSON.parse(raw); const parsed = JSON.parse(raw);
if ( if (Array.isArray(parsed) && parsed.every((item) => typeof item === "string")) {
Array.isArray(parsed) &&
parsed.every((item) => typeof item === "string")
) {
return parsed; return parsed;
} }
} catch { } catch {
@@ -119,60 +107,34 @@ function formatBytes(value: number): string {
function buildConfig(): BenchmarkConfig { function buildConfig(): BenchmarkConfig {
return { return {
caseName: readEnvString("BENCH_CASE", "couchdb-baseline"), caseName: readEnvString("BENCH_CASE", "couchdb-baseline"),
couchdbBackendUri: readEnvString( couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"),
"BENCH_COUCHDB_BACKEND_URI", couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"),
"http://127.0.0.1:5989", couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")),
), couchdbPassword: readEnvString("BENCH_COUCHDB_PASSWORD", readEnvString("password", "password")),
couchdbProxyUri: readEnvString( couchdbDbname: readEnvString("BENCH_COUCHDB_DBNAME", `bench-couchdb-${Date.now()}`),
"BENCH_COUCHDB_URI",
"http://127.0.0.1:15989",
),
couchdbUser: readEnvString(
"BENCH_COUCHDB_USER",
readEnvString("username", "admin"),
),
couchdbPassword: readEnvString(
"BENCH_COUCHDB_PASSWORD",
readEnvString("password", "password"),
),
couchdbDbname: readEnvString(
"BENCH_COUCHDB_DBNAME",
`bench-couchdb-${Date.now()}`,
),
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"), datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)), mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
mdMinSizeBytes: Math.floor( mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)),
readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024), mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)),
),
mdMaxSizeBytes: Math.floor(
readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024),
),
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)), binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)),
binSizeBytes: Math.floor( binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024),
),
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240), syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)), requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)),
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
encrypt: readEnvBool("BENCH_ENCRYPT", true), encrypt: readEnvBool("BENCH_ENCRYPT", true),
managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true), managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true),
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"), simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
networkProfile: readEnvString( networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "http-latency-proxy"),
"BENCH_NETWORK_PROFILE",
"http-latency-proxy",
),
networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-http-proxy"), networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-http-proxy"),
measurementScope: readEnvString( measurementScope: readEnvString(
"BENCH_MEASUREMENT_SCOPE", "BENCH_MEASUREMENT_SCOPE",
"Two one-shot synchronisation phases through a CouchDB-compatible remote-store path.", "Two one-shot synchronisation phases through a CouchDB-compatible remote-store path."
), ),
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [ limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
"This benchmark result is scoped to the configured dataset, remote store, and network model.", "This benchmark result is scoped to the configured dataset, remote store, and network model.",
]), ]),
verificationMode: parseBenchmarkVerificationMode( verificationMode: parseBenchmarkVerificationMode(Deno.env.get("BENCH_VERIFY_MODE")),
Deno.env.get("BENCH_VERIFY_MODE"),
),
repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)), repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)), repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
}; };
@@ -193,20 +155,17 @@ export type CouchdbProxyHandle = {
directionalDelayMs: number; directionalDelayMs: number;
}; };
export function startCouchdbProxy( export function startCouchdbProxy(options: {
options: { backendUri: string;
backendUri: string; proxyUri: string;
proxyUri: string; requestedRttMs: number;
requestedRttMs: number; delay?: (milliseconds: number) => Promise<void>;
delay?: (milliseconds: number) => Promise<void>; }): CouchdbProxyHandle {
},
): CouchdbProxyHandle {
const backend = new URL(options.backendUri); const backend = new URL(options.backendUri);
const proxy = new URL(options.proxyUri); const proxy = new URL(options.proxyUri);
const halfDelayMs = options.requestedRttMs / 2; const halfDelayMs = options.requestedRttMs / 2;
const delay = options.delay ?? const delay =
((milliseconds: number) => options.delay ?? ((milliseconds: number) => new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
const controller = new AbortController(); const controller = new AbortController();
const listener = Deno.serve( const listener = Deno.serve(
@@ -256,14 +215,13 @@ export function startCouchdbProxy(
statusText: upstream.statusText, statusText: upstream.statusText,
headers: responseHeaders, headers: responseHeaders,
}); });
}, }
); );
return { return {
applied: true, applied: true,
directionalDelayMs: halfDelayMs, directionalDelayMs: halfDelayMs,
note: note: `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms request-path and ${halfDelayMs}ms response-path delay`,
`local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms request-path and ${halfDelayMs}ms response-path delay`,
stop: async () => { stop: async () => {
controller.abort(); controller.abort();
await listener.finished.catch(() => {}); await listener.finished.catch(() => {});
@@ -287,21 +245,14 @@ async function main(): Promise<void> {
await initSettingsFile(settingsB); await initSettingsFile(settingsB);
if (config.managedCouchdb) { if (config.managedCouchdb) {
await startCouchdb( await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
config.couchdbBackendUri,
config.couchdbUser,
config.couchdbPassword,
config.couchdbDbname,
);
} else { } else {
console.log( console.log(`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`);
`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`,
);
await createCouchdbDatabase( await createCouchdbDatabase(
config.couchdbBackendUri, config.couchdbBackendUri,
config.couchdbUser, config.couchdbUser,
config.couchdbPassword, config.couchdbPassword,
config.couchdbDbname, config.couchdbDbname
); );
} }
@@ -356,28 +307,15 @@ async function main(): Promise<void> {
await runCliOrFail(vaultB, "--settings", settingsB, "sync"); await runCliOrFail(vaultB, "--settings", settingsB, "sync");
const syncBElapsed = nowMs() - syncBStart; const syncBElapsed = nowMs() - syncBStart;
const verification = await verifyBenchmarkDataset( const verification = await verifyBenchmarkDataset(seedFiles.entries, config.verificationMode, async (entry) => {
seedFiles.entries, const pulledPath = workDir.join(`pulled-${entry.relativePath.split("/").join("_")}`);
config.verificationMode, await runCliOrFail(vaultB, "--settings", settingsB, "pull", entry.relativePath, pulledPath);
async (entry) => { await assertFilesEqual(
const pulledPath = workDir.join( entry.absolutePath,
`pulled-${entry.relativePath.split("/").join("_")}`, pulledPath,
); `file mismatch after CouchDB sync: ${entry.relativePath}`
await runCliOrFail( );
vaultB, });
"--settings",
settingsB,
"pull",
entry.relativePath,
pulledPath,
);
await assertFilesEqual(
entry.absolutePath,
pulledPath,
`file mismatch after CouchDB sync: ${entry.relativePath}`,
);
},
);
const result = { const result = {
caseName: config.caseName, caseName: config.caseName,
@@ -409,40 +347,24 @@ async function main(): Promise<void> {
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
syncAElapsedMs: Number(syncAElapsed.toFixed(1)), syncAElapsedMs: Number(syncAElapsed.toFixed(1)),
syncBElapsedMs: Number(syncBElapsed.toFixed(1)), syncBElapsedMs: Number(syncBElapsed.toFixed(1)),
totalSyncElapsedMs: Number( totalSyncElapsedMs: Number((syncAElapsed + syncBElapsed).toFixed(1)),
(syncAElapsed + syncBElapsed).toFixed(1), throughputBytesPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000)).toFixed(2)),
),
throughputBytesPerSec: Number(
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000))
.toFixed(
2,
),
),
throughputMiBPerSec: Number( throughputMiBPerSec: Number(
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / (seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / 1024 / 1024).toFixed(4)
1024 /
1024).toFixed(4),
), ),
}; };
if (resultPath) { if (resultPath) {
await Deno.writeTextFile( await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2));
resultPath,
JSON.stringify(result, null, 2),
);
} }
console.log(JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
console.error( console.error(
`[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${ `[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${formatBytes(
formatBytes(seedFiles.totalBytes) seedFiles.totalBytes
}) in ${ )}) in ${formatMs(mirrorElapsed)}, synced in ${formatMs(
formatMs( syncAElapsed + syncBElapsed
mirrorElapsed, )} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
)
}, synced in ${
formatMs(syncAElapsed + syncBElapsed)
} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`,
); );
} finally { } finally {
await proxy.stop(); await proxy.stop();
src/apps/cli/testdeno/bench-latency-sweep.ts
+2 -5
View File
@@ -65,9 +65,7 @@ async function runBenchmark(options: {
repeatIndex: number; repeatIndex: number;
repeatCount: number; repeatCount: number;
}): Promise<Record<string, unknown>> { }): Promise<Record<string, unknown>> {
const suffix = options.repeatCount > 1 const suffix = options.repeatCount > 1 ? `-r${String(options.repeatIndex).padStart(2, "0")}` : "";
? `-r${String(options.repeatIndex).padStart(2, "0")}`
: "";
const resultPath = `${options.outputDir}/${options.name}${suffix}.json`; const resultPath = `${options.outputDir}/${options.name}${suffix}.json`;
const env = { const env = {
...Deno.env.toObject(), ...Deno.env.toObject(),
@@ -156,8 +154,7 @@ async function main(): Promise<void> {
const summary = { const summary = {
generatedAt: new Date().toISOString(), generatedAt: new Date().toISOString(),
outputDir, outputDir,
note: note: "This sweep applies half of each requested CouchDB RTT before forwarding requests and half before returning responses. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
"This sweep applies half of each requested CouchDB RTT before forwarding requests and half before returning responses. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
rtts, rtts,
repeatCount, repeatCount,
results, results,
src/apps/cli/testdeno/bench-network-cases.ts
+35 -91
View File
@@ -27,26 +27,16 @@ function timestamp(): string {
const d = new Date(); const d = new Date();
const pad = (n: number) => String(n).padStart(2, "0"); const pad = (n: number) => String(n).padStart(2, "0");
return ( return (
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${ `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
pad(d.getUTCDate()) `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
}-` +
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${
pad(d.getUTCSeconds())
}`
); );
} }
function buildBaseEnv(): Record<string, string> { function buildBaseEnv(): Record<string, string> {
return { return {
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"), BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
BENCH_MD_MIN_SIZE_BYTES: readEnvString( BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
"BENCH_MD_MIN_SIZE_BYTES", BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
"512",
),
BENCH_MD_MAX_SIZE_BYTES: readEnvString(
"BENCH_MD_MAX_SIZE_BYTES",
"2048",
),
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"), BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"), BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"), BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
@@ -59,7 +49,7 @@ function buildBaseEnv(): Record<string, string> {
function withScopeEnv( function withScopeEnv(
env: Record<string, string>, env: Record<string, string>,
options: Pick<BenchmarkCase, "measurementScope" | "limitations">, options: Pick<BenchmarkCase, "measurementScope" | "limitations">
): Record<string, string> { ): Record<string, string> {
return { return {
...env, ...env,
@@ -79,25 +69,15 @@ export function buildCases(): BenchmarkCase[] {
const base = buildBaseEnv(); const base = buildBaseEnv();
const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20"); const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20");
const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120"); const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120");
const localTurnServers = readEnvString( const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478");
"BENCH_LOCAL_TURN_SERVERS", const shimCouchdbUri = readEnvString("BENCH_SHIM_COUCHDB_URI", "http://couchdb-shim:5984");
"turn:127.0.0.1:3478", const signallingShimRelay = readEnvString("BENCH_SIGNAL_SHIM_RELAY", "ws://p2p-signalling-shim:7777/");
);
const shimCouchdbUri = readEnvString(
"BENCH_SHIM_COUCHDB_URI",
"http://couchdb-shim:5984",
);
const signallingShimRelay = readEnvString(
"BENCH_SIGNAL_SHIM_RELAY",
"ws://p2p-signalling-shim:7777/",
);
return [ return [
defineCase({ defineCase({
name: "couchdb-baseline", name: "couchdb-baseline",
runner: "couchdb", runner: "couchdb",
description: description: "Standard self-hosted CouchDB path through a local latency proxy.",
"Standard self-hosted CouchDB path through a local latency proxy.",
dataPath: "Device A -> CouchDB -> Device B", dataPath: "Device A -> CouchDB -> Device B",
trustBoundary: "CouchDB operator and network path", trustBoundary: "CouchDB operator and network path",
measurementScope: measurementScope:
@@ -115,8 +95,7 @@ export function buildCases(): BenchmarkCase[] {
defineCase({ defineCase({
name: "p2p-direct-local", name: "p2p-direct-local",
runner: "p2p", runner: "p2p",
description: description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
"Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
dataPath: "Device A -> Device B", dataPath: "Device A -> Device B",
trustBoundary: "Nostr relay for signalling metadata; no TURN relay", trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
measurementScope: measurementScope:
@@ -133,8 +112,7 @@ export function buildCases(): BenchmarkCase[] {
BENCH_SIMULATION_TIER: "1", BENCH_SIMULATION_TIER: "1",
BENCH_NETWORK_PROFILE: "local-direct", BENCH_NETWORK_PROFILE: "local-direct",
BENCH_NETWORK_MODEL: "local-runner-webrtc", BENCH_NETWORK_MODEL: "local-runner-webrtc",
BENCH_P2P_CANDIDATE_PATH_VERIFICATION: BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "turn-disabled-but-selected-ice-pair-not-collected",
"turn-disabled-but-selected-ice-pair-not-collected",
}, },
}), }),
defineCase({ defineCase({
@@ -142,8 +120,7 @@ export function buildCases(): BenchmarkCase[] {
runner: "couchdb", runner: "couchdb",
description: description:
"Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.", "Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.",
dataPath: dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
"Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
trustBoundary: "VPN/network path and CouchDB operator", trustBoundary: "VPN/network path and CouchDB operator",
measurementScope: measurementScope:
"Two one-shot CouchDB synchronisation phases with additional requested RTT through the local HTTP proxy.", "Two one-shot CouchDB synchronisation phases with additional requested RTT through the local HTTP proxy.",
@@ -160,10 +137,8 @@ export function buildCases(): BenchmarkCase[] {
defineCase({ defineCase({
name: "couchdb-netem-home-wifi", name: "couchdb-netem-home-wifi",
runner: "couchdb", runner: "couchdb",
description: description: "Tier 2 CouchDB path through the Compose netem TCP shim using the home-wifi profile.",
"Tier 2 CouchDB path through the Compose netem TCP shim using the home-wifi profile.", dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
dataPath:
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary: "CouchDB operator and constrained network shim", trustBoundary: "CouchDB operator and constrained network shim",
measurementScope: measurementScope:
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the home-wifi netem profile.", "Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the home-wifi netem profile.",
@@ -184,12 +159,9 @@ export function buildCases(): BenchmarkCase[] {
defineCase({ defineCase({
name: "couchdb-netem-tethering-vpn", name: "couchdb-netem-tethering-vpn",
runner: "couchdb", runner: "couchdb",
description: description: "Tier 2 CouchDB path through the Compose netem TCP shim using a tethering-vpn profile.",
"Tier 2 CouchDB path through the Compose netem TCP shim using a tethering-vpn profile.", dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
dataPath: trustBoundary: "CouchDB operator and constrained smartphone/VPN-like network shim",
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary:
"CouchDB operator and constrained smartphone/VPN-like network shim",
measurementScope: measurementScope:
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the tethering-vpn netem profile.", "Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the tethering-vpn netem profile.",
limitations: [ limitations: [
@@ -211,10 +183,8 @@ export function buildCases(): BenchmarkCase[] {
runner: "p2p", runner: "p2p",
description: description:
"Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.", "Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.",
dataPath: dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds",
"Device A -> Device B when WebRTC direct connectivity succeeds", trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata",
trustBoundary:
"Smartphone/VPN routing policy plus Nostr signalling metadata",
measurementScope: measurementScope:
"Structural placeholder for direct P2P measurements on a real smartphone tethering/VPN path.", "Structural placeholder for direct P2P measurements on a real smartphone tethering/VPN path.",
limitations: [ limitations: [
@@ -237,10 +207,8 @@ export function buildCases(): BenchmarkCase[] {
runner: "p2p", runner: "p2p",
description: description:
"Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.", "Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.",
dataPath: dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim", trustBoundary: "Nostr signalling metadata through constrained network shim; no TURN relay",
trustBoundary:
"Nostr signalling metadata through constrained network shim; no TURN relay",
measurementScope: measurementScope:
"One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the home-wifi netem profile; the selected WebRTC note-data path is unshaped.", "One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the home-wifi netem profile; the selected WebRTC note-data path is unshaped.",
limitations: [ limitations: [
@@ -265,8 +233,7 @@ export function buildCases(): BenchmarkCase[] {
runner: "p2p", runner: "p2p",
description: description:
"Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.", "Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.",
dataPath: dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary: trustBoundary:
"Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay", "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
measurementScope: measurementScope:
@@ -291,8 +258,7 @@ export function buildCases(): BenchmarkCase[] {
defineCase({ defineCase({
name: "p2p-user-turn", name: "p2p-user-turn",
runner: "p2p", runner: "p2p",
description: description: "Optional fallback path through a local user-controlled TURN server.",
"Optional fallback path through a local user-controlled TURN server.",
dataPath: "Device A -> user-controlled TURN -> Device B", dataPath: "Device A -> user-controlled TURN -> Device B",
trustBoundary: "User-controlled TURN server", trustBoundary: "User-controlled TURN server",
measurementScope: measurementScope:
@@ -319,11 +285,9 @@ async function runCase(
testCase: BenchmarkCase, testCase: BenchmarkCase,
outputDir: string, outputDir: string,
repeatIndex: number, repeatIndex: number,
repeatCount: number, repeatCount: number
): Promise<Record<string, unknown>> { ): Promise<Record<string, unknown>> {
const suffix = repeatCount > 1 const suffix = repeatCount > 1 ? `-r${String(repeatIndex).padStart(2, "0")}` : "";
? `-r${String(repeatIndex).padStart(2, "0")}`
: "";
const resultPath = `${outputDir}/${testCase.name}${suffix}.json`; const resultPath = `${outputDir}/${testCase.name}${suffix}.json`;
const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb"; const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb";
const env = { const env = {
@@ -334,12 +298,8 @@ async function runCase(
BENCH_REPEAT_COUNT: String(repeatCount), BENCH_REPEAT_COUNT: String(repeatCount),
}; };
const repeatLabel = repeatCount > 1 const repeatLabel = repeatCount > 1 ? ` (${repeatIndex}/${repeatCount})` : "";
? ` (${repeatIndex}/${repeatCount})` console.log(`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`);
: "";
console.log(
`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`,
);
const command = new Deno.Command("deno", { const command = new Deno.Command("deno", {
args: ["task", taskName], args: ["task", taskName],
cwd: import.meta.dirname, cwd: import.meta.dirname,
@@ -355,10 +315,7 @@ async function runCase(
throw new Error(`case failed: ${testCase.name} (exit ${status.code})`); throw new Error(`case failed: ${testCase.name} (exit ${status.code})`);
} }
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record< const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
string,
unknown
>;
return { return {
...testCase, ...testCase,
repeatIndex, repeatIndex,
@@ -369,10 +326,7 @@ async function runCase(
} }
function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] { function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
const requested = readEnvString( const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local");
"BENCH_CASES",
"couchdb-baseline,p2p-direct-local",
);
const names = requested const names = requested
.split(",") .split(",")
.map((v) => v.trim()) .map((v) => v.trim())
@@ -382,9 +336,7 @@ function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
const found = byName.get(name); const found = byName.get(name);
if (!found) { if (!found) {
throw new Error( throw new Error(
`Unknown BENCH_CASES entry '${name}'. Available: ${ `Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`
allCases.map((c) => c.name).join(", ")
}`,
); );
} }
return found; return found;
@@ -392,10 +344,7 @@ function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
} }
async function main(): Promise<void> { async function main(): Promise<void> {
const outRoot = readEnvString( const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`);
"BENCH_CASES_ROOT",
`${import.meta.dirname}/bench-results`,
);
const outputDir = `${outRoot}/cases-${timestamp()}`; const outputDir = `${outRoot}/cases-${timestamp()}`;
await Deno.mkdir(outputDir, { recursive: true }); await Deno.mkdir(outputDir, { recursive: true });
@@ -412,16 +361,14 @@ async function main(): Promise<void> {
availableCases: allCases, availableCases: allCases,
}, },
null, null,
2, 2
), )
); );
const results: Record<string, unknown>[] = []; const results: Record<string, unknown>[] = [];
for (const testCase of cases) { for (const testCase of cases) {
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) { for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
results.push( results.push(await runCase(testCase, outputDir, repeatIndex, repeatCount));
await runCase(testCase, outputDir, repeatIndex, repeatCount),
);
} }
} }
@@ -431,10 +378,7 @@ async function main(): Promise<void> {
repeatCount, repeatCount,
results, results,
}; };
await Deno.writeTextFile( await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
`${outputDir}/summary.json`,
JSON.stringify(summary, null, 2),
);
console.log(JSON.stringify(summary, null, 2)); console.log(JSON.stringify(summary, null, 2));
console.log(`[bench-cases] result directory: ${outputDir}`); console.log(`[bench-cases] result directory: ${outputDir}`);
} }
src/apps/cli/testdeno/bench-p2p.ts
+39 -105
View File
@@ -1,9 +1,5 @@
import { TempDir } from "./helpers/temp.ts"; import { TempDir } from "./helpers/temp.ts";
import { import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
applyP2pSettings,
applyP2pTestTweaks,
initSettingsFile,
} from "./helpers/settings.ts";
import { startCliInBackground } from "./helpers/backgroundCli.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts";
import { import {
discoverPeer, discoverPeer,
@@ -13,9 +9,7 @@ import {
stopLocalRelayIfStarted, stopLocalRelayIfStarted,
} from "./helpers/p2p.ts"; } from "./helpers/p2p.ts";
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
import { import { createDeterministicDataset } from "./helpers/dataset.ts";
createDeterministicDataset,
} from "./helpers/dataset.ts";
import { import {
type BenchmarkVerificationMode, type BenchmarkVerificationMode,
parseBenchmarkVerificationMode, parseBenchmarkVerificationMode,
@@ -107,10 +101,7 @@ function readEnvStringArray(name: string, fallback: string[]): string[] {
try { try {
const parsed = JSON.parse(raw); const parsed = JSON.parse(raw);
if ( if (Array.isArray(parsed) && parsed.every((item) => typeof item === "string")) {
Array.isArray(parsed) &&
parsed.every((item) => typeof item === "string")
) {
return parsed; return parsed;
} }
} catch { } catch {
@@ -147,48 +138,31 @@ function buildConfig(): BenchmarkConfig {
return { return {
caseName: readEnvString("BENCH_CASE", "p2p-direct-local"), caseName: readEnvString("BENCH_CASE", "p2p-direct-local"),
relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"), relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"),
appId: readEnvString( appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
"BENCH_APP_ID",
"self-hosted-livesync-cli-benchmark",
),
roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`), roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`),
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
turnServers: readEnvString("BENCH_TURN_SERVERS", ""), turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"), datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)), mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
mdMinSizeBytes: Math.floor( mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)),
readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024), mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)),
),
mdMaxSizeBytes: Math.floor(
readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024),
),
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)), binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)),
binSizeBytes: Math.floor( binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024),
),
peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20), peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20),
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240), syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"), simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "local-direct"), networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "local-direct"),
networkModel: readEnvString( networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-runner-webrtc"),
"BENCH_NETWORK_MODEL", candidatePathVerification: readEnvString("BENCH_P2P_CANDIDATE_PATH_VERIFICATION", "not-collected"),
"local-runner-webrtc",
),
candidatePathVerification: readEnvString(
"BENCH_P2P_CANDIDATE_PATH_VERIFICATION",
"not-collected",
),
measurementScope: readEnvString( measurementScope: readEnvString(
"BENCH_MEASUREMENT_SCOPE", "BENCH_MEASUREMENT_SCOPE",
"One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment; the earlier peer-list observation command is excluded.", "One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment; the earlier peer-list observation command is excluded."
), ),
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [ limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
"This benchmark result is scoped to the configured dataset, network model, and selected ICE path.", "This benchmark result is scoped to the configured dataset, network model, and selected ICE path.",
]), ]),
verificationMode: parseBenchmarkVerificationMode( verificationMode: parseBenchmarkVerificationMode(Deno.env.get("BENCH_VERIFY_MODE")),
Deno.env.get("BENCH_VERIFY_MODE"),
),
repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)), repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)), repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
}; };
@@ -202,9 +176,7 @@ function readOptionalResultPath(): string | undefined {
return raw; return raw;
} }
async function readLatestP2PConnectionStats( async function readLatestP2PConnectionStats(statsPath: string): Promise<P2PConnectionStats | undefined> {
statsPath: string,
): Promise<P2PConnectionStats | undefined> {
try { try {
const text = await Deno.readTextFile(statsPath); const text = await Deno.readTextFile(statsPath);
const lines = text const lines = text
@@ -252,7 +224,7 @@ async function main(): Promise<void> {
config.appId, config.appId,
config.relay, config.relay,
"~.*", "~.*",
config.turnServers, config.turnServers
), ),
applyP2pSettings( applyP2pSettings(
clientSettings, clientSettings,
@@ -261,21 +233,13 @@ async function main(): Promise<void> {
config.appId, config.appId,
config.relay, config.relay,
"~.*", "~.*",
config.turnServers, config.turnServers
), ),
]); ]);
await Promise.all([ await Promise.all([
applyP2pTestTweaks( applyP2pTestTweaks(hostSettings, "p2p-bench-host", config.passphrase),
hostSettings, applyP2pTestTweaks(clientSettings, "p2p-bench-client", config.passphrase),
"p2p-bench-host",
config.passphrase,
),
applyP2pTestTweaks(
clientSettings,
"p2p-bench-client",
config.passphrase,
),
]); ]);
const seedFiles = await createDeterministicDataset({ const seedFiles = await createDeterministicDataset({
@@ -293,25 +257,15 @@ async function main(): Promise<void> {
await runCliOrFail(hostVault, "--settings", hostSettings, "mirror"); await runCliOrFail(hostVault, "--settings", hostSettings, "mirror");
const mirrorElapsed = nowMs() - mirrorStart; const mirrorElapsed = nowMs() - mirrorStart;
const host = startCliInBackground( const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
hostVault,
"--settings",
hostSettings,
"p2p-host",
);
try { try {
const hostReadyStart = nowMs(); const hostReadyStart = nowMs();
await host.waitUntilContains("P2P host is running", 20000); await host.waitUntilContains("P2P host is running", 20000);
const hostReadyElapsed = nowMs() - hostReadyStart; const hostReadyElapsed = nowMs() - hostReadyStart;
const peerDiscoveryCommandStart = nowMs(); const peerDiscoveryCommandStart = nowMs();
const peer = await discoverPeer( const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds);
clientVault, const peerDiscoveryCommandElapsed = nowMs() - peerDiscoveryCommandStart;
clientSettings,
config.peersTimeoutSeconds,
);
const peerDiscoveryCommandElapsed = nowMs() -
peerDiscoveryCommandStart;
const syncStart = nowMs(); const syncStart = nowMs();
await runCliOrFail( await runCliOrFail(
@@ -320,7 +274,7 @@ async function main(): Promise<void> {
clientSettings, clientSettings,
"p2p-sync", "p2p-sync",
peer.id, peer.id,
String(config.syncTimeoutSeconds), String(config.syncTimeoutSeconds)
); );
const syncElapsed = nowMs() - syncStart; const syncElapsed = nowMs() - syncStart;
@@ -328,28 +282,24 @@ async function main(): Promise<void> {
seedFiles.entries, seedFiles.entries,
config.verificationMode, config.verificationMode,
async (entry) => { async (entry) => {
const pulledPath = workDir.join( const pulledPath = workDir.join(`pulled-${entry.relativePath.replaceAll("/", "_")}`);
`pulled-${entry.relativePath.replaceAll("/", "_")}`,
);
await runCliOrFail( await runCliOrFail(
clientVault, clientVault,
"--settings", "--settings",
clientSettings, clientSettings,
"pull", "pull",
entry.relativePath, entry.relativePath,
pulledPath, pulledPath
); );
await assertFilesEqual( await assertFilesEqual(
entry.absolutePath, entry.absolutePath,
pulledPath, pulledPath,
`file mismatch after P2P sync: ${entry.relativePath}`, `file mismatch after P2P sync: ${entry.relativePath}`
); );
}, }
); );
const p2pConnectionStats = await readLatestP2PConnectionStats( const p2pConnectionStats = await readLatestP2PConnectionStats(p2pStatsPath);
p2pStatsPath,
);
const result = { const result = {
caseName: config.caseName, caseName: config.caseName,
mode: "p2p-cli-benchmark", mode: "p2p-cli-benchmark",
@@ -363,17 +313,15 @@ async function main(): Promise<void> {
limitations: config.limitations, limitations: config.limitations,
repeatIndex: config.repeatIndex, repeatIndex: config.repeatIndex,
repeatCount: config.repeatCount, repeatCount: config.repeatCount,
p2pCandidatePathVerified: p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
p2pConnectionStats?.candidatePathCollected === true, p2pCandidatePathVerification: p2pConnectionStats?.candidatePathCollected
p2pCandidatePathVerification: ? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
p2pConnectionStats?.candidatePathCollected : config.candidatePathVerification,
? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
: config.candidatePathVerification,
p2pCandidatePathNote: p2pConnectionStats?.candidatePathCollected p2pCandidatePathNote: p2pConnectionStats?.candidatePathCollected
? "The selected ICE candidate pair was collected by the CLI benchmark. Interpret the path from the candidate types; do not infer TURN use from configuration alone." ? "The selected ICE candidate pair was collected by the CLI benchmark. Interpret the path from the candidate types; do not infer TURN use from configuration alone."
: config.turnServers.trim().length > 0 : config.turnServers.trim().length > 0
? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run." ? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run."
: "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.", : "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.",
p2pConnectionStats, p2pConnectionStats,
appId: config.appId, appId: config.appId,
roomId: config.roomId, roomId: config.roomId,
@@ -389,39 +337,25 @@ async function main(): Promise<void> {
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)), hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds, peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
peerDiscoveryCommandElapsedMs: Number( peerDiscoveryCommandElapsedMs: Number(peerDiscoveryCommandElapsed.toFixed(1)),
peerDiscoveryCommandElapsed.toFixed(1),
),
peerDiscoveryNote: peerDiscoveryNote:
"p2p-peers waits for the requested timeout before printing discovered peers, so this is command duration, not first-peer latency.", "p2p-peers waits for the requested timeout before printing discovered peers, so this is command duration, not first-peer latency.",
syncElapsedMs: Number(syncElapsed.toFixed(1)), syncElapsedMs: Number(syncElapsed.toFixed(1)),
throughputBytesPerSec: Number( throughputBytesPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2)),
(seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2), throughputMiBPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024).toFixed(4)),
),
throughputMiBPerSec: Number(
(seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024)
.toFixed(
4,
),
),
}; };
if (resultPath) { if (resultPath) {
await Deno.writeTextFile( await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2));
resultPath,
JSON.stringify(result, null, 2),
);
} }
console.log(JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
console.error( console.error(
`[Benchmark] mirrored ${seedFiles.totalFiles} files (${ `[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes(
formatBytes( seedFiles.totalBytes
seedFiles.totalBytes, )}) in ${formatMs(mirrorElapsed)}, ` +
)
}) in ${formatMs(mirrorElapsed)}, ` +
`synced in ${formatMs(syncElapsed)} ` + `synced in ${formatMs(syncElapsed)} ` +
`(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`, `(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
); );
} finally { } finally {
await host.stop(); await host.stop();
src/apps/cli/testdeno/helpers/benchmarkVerification.ts
+6 -15
View File
@@ -10,9 +10,7 @@ export type BenchmarkVerificationResult = {
}; };
function toHex(bytes: ArrayBuffer): string { function toHex(bytes: ArrayBuffer): string {
return [...new Uint8Array(bytes)] return [...new Uint8Array(bytes)].map((value) => value.toString(16).padStart(2, "0")).join("");
.map((value) => value.toString(16).padStart(2, "0"))
.join("");
} }
async function sha256(bytes: Uint8Array): Promise<string> { async function sha256(bytes: Uint8Array): Promise<string> {
@@ -23,7 +21,7 @@ async function sha256(bytes: Uint8Array): Promise<string> {
export function parseBenchmarkVerificationMode( export function parseBenchmarkVerificationMode(
raw: string | undefined, raw: string | undefined,
fallback: BenchmarkVerificationMode = "sample", fallback: BenchmarkVerificationMode = "sample"
): BenchmarkVerificationMode { ): BenchmarkVerificationMode {
const value = raw?.trim().toLowerCase(); const value = raw?.trim().toLowerCase();
if (!value) return fallback; if (!value) return fallback;
@@ -31,10 +29,7 @@ export function parseBenchmarkVerificationMode(
throw new Error(`BENCH_VERIFY_MODE must be 'all' or 'sample', got '${raw}'`); throw new Error(`BENCH_VERIFY_MODE must be 'all' or 'sample', got '${raw}'`);
} }
export function selectVerificationEntries( export function selectVerificationEntries(entries: DatasetEntry[], mode: BenchmarkVerificationMode): DatasetEntry[] {
entries: DatasetEntry[],
mode: BenchmarkVerificationMode,
): DatasetEntry[] {
if (mode === "all" || entries.length === 0) return [...entries]; if (mode === "all" || entries.length === 0) return [...entries];
const md = entries.find((entry) => entry.kind === "md"); const md = entries.find((entry) => entry.kind === "md");
@@ -48,15 +43,11 @@ export function selectVerificationEntries(
return [...selected.values()]; return [...selected.values()];
} }
export async function computeDatasetDigestSha256( export async function computeDatasetDigestSha256(entries: DatasetEntry[]): Promise<string> {
entries: DatasetEntry[],
): Promise<string> {
const manifest: string[] = []; const manifest: string[] = [];
for (const entry of entries) { for (const entry of entries) {
const contentDigest = await sha256(await Deno.readFile(entry.absolutePath)); const contentDigest = await sha256(await Deno.readFile(entry.absolutePath));
manifest.push( manifest.push(`${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`);
`${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`,
);
} }
return await sha256(new TextEncoder().encode(manifest.join("\n"))); return await sha256(new TextEncoder().encode(manifest.join("\n")));
} }
@@ -64,7 +55,7 @@ export async function computeDatasetDigestSha256(
export async function verifyBenchmarkDataset( export async function verifyBenchmarkDataset(
entries: DatasetEntry[], entries: DatasetEntry[],
mode: BenchmarkVerificationMode, mode: BenchmarkVerificationMode,
verifyEntry: (entry: DatasetEntry) => Promise<void>, verifyEntry: (entry: DatasetEntry) => Promise<void>
): Promise<BenchmarkVerificationResult> { ): Promise<BenchmarkVerificationResult> {
const selected = selectVerificationEntries(entries, mode); const selected = selectVerificationEntries(entries, mode);
for (const entry of selected) { for (const entry of selected) {
src/apps/cli/testdeno/test-benchmark-contract.ts
+26 -84
View File
@@ -1,10 +1,7 @@
import { assert, assertEquals, assertStringIncludes } from "@std/assert"; import { assert, assertEquals, assertStringIncludes } from "@std/assert";
import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts"; import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts";
import { startCouchdbProxy } from "./bench-couchdb.ts"; import { startCouchdbProxy } from "./bench-couchdb.ts";
import { import { parseBenchmarkVerificationMode, selectVerificationEntries } from "./helpers/benchmarkVerification.ts";
parseBenchmarkVerificationMode,
selectVerificationEntries,
} from "./helpers/benchmarkVerification.ts";
import type { DatasetEntry } from "./helpers/dataset.ts"; import type { DatasetEntry } from "./helpers/dataset.ts";
function getFreePort(): number { function getFreePort(): number {
@@ -24,20 +21,10 @@ function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase {
function parsedLimitations(testCase: BenchmarkCase): string[] { function parsedLimitations(testCase: BenchmarkCase): string[] {
const raw = testCase.env.BENCH_LIMITATIONS_JSON; const raw = testCase.env.BENCH_LIMITATIONS_JSON;
assert( assert(raw, `${testCase.name} must pass BENCH_LIMITATIONS_JSON to benchmark result output`);
raw,
`${testCase.name} must pass BENCH_LIMITATIONS_JSON to benchmark result output`,
);
const parsed = JSON.parse(raw); const parsed = JSON.parse(raw);
assert( assert(Array.isArray(parsed), `${testCase.name} limitations must be an array`);
Array.isArray(parsed), assert(parsed.every((item) => typeof item === "string" && item.trim().length > 0));
`${testCase.name} limitations must be an array`,
);
assert(
parsed.every((item) =>
typeof item === "string" && item.trim().length > 0
),
);
return parsed; return parsed;
} }
@@ -46,36 +33,14 @@ Deno.test("benchmark cases record scope and limitations for paper use", () => {
assert(cases.length > 0); assert(cases.length > 0);
for (const testCase of cases) { for (const testCase of cases) {
assert( assert(testCase.description.trim().length > 0, `${testCase.name} must describe the case`);
testCase.description.trim().length > 0, assert(testCase.dataPath.trim().length > 0, `${testCase.name} must describe the data path`);
`${testCase.name} must describe the case`, assert(testCase.trustBoundary.trim().length > 0, `${testCase.name} must describe the trust boundary`);
); assert(testCase.measurementScope.trim().length > 0, `${testCase.name} must describe the measurement scope`);
assert( assert(testCase.limitations.length > 0, `${testCase.name} must list limitations`);
testCase.dataPath.trim().length > 0, assertEquals(testCase.env.BENCH_MEASUREMENT_SCOPE, testCase.measurementScope);
`${testCase.name} must describe the data path`,
);
assert(
testCase.trustBoundary.trim().length > 0,
`${testCase.name} must describe the trust boundary`,
);
assert(
testCase.measurementScope.trim().length > 0,
`${testCase.name} must describe the measurement scope`,
);
assert(
testCase.limitations.length > 0,
`${testCase.name} must list limitations`,
);
assertEquals(
testCase.env.BENCH_MEASUREMENT_SCOPE,
testCase.measurementScope,
);
assertEquals(parsedLimitations(testCase), testCase.limitations); assertEquals(parsedLimitations(testCase), testCase.limitations);
assertEquals( assertEquals(testCase.env.BENCH_VERIFY_MODE, "all", `${testCase.name} must verify the complete dataset`);
testCase.env.BENCH_VERIFY_MODE,
"all",
`${testCase.name} must verify the complete dataset`,
);
} }
}); });
@@ -89,7 +54,7 @@ Deno.test("CouchDB latency proxy applies half the requested RTT in each directio
port: backendPort, port: backendPort,
onListen() {}, onListen() {},
}, },
() => new Response("ok"), () => new Response("ok")
); );
const proxy = startCouchdbProxy({ const proxy = startCouchdbProxy({
backendUri: `http://127.0.0.1:${backendPort}`, backendUri: `http://127.0.0.1:${backendPort}`,
@@ -143,34 +108,22 @@ Deno.test("benchmark verification mode selects either all files or a labelled sa
Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => { Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => {
const cases = buildCases(); const cases = buildCases();
for ( for (const name of ["p2p-signalling-netem-home-wifi", "p2p-signalling-netem-tethering-vpn"]) {
const name of [
"p2p-signalling-netem-home-wifi",
"p2p-signalling-netem-tethering-vpn",
]
) {
const testCase = getCase(cases, name); const testCase = getCase(cases, name);
assertEquals(testCase.runner, "p2p"); assertEquals(testCase.runner, "p2p");
assertEquals(testCase.env.BENCH_TURN_SERVERS, ""); assertEquals(testCase.env.BENCH_TURN_SERVERS, "");
assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2"); assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2");
assertEquals( assertEquals(testCase.env.BENCH_NETWORK_MODEL, "compose-netem-signalling-shim");
testCase.env.BENCH_NETWORK_MODEL,
"compose-netem-signalling-shim",
);
assertStringIncludes(testCase.dataPath, "WebRTC DataChannel"); assertStringIncludes(testCase.dataPath, "WebRTC DataChannel");
assertStringIncludes(testCase.dataPath, "Nostr signalling"); assertStringIncludes(testCase.dataPath, "Nostr signalling");
assertStringIncludes(testCase.measurementScope, "fresh CLI p2p-sync"); assertStringIncludes(testCase.measurementScope, "fresh CLI p2p-sync");
assert( assert(
testCase.limitations.some((limitation) => testCase.limitations.some((limitation) => limitation.includes("connection establishment")),
limitation.includes("connection establishment") `${name} must state that connection establishment is timed`
),
`${name} must state that connection establishment is timed`,
); );
assert( assert(
testCase.limitations.some((limitation) => testCase.limitations.some((limitation) => limitation.includes("does not shape the selected WebRTC")),
limitation.includes("does not shape the selected WebRTC") `${name} must avoid claiming that the P2P note-data path was shaped`
),
`${name} must avoid claiming that the P2P note-data path was shaped`,
); );
} }
}); });
@@ -182,42 +135,31 @@ Deno.test("placeholder and TURN cases are clearly non-evidence for broad P2P per
assertEquals(smartphone.env.BENCH_SIMULATION_TIER, "unmeasured"); assertEquals(smartphone.env.BENCH_SIMULATION_TIER, "unmeasured");
assertEquals(smartphone.env.BENCH_NETWORK_MODEL, "local-runner-no-netem"); assertEquals(smartphone.env.BENCH_NETWORK_MODEL, "local-runner-no-netem");
assert( assert(
smartphone.limitations.some((limitation) => smartphone.limitations.some((limitation) => limitation.includes("must not be reported as smartphone")),
limitation.includes("must not be reported as smartphone") "smartphone/VPN placeholder must not be usable as field evidence by accident"
),
"smartphone/VPN placeholder must not be usable as field evidence by accident",
); );
const turn = getCase(cases, "p2p-user-turn"); const turn = getCase(cases, "p2p-user-turn");
assertStringIncludes(turn.env.BENCH_TURN_SERVERS, "turn:"); assertStringIncludes(turn.env.BENCH_TURN_SERVERS, "turn:");
assert( assert(
turn.limitations.some((limitation) => turn.limitations.some((limitation) =>
limitation.includes( limitation.includes("does not prove that the selected ICE path was relayed")
"does not prove that the selected ICE path was relayed",
)
), ),
"TURN case must require selected ICE candidate interpretation", "TURN case must require selected ICE candidate interpretation"
); );
}); });
Deno.test("CouchDB netem cases are marked as remote-store baselines", () => { Deno.test("CouchDB netem cases are marked as remote-store baselines", () => {
const cases = buildCases(); const cases = buildCases();
for ( for (const name of ["couchdb-netem-home-wifi", "couchdb-netem-tethering-vpn"]) {
const name of ["couchdb-netem-home-wifi", "couchdb-netem-tethering-vpn"]
) {
const testCase = getCase(cases, name); const testCase = getCase(cases, name);
assertEquals(testCase.runner, "couchdb"); assertEquals(testCase.runner, "couchdb");
assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2"); assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2");
assertEquals( assertEquals(testCase.env.BENCH_NETWORK_MODEL, "compose-netem-tcp-shim");
testCase.env.BENCH_NETWORK_MODEL,
"compose-netem-tcp-shim",
);
assertStringIncludes(testCase.measurementScope, "CouchDB"); assertStringIncludes(testCase.measurementScope, "CouchDB");
assert( assert(
testCase.limitations.some((limitation) => testCase.limitations.some((limitation) => limitation.includes("not the WebRTC P2P data path")),
limitation.includes("not the WebRTC P2P data path") `${name} must remain scoped to the CouchDB remote-store path`
),
`${name} must remain scoped to the CouchDB remote-store path`,
); );
} }
}); });