public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-08-30 07:17:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add scoped CLI benchmark metadata

Browse Source
This commit is contained in:
vorotamoroz
2026-07-12 20:02:29 +09:00
parent 7d26f0ae35
commit af72cac4e3
6 changed files with 593 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/cli/testdeno/bench-network-cases.ts
+184 -53
View File
@@ -1,9 +1,11 @@
type BenchmarkCase = {
export type BenchmarkCase = {
name: string;
runner: "p2p" | "couchdb";
description: string;
dataPath: string;
trustBoundary: string;
measurementScope: string;
limitations: string[];
env: Record<string, string>;
};
@@ -25,16 +27,26 @@ function timestamp(): string {
const d = new Date();
const pad = (n: number) => String(n).padStart(2, "0");
return (
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${
pad(d.getUTCDate())
}-` +
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${
pad(d.getUTCSeconds())
}`
);
}
function buildBaseEnv(): Record<string, string> {
return {
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
BENCH_MD_MIN_SIZE_BYTES: readEnvString(
"BENCH_MD_MIN_SIZE_BYTES",
"512",
),
BENCH_MD_MAX_SIZE_BYTES: readEnvString(
"BENCH_MD_MAX_SIZE_BYTES",
"2048",
),
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
@@ -44,33 +56,74 @@ function buildBaseEnv(): Record<string, string> {
};
}
function buildCases(): BenchmarkCase[] {
function withScopeEnv(
env: Record<string, string>,
options: Pick<BenchmarkCase, "measurementScope" | "limitations">,
): Record<string, string> {
return {
...env,
BENCH_MEASUREMENT_SCOPE: options.measurementScope,
BENCH_LIMITATIONS_JSON: JSON.stringify(options.limitations),
};
}
function defineCase(testCase: BenchmarkCase): BenchmarkCase {
return {
...testCase,
env: withScopeEnv(testCase.env, testCase),
};
}
export function buildCases(): BenchmarkCase[] {
const base = buildBaseEnv();
const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20");
const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120");
const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478");
const shimCouchdbUri = readEnvString("BENCH_SHIM_COUCHDB_URI", "http://couchdb-shim:5984");
const signallingShimRelay = readEnvString("BENCH_SIGNAL_SHIM_RELAY", "ws://p2p-signalling-shim:7777/");
const localTurnServers = readEnvString(
"BENCH_LOCAL_TURN_SERVERS",
"turn:127.0.0.1:3478",
);
const shimCouchdbUri = readEnvString(
"BENCH_SHIM_COUCHDB_URI",
"http://couchdb-shim:5984",
);
const signallingShimRelay = readEnvString(
"BENCH_SIGNAL_SHIM_RELAY",
"ws://p2p-signalling-shim:7777/",
);
return [
{
defineCase({
name: "couchdb-baseline",
runner: "couchdb",
description: "Standard self-hosted CouchDB path through a local latency proxy.",
description:
"Standard self-hosted CouchDB path through a local latency proxy.",
dataPath: "Device A -> CouchDB -> Device B",
trustBoundary: "CouchDB operator and network path",
measurementScope:
"Two one-shot synchronisation phases through a CouchDB-compatible remote-store path with a local HTTP latency proxy.",
limitations: [
"This is not a full netem model of packet loss, jitter, MTU, bandwidth limits, or VPN encapsulation.",
"This result should be compared with P2P only as a remote-store baseline under the same deterministic dataset.",
],
env: {
...base,
BENCH_CASE: "couchdb-baseline",
BENCH_COUCHDB_RTT_MS: couchdbRtt,
},
},
{
}),
defineCase({
name: "p2p-direct-local",
runner: "p2p",
description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
description:
"Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
dataPath: "Device A -> Device B",
trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
measurementScope:
"One CLI P2P synchronisation phase over a local WebRTC DataChannel after Nostr signalling, with TURN disabled.",
limitations: [
"This does not measure first-peer discovery latency, public relay operation, mobile carrier behaviour, or TURN-relayed throughput.",
"This small-dataset run should not be treated as a WAN, VPN, or large binary initial synchronisation measurement.",
],
env: {
...base,
BENCH_CASE: "p2p-direct-local",
@@ -78,29 +131,44 @@ function buildCases(): BenchmarkCase[] {
BENCH_SIMULATION_TIER: "1",
BENCH_NETWORK_PROFILE: "local-direct",
BENCH_NETWORK_MODEL: "local-runner-webrtc",
BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "turn-disabled-but-selected-ice-pair-not-collected",
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"turn-disabled-but-selected-ice-pair-not-collected",
},
},
{
}),
defineCase({
name: "couchdb-tethering-vpn-proxy",
runner: "couchdb",
description:
"Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.",
dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
dataPath:
"Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
trustBoundary: "VPN/network path and CouchDB operator",
measurementScope:
"Two one-shot CouchDB synchronisation phases with additional requested RTT through the local HTTP proxy.",
limitations: [
"This approximates request latency only and does not model loss, jitter, MTU, bandwidth limits, carrier NAT, or VPN encapsulation.",
"Use the Tier 2 netem shim cases for a stronger constrained-network fixture.",
],
env: {
...base,
BENCH_CASE: "couchdb-tethering-vpn-proxy",
BENCH_COUCHDB_RTT_MS: tetheringVpnRtt,
},
},
{
}),
defineCase({
name: "couchdb-netem-home-wifi",
runner: "couchdb",
description:
"Tier 2 CouchDB path through the Compose netem TCP shim using the home-wifi profile.",
dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
dataPath:
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary: "CouchDB operator and constrained network shim",
measurementScope:
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the home-wifi netem profile.",
limitations: [
"This shapes the CouchDB TCP path, not the WebRTC P2P data path.",
"The fixture remains a reproducible network emulation, not a field measurement on a real home network.",
],
env: {
...base,
BENCH_CASE: "couchdb-netem-home-wifi",
@@ -110,14 +178,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_NETWORK_PROFILE: "home-wifi",
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
},
},
{
}),
defineCase({
name: "couchdb-netem-tethering-vpn",
runner: "couchdb",
description:
"Tier 2 CouchDB path through the Compose netem TCP shim using a tethering-vpn profile.",
dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary: "CouchDB operator and constrained smartphone/VPN-like network shim",
dataPath:
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
trustBoundary:
"CouchDB operator and constrained smartphone/VPN-like network shim",
measurementScope:
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the tethering-vpn netem profile.",
limitations: [
"This shapes the CouchDB TCP path, not the WebRTC P2P data path.",
"The profile approximates smartphone/VPN constraints but is not a field measurement on a real tethered VPN connection.",
],
env: {
...base,
BENCH_CASE: "couchdb-netem-tethering-vpn",
@@ -127,14 +203,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_NETWORK_PROFILE: "tethering-vpn",
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
},
},
{
}),
defineCase({
name: "p2p-smartphone-vpn-direct",
runner: "p2p",
description:
"Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.",
dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds",
trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata",
dataPath:
"Device A -> Device B when WebRTC direct connectivity succeeds",
trustBoundary:
"Smartphone/VPN routing policy plus Nostr signalling metadata",
measurementScope:
"Structural placeholder for direct P2P measurements on a real smartphone tethering/VPN path.",
limitations: [
"In the local runner this is unshaped and must not be reported as smartphone, VPN, WAN, or Tier 2 evidence.",
"Use only when the command is executed on the intended real network path and the selected ICE candidate pair is recorded.",
],
env: {
...base,
BENCH_CASE: "p2p-smartphone-vpn-direct",
@@ -145,14 +229,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"structural-placeholder-only; selected ICE pair may be collected, but the path is not shaped",
},
},
{
}),
defineCase({
name: "p2p-signalling-netem-home-wifi",
runner: "p2p",
description:
"Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.",
dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary: "Nostr signalling metadata through constrained network shim; no TURN relay",
dataPath:
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary:
"Nostr signalling metadata through constrained network shim; no TURN relay",
measurementScope:
"Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the home-wifi netem profile.",
limitations: [
"This does not shape the selected WebRTC DataChannel note-data path.",
"This supports only the claim that constrained signalling access does not place note data on the relay path when a non-relayed ICE path is selected.",
],
env: {
...base,
BENCH_CASE: "p2p-signalling-netem-home-wifi",
@@ -164,14 +256,22 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"selected ICE pair collected; only Nostr signalling path is shaped",
},
},
{
}),
defineCase({
name: "p2p-signalling-netem-tethering-vpn",
runner: "p2p",
description:
"Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.",
dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary: "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
dataPath:
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
trustBoundary:
"Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
measurementScope:
"Tier 2 P2P synchronisation where only the Nostr signalling path is shaped by the tethering-vpn netem profile.",
limitations: [
"This does not shape the selected WebRTC DataChannel note-data path.",
"The profile approximates constrained relay access and is not a field measurement on a real tethered VPN connection.",
],
env: {
...base,
BENCH_CASE: "p2p-signalling-netem-tethering-vpn",
@@ -183,13 +283,20 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"selected ICE pair collected; only Nostr signalling path is shaped",
},
},
{
}),
defineCase({
name: "p2p-user-turn",
runner: "p2p",
description: "Optional fallback path through a local user-controlled TURN server.",
description:
"Optional fallback path through a local user-controlled TURN server.",
dataPath: "Device A -> user-controlled TURN -> Device B",
trustBoundary: "User-controlled TURN server",
measurementScope:
"Optional local TURN fallback wiring check with a user-controlled TURN server configured.",
limitations: [
"TURN configuration does not prove that the selected ICE path was relayed; interpret the recorded candidate pair.",
"This is not evidence for public TURN relay privacy, throughput, or availability.",
],
env: {
...base,
BENCH_CASE: "p2p-user-turn",
@@ -200,7 +307,7 @@ function buildCases(): BenchmarkCase[] {
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
"turn-configured; selected ICE pair may still be direct or relayed, so interpret the recorded candidate types",
},
},
}),
];
}
@@ -208,9 +315,11 @@ async function runCase(
testCase: BenchmarkCase,
outputDir: string,
repeatIndex: number,
repeatCount: number
repeatCount: number,
): Promise<Record<string, unknown>> {
const suffix = repeatCount > 1 ? `-r${String(repeatIndex).padStart(2, "0")}` : "";
const suffix = repeatCount > 1
? `-r${String(repeatIndex).padStart(2, "0")}`
: "";
const resultPath = `${outputDir}/${testCase.name}${suffix}.json`;
const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb";
const env = {
@@ -221,8 +330,12 @@ async function runCase(
BENCH_REPEAT_COUNT: String(repeatCount),
};
const repeatLabel = repeatCount > 1 ? ` (${repeatIndex}/${repeatCount})` : "";
console.log(`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`);
const repeatLabel = repeatCount > 1
? ` (${repeatIndex}/${repeatCount})`
: "";
console.log(
`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`,
);
const command = new Deno.Command("deno", {
args: ["task", taskName],
cwd: import.meta.dirname,
@@ -238,7 +351,10 @@ async function runCase(
throw new Error(`case failed: ${testCase.name} (exit ${status.code})`);
}
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<
string,
unknown
>;
return {
...testCase,
repeatIndex,
@@ -249,7 +365,10 @@ async function runCase(
}
function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local");
const requested = readEnvString(
"BENCH_CASES",
"couchdb-baseline,p2p-direct-local",
);
const names = requested
.split(",")
.map((v) => v.trim())
@@ -258,14 +377,21 @@ function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
return names.map((name) => {
const found = byName.get(name);
if (!found) {
throw new Error(`Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`);
throw new Error(
`Unknown BENCH_CASES entry '${name}'. Available: ${
allCases.map((c) => c.name).join(", ")
}`,
);
}
return found;
});
}
async function main(): Promise<void> {
const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`);
const outRoot = readEnvString(
"BENCH_CASES_ROOT",
`${import.meta.dirname}/bench-results`,
);
const outputDir = `${outRoot}/cases-${timestamp()}`;
await Deno.mkdir(outputDir, { recursive: true });
@@ -282,14 +408,16 @@ async function main(): Promise<void> {
availableCases: allCases,
},
null,
2
)
2,
),
);
const results: Record<string, unknown>[] = [];
for (const testCase of cases) {
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
results.push(await runCase(testCase, outputDir, repeatIndex, repeatCount));
results.push(
await runCase(testCase, outputDir, repeatIndex, repeatCount),
);
}
}
@@ -299,7 +427,10 @@ async function main(): Promise<void> {
repeatCount,
results,
};
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
await Deno.writeTextFile(
`${outputDir}/summary.json`,
JSON.stringify(summary, null, 2),
);
console.log(JSON.stringify(summary, null, 2));
console.log(`[bench-cases] result directory: ${outputDir}`);
}