public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-07-17 10:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Compose-based CLI network benchmarks

Browse Source
This commit is contained in:
vorotamoroz
2026-07-08 06:30:12 +00:00
parent 78eb6ec3bc
commit ef655a2976
14 changed files with 704 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/cli/testdeno/bench-couchdb.ts
+21 -3
View File
@@ -1,10 +1,11 @@
import { TempDir } from "./helpers/temp.ts";
import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
import { startCouchdb, stopCouchdb } from "./helpers/docker.ts";
import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts";
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
type BenchmarkConfig = {
caseName: string;
couchdbBackendUri: string;
couchdbProxyUri: string;
couchdbUser: string;
@@ -21,6 +22,7 @@ type BenchmarkConfig = {
requestedRttMs: number;
passphrase: string;
encrypt: boolean;
managedCouchdb: boolean;
};
function readEnvString(name: string, fallback: string): string {
@@ -70,6 +72,7 @@ function formatBytes(value: number): string {
function buildConfig(): BenchmarkConfig {
return {
caseName: readEnvString("BENCH_CASE", "couchdb-baseline"),
couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"),
couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"),
couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")),
@@ -86,6 +89,7 @@ function buildConfig(): BenchmarkConfig {
requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)),
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
encrypt: readEnvBool("BENCH_ENCRYPT", true),
managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true),
};
}
@@ -200,7 +204,17 @@ async function main(): Promise<void> {
await initSettingsFile(settingsA);
await initSettingsFile(settingsB);
await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
if (config.managedCouchdb) {
await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
} else {
console.log(`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`);
await createCouchdbDatabase(
config.couchdbBackendUri,
config.couchdbUser,
config.couchdbPassword,
config.couchdbDbname
);
}
const proxy = startCouchdbProxy({
backendUri: config.couchdbBackendUri,
@@ -265,10 +279,12 @@ async function main(): Promise<void> {
}
const result = {
caseName: config.caseName,
mode: "couchdb-cli-benchmark",
couchdbBackendUri: config.couchdbBackendUri,
couchdbProxyUri: config.couchdbProxyUri,
couchdbDbname: config.couchdbDbname,
managedCouchdb: config.managedCouchdb,
rttRequestedMs: config.requestedRttMs,
proxyApplied: proxy.applied,
proxyNote: proxy.note,
@@ -300,7 +316,9 @@ async function main(): Promise<void> {
);
} finally {
await proxy.stop();
await stopCouchdb().catch(() => {});
if (config.managedCouchdb) {
await stopCouchdb().catch(() => {});
}
}
}
src/apps/cli/testdeno/bench-latency-sweep.ts
+133
View File
@@ -0,0 +1,133 @@
type SweepResult = {
name: string;
runner: "p2p" | "couchdb";
rttMs?: number;
result: Record<string, unknown>;
};
function readEnvString(name: string, fallback: string): string {
const value = Deno.env.get(name)?.trim();
return value && value.length > 0 ? value : fallback;
}
function timestamp(): string {
const d = new Date();
const pad = (n: number) => String(n).padStart(2, "0");
return (
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
);
}
function parseRttList(raw: string): number[] {
const values = raw
.split(",")
.map((value) => Number(value.trim()))
.filter((value) => Number.isFinite(value) && value > 0)
.map((value) => Math.floor(value));
if (values.length === 0) {
throw new Error(`BENCH_SWEEP_RTT_MS must contain at least one positive number, got '${raw}'`);
}
return values;
}
function buildBaseEnv(): Record<string, string> {
return {
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
};
}
async function runBenchmark(options: {
taskName: "bench:p2p" | "bench:couchdb";
name: string;
outputDir: string;
env: Record<string, string>;
}): Promise<Record<string, unknown>> {
const resultPath = `${options.outputDir}/${options.name}.json`;
const env = {
...Deno.env.toObject(),
...options.env,
BENCH_RESULT_JSON: resultPath,
};
console.log(`[latency-sweep] running ${options.name}`);
const child = new Deno.Command("deno", {
args: ["task", options.taskName],
cwd: import.meta.dirname,
env,
stdin: "null",
stdout: "inherit",
stderr: "inherit",
}).spawn();
const status = await child.status;
if (status.code !== 0) {
throw new Error(`benchmark failed: ${options.name} (exit ${status.code})`);
}
return JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
}
async function main(): Promise<void> {
const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`);
const outputDir = `${outRoot}/latency-sweep-${timestamp()}`;
const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300"));
const base = buildBaseEnv();
await Deno.mkdir(outputDir, { recursive: true });
const results: SweepResult[] = [];
if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") {
const p2pResult = await runBenchmark({
taskName: "bench:p2p",
name: "p2p-direct-local",
outputDir,
env: {
...base,
BENCH_CASE: "p2p-direct-local",
BENCH_TURN_SERVERS: "",
},
});
results.push({ name: "p2p-direct-local", runner: "p2p", result: p2pResult });
}
for (const rtt of rtts) {
const name = `couchdb-rtt-${rtt}ms`;
const couchdbResult = await runBenchmark({
taskName: "bench:couchdb",
name,
outputDir,
env: {
...base,
BENCH_CASE: name,
BENCH_COUCHDB_RTT_MS: String(rtt),
},
});
results.push({ name, runner: "couchdb", rttMs: rtt, result: couchdbResult });
}
const summary = {
generatedAt: new Date().toISOString(),
outputDir,
note:
"This sweep models additional remote CouchDB request latency through the existing HTTP proxy. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
rtts,
results,
};
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
console.log(JSON.stringify(summary, null, 2));
console.log(`[latency-sweep] result directory: ${outputDir}`);
}
if (import.meta.main) {
main().catch((error) => {
console.error("[Fatal Error]", error);
Deno.exit(1);
});
}
src/apps/cli/testdeno/bench-network-cases.ts
+198
View File
@@ -0,0 +1,198 @@
type BenchmarkCase = {
name: string;
runner: "p2p" | "couchdb";
description: string;
dataPath: string;
trustBoundary: string;
env: Record<string, string>;
};
function readEnvString(name: string, fallback: string): string {
const value = Deno.env.get(name)?.trim();
return value && value.length > 0 ? value : fallback;
}
function timestamp(): string {
const d = new Date();
const pad = (n: number) => String(n).padStart(2, "0");
return (
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
);
}
function buildBaseEnv(): Record<string, string> {
return {
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
};
}
function buildCases(): BenchmarkCase[] {
const base = buildBaseEnv();
const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20");
const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120");
const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478");
return [
{
name: "couchdb-baseline",
runner: "couchdb",
description: "Standard self-hosted CouchDB path through a local latency proxy.",
dataPath: "Device A -> CouchDB -> Device B",
trustBoundary: "CouchDB operator and network path",
env: {
...base,
BENCH_CASE: "couchdb-baseline",
BENCH_COUCHDB_RTT_MS: couchdbRtt,
},
},
{
name: "p2p-direct-local",
runner: "p2p",
description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
dataPath: "Device A -> Device B",
trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
env: {
...base,
BENCH_CASE: "p2p-direct-local",
BENCH_TURN_SERVERS: "",
},
},
{
name: "couchdb-tethering-vpn-proxy",
runner: "couchdb",
description:
"Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.",
dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
trustBoundary: "VPN/network path and CouchDB operator",
env: {
...base,
BENCH_CASE: "couchdb-tethering-vpn-proxy",
BENCH_COUCHDB_RTT_MS: tetheringVpnRtt,
},
},
{
name: "p2p-smartphone-vpn-direct",
runner: "p2p",
description:
"Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.",
dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds",
trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata",
env: {
...base,
BENCH_CASE: "p2p-smartphone-vpn-direct",
BENCH_TURN_SERVERS: "",
},
},
{
name: "p2p-user-turn",
runner: "p2p",
description: "Optional fallback path through a local user-controlled TURN server.",
dataPath: "Device A -> user-controlled TURN -> Device B",
trustBoundary: "User-controlled TURN server",
env: {
...base,
BENCH_CASE: "p2p-user-turn",
BENCH_TURN_SERVERS: localTurnServers,
},
},
];
}
async function runCase(testCase: BenchmarkCase, outputDir: string): Promise<Record<string, unknown>> {
const resultPath = `${outputDir}/${testCase.name}.json`;
const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb";
const env = {
...Deno.env.toObject(),
...testCase.env,
BENCH_RESULT_JSON: resultPath,
};
console.log(`[bench-cases] running ${testCase.name}: ${testCase.description}`);
const command = new Deno.Command("deno", {
args: ["task", taskName],
cwd: import.meta.dirname,
env,
stdin: "null",
stdout: "inherit",
stderr: "inherit",
});
const child = command.spawn();
const status = await child.status;
if (status.code !== 0) {
throw new Error(`case failed: ${testCase.name} (exit ${status.code})`);
}
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
return {
...testCase,
result,
};
}
function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local");
const names = requested
.split(",")
.map((v) => v.trim())
.filter((v) => v.length > 0);
const byName = new Map(allCases.map((c) => [c.name, c]));
return names.map((name) => {
const found = byName.get(name);
if (!found) {
throw new Error(`Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`);
}
return found;
});
}
async function main(): Promise<void> {
const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`);
const outputDir = `${outRoot}/cases-${timestamp()}`;
await Deno.mkdir(outputDir, { recursive: true });
const allCases = buildCases();
const cases = selectCases(allCases);
await Deno.writeTextFile(
`${outputDir}/case-manifest.json`,
JSON.stringify(
{
generatedAt: new Date().toISOString(),
selectedCases: cases,
availableCases: allCases,
},
null,
2
)
);
const results: Record<string, unknown>[] = [];
for (const testCase of cases) {
results.push(await runCase(testCase, outputDir));
}
const summary = {
generatedAt: new Date().toISOString(),
outputDir,
results,
};
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
console.log(JSON.stringify(summary, null, 2));
console.log(`[bench-cases] result directory: ${outputDir}`);
}
if (import.meta.main) {
main().catch((error) => {
console.error("[Fatal Error]", error);
Deno.exit(1);
});
}
src/apps/cli/testdeno/bench-p2p.ts
+34 -3
View File
@@ -1,15 +1,23 @@
import { TempDir } from "./helpers/temp.ts";
import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
import { startCliInBackground } from "./helpers/backgroundCli.ts";
import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts";
import {
discoverPeer,
maybeStartCoturn,
maybeStartLocalRelay,
stopCoturnIfStarted,
stopLocalRelayIfStarted,
} from "./helpers/p2p.ts";
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
type BenchmarkConfig = {
caseName: string;
relay: string;
appId: string;
roomId: string;
passphrase: string;
turnServers: string;
datasetDirName: string;
datasetSeed: string;
mdFileCount: number;
@@ -61,10 +69,12 @@ function formatBytes(value: number): string {
function buildConfig(): BenchmarkConfig {
return {
caseName: readEnvString("BENCH_CASE", "p2p-direct-local"),
relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"),
appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`),
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
@@ -107,6 +117,7 @@ async function main(): Promise<void> {
const resultPath = readOptionalResultPath();
const relayStarted = await maybeStartLocalRelay(config.relay);
const coturnStarted = await maybeStartCoturn(config.turnServers);
await using workDir = await TempDir.create("livesync-cli-p2p-bench");
const hostVault = workDir.join("vault-host");
@@ -122,8 +133,24 @@ async function main(): Promise<void> {
]);
await Promise.all([
applyP2pSettings(hostSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
applyP2pSettings(clientSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
applyP2pSettings(
hostSettings,
config.roomId,
config.passphrase,
config.appId,
config.relay,
"~.*",
config.turnServers
),
applyP2pSettings(
clientSettings,
config.roomId,
config.passphrase,
config.appId,
config.relay,
"~.*",
config.turnServers
),
]);
await Promise.all([
@@ -179,8 +206,11 @@ async function main(): Promise<void> {
}
const result = {
caseName: config.caseName,
mode: "p2p-cli-benchmark",
relay: config.relay,
turnServers: config.turnServers,
turnEnabled: config.turnServers.trim().length > 0,
appId: config.appId,
roomId: config.roomId,
datasetSeed: config.datasetSeed,
@@ -211,6 +241,7 @@ async function main(): Promise<void> {
);
} finally {
await host.stop();
await stopCoturnIfStarted(coturnStarted);
await stopLocalRelayIfStarted(relayStarted);
}
}
src/apps/cli/testdeno/deno.json
+2
View File
@@ -17,6 +17,8 @@
"test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts",
"bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts",
"bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts",
"bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts",
"bench:latency-sweep": "deno run --env-file=.test.env -A --no-check bench-latency-sweep.ts",
"bench:item1": "bash ./bench-run-item1.sh",
"bench:item1:full": "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh",
"test:e2e-couchdb": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-couchdb.ts",
src/apps/cli/testdeno/helpers/net.ts
+1 -1
View File
@@ -9,7 +9,7 @@ function sleep(ms: number): Promise<void> {
}
async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise<void> {
let timer: number | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const connPromise = Deno.connect({ hostname, port });
const timeoutPromise = new Promise<never>((_, reject) => {
src/apps/cli/testdeno/helpers/p2p.ts
+8 -4
View File
@@ -76,8 +76,10 @@ export async function discoverPeer(
}
export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
if (!isLocalP2pRelay(relay)) return false;
await startP2pRelay();
const shouldStart = isLocalP2pRelay(relay);
if (shouldStart) {
await startP2pRelay();
}
const endpoint = parseRelayEndpoint(relay);
await waitForPort(endpoint.hostname, endpoint.port, {
timeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000"),
@@ -86,8 +88,10 @@ export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
});
// Docker proxy accepts TCP connections instantly before the container's internal process is fully ready.
// Wait an additional few seconds to ensure strfry is actually accepting WebSockets.
await sleep(3000);
return true;
if (shouldStart) {
await sleep(3000);
}
return shouldStart;
}
export async function stopLocalRelayIfStarted(started: boolean): Promise<void> {