mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-07-08 13:55:22 +00:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 236f9c434e | |||
| 708bd187d8 | |||
| 46081a5f47 | |||
| e5aa8d5c88 | |||
| abae4bda28 | |||
| dc9b4fd37e | |||
| cb1fb0f874 | |||
| 2cca2355fd | |||
| 5e090a3971 | |||
| fd251346e3 |
@@ -17,6 +17,8 @@ production/
|
||||
|
||||
# Test coverage and reports
|
||||
coverage/
|
||||
test/bench-network/bench-results/
|
||||
src/apps/cli/testdeno/bench-results/
|
||||
|
||||
# Local environment / secrets
|
||||
.env
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# Run the Compose-packaged CLI P2P smoke benchmark.
|
||||
#
|
||||
# This workflow is intentionally non-required at first. It exercises the local
|
||||
# Compose package for CouchDB + Nostr relay + CLI runner, and uploads the
|
||||
# benchmark JSON results for inspection.
|
||||
name: cli-p2p-compose-smoke
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/cli-p2p-compose-smoke.yml'
|
||||
- 'package.json'
|
||||
- 'package-lock.json'
|
||||
- 'src/apps/cli/**'
|
||||
- 'test/bench-network/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
cases:
|
||||
description: 'Comma-separated benchmark cases'
|
||||
required: false
|
||||
default: 'couchdb-baseline,p2p-direct-local'
|
||||
md_files:
|
||||
description: 'Markdown file count'
|
||||
required: false
|
||||
default: '2'
|
||||
bin_files:
|
||||
description: 'Binary file count'
|
||||
required: false
|
||||
default: '1'
|
||||
couchdb_rtt_ms:
|
||||
description: 'Requested CouchDB RTT in milliseconds'
|
||||
required: false
|
||||
default: '20'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Show Docker versions
|
||||
run: |
|
||||
docker --version
|
||||
docker compose version
|
||||
|
||||
- name: Run Compose P2P smoke benchmark
|
||||
env:
|
||||
BENCH_CASES: ${{ inputs.cases || 'couchdb-baseline,p2p-direct-local' }}
|
||||
BENCH_MD_FILE_COUNT: ${{ inputs.md_files || '2' }}
|
||||
BENCH_MD_MIN_SIZE_BYTES: '128'
|
||||
BENCH_MD_MAX_SIZE_BYTES: '256'
|
||||
BENCH_BIN_FILE_COUNT: ${{ inputs.bin_files || '1' }}
|
||||
BENCH_BIN_SIZE_BYTES: '512'
|
||||
BENCH_COUCHDB_RTT_MS: ${{ inputs.couchdb_rtt_ms || '20' }}
|
||||
BENCH_SYNC_TIMEOUT: '180'
|
||||
BENCH_PEERS_TIMEOUT: '20'
|
||||
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: '60000'
|
||||
BENCH_LIVESYNC_TEST_TEE: '0'
|
||||
run: docker compose -f test/bench-network/compose.yml run --rm bench-runner
|
||||
|
||||
- name: Show Compose diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
docker compose -f test/bench-network/compose.yml ps
|
||||
docker compose -f test/bench-network/compose.yml logs --no-color couchdb nostr-relay
|
||||
|
||||
- name: Upload benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cli-p2p-compose-smoke-results
|
||||
path: test/bench-network/bench-results/**
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Stop Compose services
|
||||
if: always()
|
||||
run: docker compose -f test/bench-network/compose.yml down -v --remove-orphans
|
||||
@@ -4,12 +4,21 @@ import type { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import { LiveSyncTrysteroReplicator } from "@lib/replication/trystero/LiveSyncTrysteroReplicator";
|
||||
import { compatGlobal } from "@lib/common/coreEnvFunctions.ts";
|
||||
import { LiveSyncError } from "@lib/common/LSError";
|
||||
import { getPeerConnectionStats } from "@lib/rpc/transports/DiagRTCPeerConnections.utils";
|
||||
import { appendFile } from "node:fs/promises";
|
||||
|
||||
type CLIP2PPeer = {
|
||||
peerId: string;
|
||||
name: string;
|
||||
};
|
||||
|
||||
type CandidateSummary = {
|
||||
id: string | "unknown";
|
||||
candidateType: string | "unknown";
|
||||
protocol: string | "unknown";
|
||||
relayProtocol: string | "unknown";
|
||||
};
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => compatGlobal.setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -81,6 +90,74 @@ function resolvePeer(peers: CLIP2PPeer[], peerToken: string): CLIP2PPeer | undef
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getReportValue<T extends string | number>(
|
||||
report: Record<string, unknown> | undefined,
|
||||
key: string
|
||||
): T | "unknown" {
|
||||
const value = report?.[key];
|
||||
return typeof value === "string" || typeof value === "number" ? (value as T) : "unknown";
|
||||
}
|
||||
|
||||
function summariseCandidate(reports: unknown[], candidateId: string | "unknown"): CandidateSummary | undefined {
|
||||
if (candidateId === "unknown") {
|
||||
return undefined;
|
||||
}
|
||||
const report = reports.map((r) => r as Record<string, unknown>).find((r) => r.id === candidateId);
|
||||
if (!report) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
id: candidateId,
|
||||
candidateType: getReportValue<string>(report, "candidateType"),
|
||||
protocol: getReportValue<string>(report, "protocol"),
|
||||
relayProtocol: getReportValue<string>(report, "relayProtocol"),
|
||||
};
|
||||
}
|
||||
|
||||
async function writePeerConnectionStatsIfRequested(
|
||||
replicator: LiveSyncTrysteroReplicator,
|
||||
peer: CLIP2PPeer
|
||||
): Promise<void> {
|
||||
const outputPath = process.env.LIVESYNC_P2P_STATS_JSONL?.trim();
|
||||
if (!outputPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
const peerConnection = replicator.rawHost?.room?.getPeers()[peer.peerId];
|
||||
const stats = peerConnection ? await getPeerConnectionStats(`cli-p2p-${peer.peerId}`, peerConnection) : undefined;
|
||||
const localCandidate = summariseCandidate(stats?.reports ?? [], stats?.localCandidateId ?? "unknown");
|
||||
const remoteCandidate = summariseCandidate(stats?.reports ?? [], stats?.remoteCandidateId ?? "unknown");
|
||||
const selectedPath =
|
||||
localCandidate && remoteCandidate
|
||||
? `${localCandidate.candidateType}<->${remoteCandidate.candidateType}`
|
||||
: "unknown";
|
||||
|
||||
const payload = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
command: "p2p-sync",
|
||||
peerId: peer.peerId,
|
||||
peerName: peer.name,
|
||||
candidatePathCollected: !!stats?.selectedPair,
|
||||
selectedPath,
|
||||
selectedPair: stats
|
||||
? {
|
||||
id: stats.selectedPairId,
|
||||
state: stats.state,
|
||||
currentRoundTripTime: stats.currentRoundTripTime,
|
||||
totalRoundTripTime: stats.totalRoundTripTime,
|
||||
requestsSent: stats.requestsSent,
|
||||
responsesReceived: stats.responsesReceived,
|
||||
packetsDiscardedOnSend: stats.packetsDiscardedOnSend,
|
||||
bytesSent: stats.bytesSent,
|
||||
bytesReceived: stats.bytesReceived,
|
||||
}
|
||||
: undefined,
|
||||
localCandidate,
|
||||
remoteCandidate,
|
||||
};
|
||||
await appendFile(outputPath, `${JSON.stringify(payload)}\n`, "utf8");
|
||||
}
|
||||
|
||||
export async function syncWithPeer(
|
||||
core: LiveSyncBaseCore<ServiceContext, never>,
|
||||
peerToken: string,
|
||||
@@ -118,6 +195,7 @@ export async function syncWithPeer(
|
||||
: LiveSyncError.fromError(err ?? "P2P sync failed while requesting remote sync");
|
||||
}
|
||||
|
||||
await writePeerConnectionStatsIfRequested(replicator, targetPeer);
|
||||
return targetPeer;
|
||||
} finally {
|
||||
await replicator.close();
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
|
||||
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
|
||||
import { startCouchdb, stopCouchdb } from "./helpers/docker.ts";
|
||||
import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts";
|
||||
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
|
||||
|
||||
type BenchmarkConfig = {
|
||||
caseName: string;
|
||||
couchdbBackendUri: string;
|
||||
couchdbProxyUri: string;
|
||||
couchdbUser: string;
|
||||
@@ -21,6 +22,10 @@ type BenchmarkConfig = {
|
||||
requestedRttMs: number;
|
||||
passphrase: string;
|
||||
encrypt: boolean;
|
||||
managedCouchdb: boolean;
|
||||
simulationTier: string;
|
||||
networkProfile: string;
|
||||
networkModel: string;
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
@@ -70,6 +75,7 @@ function formatBytes(value: number): string {
|
||||
|
||||
function buildConfig(): BenchmarkConfig {
|
||||
return {
|
||||
caseName: readEnvString("BENCH_CASE", "couchdb-baseline"),
|
||||
couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"),
|
||||
couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"),
|
||||
couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")),
|
||||
@@ -86,6 +92,10 @@ function buildConfig(): BenchmarkConfig {
|
||||
requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)),
|
||||
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
|
||||
encrypt: readEnvBool("BENCH_ENCRYPT", true),
|
||||
managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true),
|
||||
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
|
||||
networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "http-latency-proxy"),
|
||||
networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-http-proxy"),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -200,7 +210,17 @@ async function main(): Promise<void> {
|
||||
await initSettingsFile(settingsA);
|
||||
await initSettingsFile(settingsB);
|
||||
|
||||
if (config.managedCouchdb) {
|
||||
await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
|
||||
} else {
|
||||
console.log(`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`);
|
||||
await createCouchdbDatabase(
|
||||
config.couchdbBackendUri,
|
||||
config.couchdbUser,
|
||||
config.couchdbPassword,
|
||||
config.couchdbDbname
|
||||
);
|
||||
}
|
||||
|
||||
const proxy = startCouchdbProxy({
|
||||
backendUri: config.couchdbBackendUri,
|
||||
@@ -265,10 +285,15 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
const result = {
|
||||
caseName: config.caseName,
|
||||
mode: "couchdb-cli-benchmark",
|
||||
couchdbBackendUri: config.couchdbBackendUri,
|
||||
couchdbProxyUri: config.couchdbProxyUri,
|
||||
couchdbDbname: config.couchdbDbname,
|
||||
managedCouchdb: config.managedCouchdb,
|
||||
simulationTier: config.simulationTier,
|
||||
networkProfile: config.networkProfile,
|
||||
networkModel: config.networkModel,
|
||||
rttRequestedMs: config.requestedRttMs,
|
||||
proxyApplied: proxy.applied,
|
||||
proxyNote: proxy.note,
|
||||
@@ -300,8 +325,10 @@ async function main(): Promise<void> {
|
||||
);
|
||||
} finally {
|
||||
await proxy.stop();
|
||||
if (config.managedCouchdb) {
|
||||
await stopCouchdb().catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
type SweepResult = {
|
||||
name: string;
|
||||
runner: "p2p" | "couchdb";
|
||||
rttMs?: number;
|
||||
result: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
const value = Deno.env.get(name)?.trim();
|
||||
return value && value.length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function timestamp(): string {
|
||||
const d = new Date();
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
return (
|
||||
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
|
||||
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
|
||||
);
|
||||
}
|
||||
|
||||
function parseRttList(raw: string): number[] {
|
||||
const values = raw
|
||||
.split(",")
|
||||
.map((value) => Number(value.trim()))
|
||||
.filter((value) => Number.isFinite(value) && value > 0)
|
||||
.map((value) => Math.floor(value));
|
||||
if (values.length === 0) {
|
||||
throw new Error(`BENCH_SWEEP_RTT_MS must contain at least one positive number, got '${raw}'`);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function buildBaseEnv(): Record<string, string> {
|
||||
return {
|
||||
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
|
||||
BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
|
||||
BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
|
||||
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
|
||||
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
|
||||
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
|
||||
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
|
||||
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
|
||||
};
|
||||
}
|
||||
|
||||
async function runBenchmark(options: {
|
||||
taskName: "bench:p2p" | "bench:couchdb";
|
||||
name: string;
|
||||
outputDir: string;
|
||||
env: Record<string, string>;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
const resultPath = `${options.outputDir}/${options.name}.json`;
|
||||
const env = {
|
||||
...Deno.env.toObject(),
|
||||
...options.env,
|
||||
BENCH_RESULT_JSON: resultPath,
|
||||
};
|
||||
|
||||
console.log(`[latency-sweep] running ${options.name}`);
|
||||
const child = new Deno.Command("deno", {
|
||||
args: ["task", options.taskName],
|
||||
cwd: import.meta.dirname,
|
||||
env,
|
||||
stdin: "null",
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
}).spawn();
|
||||
const status = await child.status;
|
||||
if (status.code !== 0) {
|
||||
throw new Error(`benchmark failed: ${options.name} (exit ${status.code})`);
|
||||
}
|
||||
return JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`);
|
||||
const outputDir = `${outRoot}/latency-sweep-${timestamp()}`;
|
||||
const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300"));
|
||||
const base = buildBaseEnv();
|
||||
|
||||
await Deno.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const results: SweepResult[] = [];
|
||||
if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") {
|
||||
const p2pResult = await runBenchmark({
|
||||
taskName: "bench:p2p",
|
||||
name: "p2p-direct-local",
|
||||
outputDir,
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-direct-local",
|
||||
BENCH_TURN_SERVERS: "",
|
||||
},
|
||||
});
|
||||
results.push({ name: "p2p-direct-local", runner: "p2p", result: p2pResult });
|
||||
}
|
||||
|
||||
for (const rtt of rtts) {
|
||||
const name = `couchdb-rtt-${rtt}ms`;
|
||||
const couchdbResult = await runBenchmark({
|
||||
taskName: "bench:couchdb",
|
||||
name,
|
||||
outputDir,
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: name,
|
||||
BENCH_COUCHDB_RTT_MS: String(rtt),
|
||||
},
|
||||
});
|
||||
results.push({ name, runner: "couchdb", rttMs: rtt, result: couchdbResult });
|
||||
}
|
||||
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
outputDir,
|
||||
note:
|
||||
"This sweep models additional remote CouchDB request latency through the existing HTTP proxy. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
|
||||
rtts,
|
||||
results,
|
||||
};
|
||||
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
console.log(`[latency-sweep] result directory: ${outputDir}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error("[Fatal Error]", error);
|
||||
Deno.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
type BenchmarkCase = {
|
||||
name: string;
|
||||
runner: "p2p" | "couchdb";
|
||||
description: string;
|
||||
dataPath: string;
|
||||
trustBoundary: string;
|
||||
env: Record<string, string>;
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
const value = Deno.env.get(name)?.trim();
|
||||
return value && value.length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function readEnvInteger(name: string, fallback: number): number {
|
||||
const value = readEnvString(name, String(fallback));
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`${name} must be a positive integer, got '${value}'`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function timestamp(): string {
|
||||
const d = new Date();
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
return (
|
||||
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
|
||||
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
|
||||
);
|
||||
}
|
||||
|
||||
function buildBaseEnv(): Record<string, string> {
|
||||
return {
|
||||
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
|
||||
BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
|
||||
BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
|
||||
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
|
||||
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
|
||||
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
|
||||
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
|
||||
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
|
||||
};
|
||||
}
|
||||
|
||||
function buildCases(): BenchmarkCase[] {
|
||||
const base = buildBaseEnv();
|
||||
const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20");
|
||||
const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120");
|
||||
const localTurnServers = readEnvString("BENCH_LOCAL_TURN_SERVERS", "turn:127.0.0.1:3478");
|
||||
const shimCouchdbUri = readEnvString("BENCH_SHIM_COUCHDB_URI", "http://couchdb-shim:5984");
|
||||
const signallingShimRelay = readEnvString("BENCH_SIGNAL_SHIM_RELAY", "ws://p2p-signalling-shim:7777/");
|
||||
|
||||
return [
|
||||
{
|
||||
name: "couchdb-baseline",
|
||||
runner: "couchdb",
|
||||
description: "Standard self-hosted CouchDB path through a local latency proxy.",
|
||||
dataPath: "Device A -> CouchDB -> Device B",
|
||||
trustBoundary: "CouchDB operator and network path",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-baseline",
|
||||
BENCH_COUCHDB_RTT_MS: couchdbRtt,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "p2p-direct-local",
|
||||
runner: "p2p",
|
||||
description: "Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
|
||||
dataPath: "Device A -> Device B",
|
||||
trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-direct-local",
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "1",
|
||||
BENCH_NETWORK_PROFILE: "local-direct",
|
||||
BENCH_NETWORK_MODEL: "local-runner-webrtc",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION: "turn-disabled-but-selected-ice-pair-not-collected",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "couchdb-tethering-vpn-proxy",
|
||||
runner: "couchdb",
|
||||
description:
|
||||
"Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.",
|
||||
dataPath: "Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
|
||||
trustBoundary: "VPN/network path and CouchDB operator",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-tethering-vpn-proxy",
|
||||
BENCH_COUCHDB_RTT_MS: tetheringVpnRtt,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "couchdb-netem-home-wifi",
|
||||
runner: "couchdb",
|
||||
description:
|
||||
"Tier 2 CouchDB path through the Compose netem TCP shim using the home-wifi profile.",
|
||||
dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
|
||||
trustBoundary: "CouchDB operator and constrained network shim",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-netem-home-wifi",
|
||||
BENCH_COUCHDB_BACKEND_URI: shimCouchdbUri,
|
||||
BENCH_COUCHDB_RTT_MS: "1",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "home-wifi",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "couchdb-netem-tethering-vpn",
|
||||
runner: "couchdb",
|
||||
description:
|
||||
"Tier 2 CouchDB path through the Compose netem TCP shim using a tethering-vpn profile.",
|
||||
dataPath: "Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
|
||||
trustBoundary: "CouchDB operator and constrained smartphone/VPN-like network shim",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-netem-tethering-vpn",
|
||||
BENCH_COUCHDB_BACKEND_URI: shimCouchdbUri,
|
||||
BENCH_COUCHDB_RTT_MS: "1",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "tethering-vpn",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "p2p-smartphone-vpn-direct",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.",
|
||||
dataPath: "Device A -> Device B when WebRTC direct connectivity succeeds",
|
||||
trustBoundary: "Smartphone/VPN routing policy plus Nostr signalling metadata",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-smartphone-vpn-direct",
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "unmeasured",
|
||||
BENCH_NETWORK_PROFILE: "smartphone-vpn-direct-placeholder",
|
||||
BENCH_NETWORK_MODEL: "local-runner-no-netem",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"structural-placeholder-only; selected ICE pair may be collected, but the path is not shaped",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "p2p-signalling-netem-home-wifi",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.",
|
||||
dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
|
||||
trustBoundary: "Nostr signalling metadata through constrained network shim; no TURN relay",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-signalling-netem-home-wifi",
|
||||
BENCH_RELAY: signallingShimRelay,
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "home-wifi",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-signalling-shim",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"selected ICE pair collected; only Nostr signalling path is shaped",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "p2p-signalling-netem-tethering-vpn",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.",
|
||||
dataPath: "Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
|
||||
trustBoundary: "Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-signalling-netem-tethering-vpn",
|
||||
BENCH_RELAY: signallingShimRelay,
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "tethering-vpn",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-signalling-shim",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"selected ICE pair collected; only Nostr signalling path is shaped",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "p2p-user-turn",
|
||||
runner: "p2p",
|
||||
description: "Optional fallback path through a local user-controlled TURN server.",
|
||||
dataPath: "Device A -> user-controlled TURN -> Device B",
|
||||
trustBoundary: "User-controlled TURN server",
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-user-turn",
|
||||
BENCH_TURN_SERVERS: localTurnServers,
|
||||
BENCH_SIMULATION_TIER: "1",
|
||||
BENCH_NETWORK_PROFILE: "local-turn-fallback",
|
||||
BENCH_NETWORK_MODEL: "local-runner-webrtc-turn-configured",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"turn-configured; selected ICE pair may still be direct or relayed, so interpret the recorded candidate types",
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
async function runCase(
|
||||
testCase: BenchmarkCase,
|
||||
outputDir: string,
|
||||
repeatIndex: number,
|
||||
repeatCount: number
|
||||
): Promise<Record<string, unknown>> {
|
||||
const suffix = repeatCount > 1 ? `-r${String(repeatIndex).padStart(2, "0")}` : "";
|
||||
const resultPath = `${outputDir}/${testCase.name}${suffix}.json`;
|
||||
const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb";
|
||||
const env = {
|
||||
...Deno.env.toObject(),
|
||||
...testCase.env,
|
||||
BENCH_RESULT_JSON: resultPath,
|
||||
BENCH_REPEAT_INDEX: String(repeatIndex),
|
||||
BENCH_REPEAT_COUNT: String(repeatCount),
|
||||
};
|
||||
|
||||
const repeatLabel = repeatCount > 1 ? ` (${repeatIndex}/${repeatCount})` : "";
|
||||
console.log(`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`);
|
||||
const command = new Deno.Command("deno", {
|
||||
args: ["task", taskName],
|
||||
cwd: import.meta.dirname,
|
||||
env,
|
||||
stdin: "null",
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
});
|
||||
|
||||
const child = command.spawn();
|
||||
const status = await child.status;
|
||||
if (status.code !== 0) {
|
||||
throw new Error(`case failed: ${testCase.name} (exit ${status.code})`);
|
||||
}
|
||||
|
||||
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
|
||||
return {
|
||||
...testCase,
|
||||
repeatIndex,
|
||||
repeatCount,
|
||||
resultPath,
|
||||
result,
|
||||
};
|
||||
}
|
||||
|
||||
function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
|
||||
const requested = readEnvString("BENCH_CASES", "couchdb-baseline,p2p-direct-local");
|
||||
const names = requested
|
||||
.split(",")
|
||||
.map((v) => v.trim())
|
||||
.filter((v) => v.length > 0);
|
||||
const byName = new Map(allCases.map((c) => [c.name, c]));
|
||||
return names.map((name) => {
|
||||
const found = byName.get(name);
|
||||
if (!found) {
|
||||
throw new Error(`Unknown BENCH_CASES entry '${name}'. Available: ${allCases.map((c) => c.name).join(", ")}`);
|
||||
}
|
||||
return found;
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const outRoot = readEnvString("BENCH_CASES_ROOT", `${import.meta.dirname}/bench-results`);
|
||||
const outputDir = `${outRoot}/cases-${timestamp()}`;
|
||||
await Deno.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const allCases = buildCases();
|
||||
const cases = selectCases(allCases);
|
||||
const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1);
|
||||
await Deno.writeTextFile(
|
||||
`${outputDir}/case-manifest.json`,
|
||||
JSON.stringify(
|
||||
{
|
||||
generatedAt: new Date().toISOString(),
|
||||
repeatCount,
|
||||
selectedCases: cases,
|
||||
availableCases: allCases,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
|
||||
const results: Record<string, unknown>[] = [];
|
||||
for (const testCase of cases) {
|
||||
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
|
||||
results.push(await runCase(testCase, outputDir, repeatIndex, repeatCount));
|
||||
}
|
||||
}
|
||||
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
outputDir,
|
||||
repeatCount,
|
||||
results,
|
||||
};
|
||||
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
console.log(`[bench-cases] result directory: ${outputDir}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error("[Fatal Error]", error);
|
||||
Deno.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
import { join } from "@std/path";
|
||||
import { startCliInBackground } from "./helpers/backgroundCli.ts";
|
||||
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
|
||||
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
|
||||
import { discoverPeer } from "./helpers/p2p.ts";
|
||||
import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
|
||||
|
||||
type Role = "host" | "client";
|
||||
|
||||
type NetemSummary = {
|
||||
enabled: boolean;
|
||||
profile: string;
|
||||
interface: string;
|
||||
delayMs: number;
|
||||
jitterMs: number;
|
||||
lossPercent: number;
|
||||
bandwidthMbit: number;
|
||||
mtu: number;
|
||||
tcQdisc?: string;
|
||||
ipAddr?: string;
|
||||
ipRoute?: string;
|
||||
};
|
||||
|
||||
type HostReady = {
|
||||
generatedAt: string;
|
||||
totalFiles: number;
|
||||
totalBytes: number;
|
||||
mdFileCount: number;
|
||||
binFileCount: number;
|
||||
mirrorElapsedMs: number;
|
||||
netem: NetemSummary;
|
||||
};
|
||||
|
||||
type P2PConnectionStats = {
|
||||
candidatePathCollected: boolean;
|
||||
selectedPath: string;
|
||||
localCandidate?: { candidateType: string; protocol: string; relayProtocol: string };
|
||||
remoteCandidate?: { candidateType: string; protocol: string; relayProtocol: string };
|
||||
};
|
||||
|
||||
function errorToRecord(error: unknown): Record<string, unknown> {
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
};
|
||||
}
|
||||
return {
|
||||
name: "UnknownError",
|
||||
message: String(error),
|
||||
};
|
||||
}
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
const value = Deno.env.get(name)?.trim();
|
||||
return value && value.length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function readEnvNumber(name: string, fallback: number): number {
|
||||
const raw = Deno.env.get(name);
|
||||
if (raw === undefined || raw.trim() === "") {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) {
|
||||
throw new Error(`${name} must be a non-negative number, got '${raw}'`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function nowMs(): number {
|
||||
return performance.now();
|
||||
}
|
||||
|
||||
async function commandOutput(command: string, args: string[]): Promise<string> {
|
||||
const output = await new Deno.Command(command, {
|
||||
args,
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
}).output();
|
||||
const stdout = new TextDecoder().decode(output.stdout);
|
||||
const stderr = new TextDecoder().decode(output.stderr);
|
||||
if (!output.success) {
|
||||
throw new Error(`${command} ${args.join(" ")} failed\nstdout: ${stdout}\nstderr: ${stderr}`);
|
||||
}
|
||||
return stdout.trim();
|
||||
}
|
||||
|
||||
async function commandOk(command: string, args: string[]): Promise<void> {
|
||||
await commandOutput(command, args);
|
||||
}
|
||||
|
||||
async function applyNetemIfRequested(): Promise<NetemSummary> {
|
||||
const enabled = readEnvString("BENCH_NETEM_ENABLED", "0") === "1";
|
||||
const profile = readEnvString("NETEM_PROFILE", "home-wifi");
|
||||
const iface = readEnvString("NETEM_INTERFACE", "eth0");
|
||||
const delayMs = readEnvNumber("NETEM_DELAY_MS", 20);
|
||||
const jitterMs = readEnvNumber("NETEM_JITTER_MS", 5);
|
||||
const lossPercent = readEnvNumber("NETEM_LOSS_PERCENT", 0.1);
|
||||
const bandwidthMbit = readEnvNumber("NETEM_BANDWIDTH_MBIT", 100);
|
||||
const mtu = readEnvNumber("NETEM_MTU", 1500);
|
||||
|
||||
const summary: NetemSummary = {
|
||||
enabled,
|
||||
profile,
|
||||
interface: iface,
|
||||
delayMs,
|
||||
jitterMs,
|
||||
lossPercent,
|
||||
bandwidthMbit,
|
||||
mtu,
|
||||
};
|
||||
|
||||
if (!enabled) {
|
||||
return summary;
|
||||
}
|
||||
|
||||
await commandOk("ip", ["link", "set", "dev", iface, "mtu", String(mtu)]);
|
||||
await new Deno.Command("tc", { args: ["qdisc", "del", "dev", iface, "root"] }).output();
|
||||
await commandOk("tc", [
|
||||
"qdisc",
|
||||
"add",
|
||||
"dev",
|
||||
iface,
|
||||
"root",
|
||||
"netem",
|
||||
"delay",
|
||||
`${delayMs}ms`,
|
||||
`${jitterMs}ms`,
|
||||
"loss",
|
||||
`${lossPercent}%`,
|
||||
"rate",
|
||||
`${bandwidthMbit}mbit`,
|
||||
]);
|
||||
summary.tcQdisc = await commandOutput("tc", ["qdisc", "show", "dev", iface]);
|
||||
summary.ipAddr = await commandOutput("ip", ["addr", "show", iface]);
|
||||
summary.ipRoute = await commandOutput("ip", ["route"]);
|
||||
return summary;
|
||||
}
|
||||
|
||||
async function waitForFile(path: string, timeoutMs: number): Promise<void> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
try {
|
||||
const stat = await Deno.stat(path);
|
||||
if (stat.isFile) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// wait
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${path}`);
|
||||
}
|
||||
|
||||
async function readJsonFile<T>(path: string): Promise<T> {
|
||||
return JSON.parse(await Deno.readTextFile(path)) as T;
|
||||
}
|
||||
|
||||
function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
|
||||
const unique = new Map<string, DatasetEntry>();
|
||||
for (const entry of [entries.find((e) => e.kind === "md"), entries.find((e) => e.kind === "bin"), entries.at(-1)]) {
|
||||
if (entry) {
|
||||
unique.set(entry.relativePath, entry);
|
||||
}
|
||||
}
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
async function readLatestP2PConnectionStats(path: string): Promise<P2PConnectionStats | undefined> {
|
||||
try {
|
||||
const lines = (await Deno.readTextFile(path))
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0);
|
||||
return lines.length === 0 ? undefined : (JSON.parse(lines.at(-1)!) as P2PConnectionStats);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function buildCommonConfig() {
|
||||
const runId = readEnvString("BENCH_SPLIT_RUN_ID", readEnvString("BENCH_ROOM_ID", "bench-split-run"));
|
||||
const baseWorkRoot = readEnvString("BENCH_SPLIT_WORK_ROOT", "/p2p-work");
|
||||
return {
|
||||
runId,
|
||||
workRoot: join(baseWorkRoot, runId),
|
||||
resultRoot: readEnvString("BENCH_SPLIT_RESULT_ROOT", "/workspace/src/apps/cli/testdeno/bench-results"),
|
||||
relay: readEnvString("BENCH_RELAY", "ws://nostr-relay:7777/"),
|
||||
appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
|
||||
roomId: readEnvString("BENCH_ROOM_ID", "bench-split-room"),
|
||||
passphrase: readEnvString("BENCH_PASSPHRASE", "bench-split-passphrase"),
|
||||
turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
|
||||
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
|
||||
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 20)),
|
||||
mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 512)),
|
||||
mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 2048)),
|
||||
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 5)),
|
||||
binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 8192)),
|
||||
peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 60),
|
||||
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 300),
|
||||
nodeTimeoutMs: readEnvNumber("BENCH_SPLIT_NODE_TIMEOUT_MS", 360_000),
|
||||
profile: readEnvString("BENCH_NETWORK_PROFILE", readEnvString("NETEM_PROFILE", "split-compose")),
|
||||
};
|
||||
}
|
||||
|
||||
async function prepareP2PSettings(
|
||||
settingsPath: string,
|
||||
peerName: string,
|
||||
config: ReturnType<typeof buildCommonConfig>
|
||||
) {
|
||||
await initSettingsFile(settingsPath);
|
||||
await applyP2pSettings(
|
||||
settingsPath,
|
||||
config.roomId,
|
||||
config.passphrase,
|
||||
config.appId,
|
||||
config.relay,
|
||||
"~.*",
|
||||
config.turnServers
|
||||
);
|
||||
await applyP2pTestTweaks(settingsPath, peerName, config.passphrase);
|
||||
}
|
||||
|
||||
async function runHost(): Promise<void> {
|
||||
const config = buildCommonConfig();
|
||||
const netem = await applyNetemIfRequested();
|
||||
await Deno.mkdir(config.workRoot, { recursive: true });
|
||||
await Deno.mkdir(config.resultRoot, { recursive: true });
|
||||
|
||||
const hostVault = join(config.workRoot, "vault-host");
|
||||
const hostSettings = join(config.workRoot, "settings-host.json");
|
||||
await Deno.mkdir(hostVault, { recursive: true });
|
||||
await prepareP2PSettings(hostSettings, "p2p-split-host", config);
|
||||
|
||||
const seedFiles = await createDeterministicDataset({
|
||||
rootDir: hostVault,
|
||||
datasetDirName: config.datasetDirName,
|
||||
seed: config.datasetSeed,
|
||||
mdCount: config.mdFileCount,
|
||||
mdMinSizeBytes: config.mdMinSizeBytes,
|
||||
mdMaxSizeBytes: config.mdMaxSizeBytes,
|
||||
binCount: config.binFileCount,
|
||||
binSizeBytes: config.binSizeBytes,
|
||||
});
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "sample-files.json"),
|
||||
JSON.stringify(pickSampleFiles(seedFiles.entries), null, 2)
|
||||
);
|
||||
|
||||
const mirrorStart = nowMs();
|
||||
await runCliOrFail(hostVault, "--settings", hostSettings, "mirror");
|
||||
const mirrorElapsedMs = Number((nowMs() - mirrorStart).toFixed(1));
|
||||
const hostReady: HostReady = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
totalFiles: seedFiles.totalFiles,
|
||||
totalBytes: seedFiles.totalBytes,
|
||||
mdFileCount: seedFiles.mdCount,
|
||||
binFileCount: seedFiles.binCount,
|
||||
mirrorElapsedMs,
|
||||
netem,
|
||||
};
|
||||
await Deno.writeTextFile(join(config.workRoot, "host-ready.json"), JSON.stringify(hostReady, null, 2));
|
||||
|
||||
const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
|
||||
try {
|
||||
await host.waitUntilContains("P2P host is running", 20_000);
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "p2p-host-ready.json"),
|
||||
JSON.stringify({ generatedAt: new Date().toISOString() })
|
||||
);
|
||||
await waitForFile(join(config.workRoot, "client-done.json"), config.nodeTimeoutMs);
|
||||
} finally {
|
||||
await host.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function runClient(): Promise<void> {
|
||||
const config = buildCommonConfig();
|
||||
const netem = await applyNetemIfRequested();
|
||||
await Deno.mkdir(config.resultRoot, { recursive: true });
|
||||
await waitForFile(join(config.workRoot, "host-ready.json"), config.nodeTimeoutMs);
|
||||
await waitForFile(join(config.workRoot, "p2p-host-ready.json"), config.nodeTimeoutMs);
|
||||
|
||||
const clientVault = join(config.workRoot, "vault-client");
|
||||
const clientSettings = join(config.workRoot, "settings-client.json");
|
||||
const statsPath = join(config.workRoot, "p2p-connection-stats.jsonl");
|
||||
await Deno.mkdir(clientVault, { recursive: true });
|
||||
await prepareP2PSettings(clientSettings, "p2p-split-client", config);
|
||||
|
||||
const hostReady = await readJsonFile<HostReady>(join(config.workRoot, "host-ready.json"));
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/g, "").slice(0, 15);
|
||||
const outputDir = join(config.resultRoot, `p2p-split-${config.profile}-${timestamp}`);
|
||||
await Deno.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const previousStatsPath = Deno.env.get("LIVESYNC_P2P_STATS_JSONL");
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", statsPath);
|
||||
let stage = "peer-discovery";
|
||||
let peerDiscoveryCommandElapsedMs: number | undefined;
|
||||
let syncElapsedMs: number | undefined;
|
||||
try {
|
||||
const peerDiscoveryCommandStart = nowMs();
|
||||
const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds);
|
||||
peerDiscoveryCommandElapsedMs = Number((nowMs() - peerDiscoveryCommandStart).toFixed(1));
|
||||
|
||||
stage = "p2p-sync";
|
||||
const syncStart = nowMs();
|
||||
await runCliOrFail(
|
||||
clientVault,
|
||||
"--settings",
|
||||
clientSettings,
|
||||
"p2p-sync",
|
||||
peer.id,
|
||||
String(config.syncTimeoutSeconds)
|
||||
);
|
||||
syncElapsedMs = Number((nowMs() - syncStart).toFixed(1));
|
||||
|
||||
stage = "sample-verification";
|
||||
const samples = await readJsonFile<DatasetEntry[]>(join(config.workRoot, "sample-files.json"));
|
||||
for (const sample of samples) {
|
||||
const pulledPath = join(config.workRoot, `pulled-${sample.relativePath.replaceAll("/", "_")}`);
|
||||
await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath);
|
||||
await assertFilesEqual(
|
||||
sample.absolutePath,
|
||||
pulledPath,
|
||||
`sample file mismatch after split sync: ${sample.relativePath}`
|
||||
);
|
||||
}
|
||||
|
||||
const p2pConnectionStats = await readLatestP2PConnectionStats(statsPath);
|
||||
const result = {
|
||||
ok: true,
|
||||
generatedAt: new Date().toISOString(),
|
||||
caseName: "p2p-split-compose",
|
||||
mode: "p2p-split-compose-benchmark",
|
||||
runId: config.runId,
|
||||
simulationTier: Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "2" : "1",
|
||||
networkProfile: config.profile,
|
||||
networkModel:
|
||||
Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "split-compose-egress-netem" : "split-compose-no-netem",
|
||||
relay: config.relay,
|
||||
turnServers: config.turnServers,
|
||||
turnEnabled: config.turnServers.trim().length > 0,
|
||||
p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
|
||||
p2pConnectionStats,
|
||||
hostNetem: hostReady.netem,
|
||||
clientNetem: netem,
|
||||
totalFiles: hostReady.totalFiles,
|
||||
totalBytes: hostReady.totalBytes,
|
||||
mdFileCount: hostReady.mdFileCount,
|
||||
binFileCount: hostReady.binFileCount,
|
||||
mirrorElapsedMs: hostReady.mirrorElapsedMs,
|
||||
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
|
||||
peerDiscoveryCommandElapsedMs,
|
||||
syncElapsedMs,
|
||||
throughputBytesPerSec: Number((hostReady.totalBytes / (syncElapsedMs / 1000)).toFixed(2)),
|
||||
throughputMiBPerSec: Number((hostReady.totalBytes / (syncElapsedMs / 1000) / 1024 / 1024).toFixed(4)),
|
||||
};
|
||||
await Deno.writeTextFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2));
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "client-done.json"),
|
||||
JSON.stringify({ generatedAt: new Date().toISOString(), outputDir, ok: true })
|
||||
);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
const p2pConnectionStats = await readLatestP2PConnectionStats(statsPath);
|
||||
const result = {
|
||||
ok: false,
|
||||
generatedAt: new Date().toISOString(),
|
||||
caseName: "p2p-split-compose",
|
||||
mode: "p2p-split-compose-benchmark",
|
||||
runId: config.runId,
|
||||
simulationTier: Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "2" : "1",
|
||||
networkProfile: config.profile,
|
||||
networkModel:
|
||||
Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "split-compose-egress-netem" : "split-compose-no-netem",
|
||||
relay: config.relay,
|
||||
turnServers: config.turnServers,
|
||||
turnEnabled: config.turnServers.trim().length > 0,
|
||||
p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
|
||||
p2pConnectionStats,
|
||||
hostNetem: hostReady.netem,
|
||||
clientNetem: netem,
|
||||
totalFiles: hostReady.totalFiles,
|
||||
totalBytes: hostReady.totalBytes,
|
||||
mdFileCount: hostReady.mdFileCount,
|
||||
binFileCount: hostReady.binFileCount,
|
||||
mirrorElapsedMs: hostReady.mirrorElapsedMs,
|
||||
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
|
||||
peerDiscoveryCommandElapsedMs,
|
||||
syncElapsedMs,
|
||||
failure: {
|
||||
stage,
|
||||
...errorToRecord(error),
|
||||
},
|
||||
};
|
||||
await Deno.writeTextFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2));
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "client-done.json"),
|
||||
JSON.stringify({ generatedAt: new Date().toISOString(), outputDir, ok: false })
|
||||
);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
throw error;
|
||||
} finally {
|
||||
if (previousStatsPath === undefined) {
|
||||
Deno.env.delete("LIVESYNC_P2P_STATS_JSONL");
|
||||
} else {
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", previousStatsPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const role = readEnvString("BENCH_P2P_SPLIT_ROLE", "") as Role;
|
||||
if (role === "host") {
|
||||
await runHost();
|
||||
return;
|
||||
}
|
||||
if (role === "client") {
|
||||
await runClient();
|
||||
return;
|
||||
}
|
||||
throw new Error("BENCH_P2P_SPLIT_ROLE must be 'host' or 'client'");
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error("[Fatal Error]", error);
|
||||
Deno.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,15 +1,23 @@
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
|
||||
import { startCliInBackground } from "./helpers/backgroundCli.ts";
|
||||
import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts";
|
||||
import {
|
||||
discoverPeer,
|
||||
maybeStartCoturn,
|
||||
maybeStartLocalRelay,
|
||||
stopCoturnIfStarted,
|
||||
stopLocalRelayIfStarted,
|
||||
} from "./helpers/p2p.ts";
|
||||
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
|
||||
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
|
||||
|
||||
type BenchmarkConfig = {
|
||||
caseName: string;
|
||||
relay: string;
|
||||
appId: string;
|
||||
roomId: string;
|
||||
passphrase: string;
|
||||
turnServers: string;
|
||||
datasetDirName: string;
|
||||
datasetSeed: string;
|
||||
mdFileCount: number;
|
||||
@@ -19,6 +27,42 @@ type BenchmarkConfig = {
|
||||
binSizeBytes: number;
|
||||
peersTimeoutSeconds: number;
|
||||
syncTimeoutSeconds: number;
|
||||
simulationTier: string;
|
||||
networkProfile: string;
|
||||
networkModel: string;
|
||||
candidatePathVerification: string;
|
||||
};
|
||||
|
||||
type P2PConnectionStats = {
|
||||
generatedAt: string;
|
||||
command: string;
|
||||
peerId: string;
|
||||
peerName: string;
|
||||
candidatePathCollected: boolean;
|
||||
selectedPath: string;
|
||||
selectedPair?: {
|
||||
id: string;
|
||||
state: string;
|
||||
currentRoundTripTime: number | "unknown";
|
||||
totalRoundTripTime: number | "unknown";
|
||||
requestsSent: number | "unknown";
|
||||
responsesReceived: number | "unknown";
|
||||
packetsDiscardedOnSend: number | "unknown";
|
||||
bytesSent: number | "unknown";
|
||||
bytesReceived: number | "unknown";
|
||||
};
|
||||
localCandidate?: {
|
||||
id: string;
|
||||
candidateType: string;
|
||||
protocol: string;
|
||||
relayProtocol: string;
|
||||
};
|
||||
remoteCandidate?: {
|
||||
id: string;
|
||||
candidateType: string;
|
||||
protocol: string;
|
||||
relayProtocol: string;
|
||||
};
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
@@ -61,10 +105,12 @@ function formatBytes(value: number): string {
|
||||
|
||||
function buildConfig(): BenchmarkConfig {
|
||||
return {
|
||||
caseName: readEnvString("BENCH_CASE", "p2p-direct-local"),
|
||||
relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"),
|
||||
appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
|
||||
roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`),
|
||||
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
|
||||
turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
|
||||
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
|
||||
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
|
||||
@@ -74,6 +120,10 @@ function buildConfig(): BenchmarkConfig {
|
||||
binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
|
||||
peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20),
|
||||
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
|
||||
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
|
||||
networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "local-direct"),
|
||||
networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-runner-webrtc"),
|
||||
candidatePathVerification: readEnvString("BENCH_P2P_CANDIDATE_PATH_VERIFICATION", "not-collected"),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -102,18 +152,39 @@ function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
async function readLatestP2PConnectionStats(statsPath: string): Promise<P2PConnectionStats | undefined> {
|
||||
try {
|
||||
const text = await Deno.readTextFile(statsPath);
|
||||
const lines = text
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0);
|
||||
if (lines.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return JSON.parse(lines[lines.length - 1]) as P2PConnectionStats;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const config = buildConfig();
|
||||
const resultPath = readOptionalResultPath();
|
||||
|
||||
const relayStarted = await maybeStartLocalRelay(config.relay);
|
||||
const coturnStarted = await maybeStartCoturn(config.turnServers);
|
||||
await using workDir = await TempDir.create("livesync-cli-p2p-bench");
|
||||
|
||||
const hostVault = workDir.join("vault-host");
|
||||
const clientVault = workDir.join("vault-client");
|
||||
const hostSettings = workDir.join("settings-host.json");
|
||||
const clientSettings = workDir.join("settings-client.json");
|
||||
const p2pStatsPath = workDir.join("p2p-connection-stats.jsonl");
|
||||
const previousStatsPath = Deno.env.get("LIVESYNC_P2P_STATS_JSONL");
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", p2pStatsPath);
|
||||
|
||||
try {
|
||||
await Promise.all([
|
||||
Deno.mkdir(hostVault, { recursive: true }),
|
||||
Deno.mkdir(clientVault, { recursive: true }),
|
||||
@@ -122,8 +193,24 @@ async function main(): Promise<void> {
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
applyP2pSettings(hostSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
|
||||
applyP2pSettings(clientSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
|
||||
applyP2pSettings(
|
||||
hostSettings,
|
||||
config.roomId,
|
||||
config.passphrase,
|
||||
config.appId,
|
||||
config.relay,
|
||||
"~.*",
|
||||
config.turnServers
|
||||
),
|
||||
applyP2pSettings(
|
||||
clientSettings,
|
||||
config.roomId,
|
||||
config.passphrase,
|
||||
config.appId,
|
||||
config.relay,
|
||||
"~.*",
|
||||
config.turnServers
|
||||
),
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
@@ -152,9 +239,9 @@ async function main(): Promise<void> {
|
||||
await host.waitUntilContains("P2P host is running", 20000);
|
||||
const hostReadyElapsed = nowMs() - hostReadyStart;
|
||||
|
||||
const peerDiscoveryStart = nowMs();
|
||||
const peerDiscoveryCommandStart = nowMs();
|
||||
const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds);
|
||||
const peerDiscoveryElapsed = nowMs() - peerDiscoveryStart;
|
||||
const peerDiscoveryCommandElapsed = nowMs() - peerDiscoveryCommandStart;
|
||||
|
||||
const syncStart = nowMs();
|
||||
await runCliOrFail(
|
||||
@@ -178,9 +265,26 @@ async function main(): Promise<void> {
|
||||
);
|
||||
}
|
||||
|
||||
const p2pConnectionStats = await readLatestP2PConnectionStats(p2pStatsPath);
|
||||
const result = {
|
||||
caseName: config.caseName,
|
||||
mode: "p2p-cli-benchmark",
|
||||
relay: config.relay,
|
||||
turnServers: config.turnServers,
|
||||
turnEnabled: config.turnServers.trim().length > 0,
|
||||
simulationTier: config.simulationTier,
|
||||
networkProfile: config.networkProfile,
|
||||
networkModel: config.networkModel,
|
||||
p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
|
||||
p2pCandidatePathVerification: p2pConnectionStats?.candidatePathCollected
|
||||
? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
|
||||
: config.candidatePathVerification,
|
||||
p2pCandidatePathNote: p2pConnectionStats?.candidatePathCollected
|
||||
? "The selected ICE candidate pair was collected by the CLI benchmark. Interpret the path from the candidate types; do not infer TURN use from configuration alone."
|
||||
: config.turnServers.trim().length > 0
|
||||
? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run."
|
||||
: "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.",
|
||||
p2pConnectionStats,
|
||||
appId: config.appId,
|
||||
roomId: config.roomId,
|
||||
datasetSeed: config.datasetSeed,
|
||||
@@ -193,7 +297,10 @@ async function main(): Promise<void> {
|
||||
binFileCount: seedFiles.binCount,
|
||||
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
|
||||
hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
|
||||
peerDiscoveryElapsedMs: Number(peerDiscoveryElapsed.toFixed(1)),
|
||||
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
|
||||
peerDiscoveryCommandElapsedMs: Number(peerDiscoveryCommandElapsed.toFixed(1)),
|
||||
peerDiscoveryNote:
|
||||
"p2p-peers waits for the requested timeout before printing discovered peers, so this is command duration, not first-peer latency.",
|
||||
syncElapsedMs: Number(syncElapsed.toFixed(1)),
|
||||
throughputBytesPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2)),
|
||||
throughputMiBPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024).toFixed(4)),
|
||||
@@ -205,12 +312,22 @@ async function main(): Promise<void> {
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
console.error(
|
||||
`[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes(seedFiles.totalBytes)}) in ${formatMs(mirrorElapsed)}, ` +
|
||||
`[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes(
|
||||
seedFiles.totalBytes
|
||||
)}) in ${formatMs(mirrorElapsed)}, ` +
|
||||
`synced in ${formatMs(syncElapsed)} ` +
|
||||
`(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
|
||||
);
|
||||
} finally {
|
||||
await host.stop();
|
||||
}
|
||||
} finally {
|
||||
if (previousStatsPath === undefined) {
|
||||
Deno.env.delete("LIVESYNC_P2P_STATS_JSONL");
|
||||
} else {
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", previousStatsPath);
|
||||
}
|
||||
await stopCoturnIfStarted(coturnStarted);
|
||||
await stopLocalRelayIfStarted(relayStarted);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,9 @@
|
||||
"test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts",
|
||||
"bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts",
|
||||
"bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts",
|
||||
"bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts",
|
||||
"bench:latency-sweep": "deno run --env-file=.test.env -A --no-check bench-latency-sweep.ts",
|
||||
"bench:p2p-split-node": "deno run --env-file=.test.env -A --no-check bench-p2p-split-node.ts",
|
||||
"bench:item1": "bash ./bench-run-item1.sh",
|
||||
"bench:item1:full": "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh",
|
||||
"test:e2e-couchdb": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-couchdb.ts",
|
||||
|
||||
@@ -9,7 +9,7 @@ function sleep(ms: number): Promise<void> {
|
||||
}
|
||||
|
||||
async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise<void> {
|
||||
let timer: number | undefined;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const connPromise = Deno.connect({ hostname, port });
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
|
||||
@@ -76,8 +76,10 @@ export async function discoverPeer(
|
||||
}
|
||||
|
||||
export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
|
||||
if (!isLocalP2pRelay(relay)) return false;
|
||||
const shouldStart = isLocalP2pRelay(relay);
|
||||
if (shouldStart) {
|
||||
await startP2pRelay();
|
||||
}
|
||||
const endpoint = parseRelayEndpoint(relay);
|
||||
await waitForPort(endpoint.hostname, endpoint.port, {
|
||||
timeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000"),
|
||||
@@ -86,8 +88,10 @@ export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
|
||||
});
|
||||
// Docker proxy accepts TCP connections instantly before the container's internal process is fully ready.
|
||||
// Wait an additional few seconds to ensure strfry is actually accepting WebSockets.
|
||||
if (shouldStart) {
|
||||
await sleep(3000);
|
||||
return true;
|
||||
}
|
||||
return shouldStart;
|
||||
}
|
||||
|
||||
export async function stopLocalRelayIfStarted(started: boolean): Promise<void> {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
bench-results/
|
||||
@@ -0,0 +1,8 @@
|
||||
FROM alpine:3.22
|
||||
|
||||
RUN apk add --no-cache iproute2
|
||||
|
||||
COPY test/bench-network/netem-smoke.sh /usr/local/bin/livesync-netem-smoke
|
||||
RUN chmod +x /usr/local/bin/livesync-netem-smoke
|
||||
|
||||
CMD ["livesync-netem-smoke"]
|
||||
@@ -0,0 +1,35 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
FROM node:24-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl unzip python3 make g++ iproute2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV DENO_INSTALL=/usr/local
|
||||
RUN curl -fsSL https://deno.land/install.sh | sh
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
COPY src/apps/cli/package.json ./src/apps/cli/package.json
|
||||
COPY src/apps/webapp/package.json ./src/apps/webapp/package.json
|
||||
COPY src/apps/webpeer/package.json ./src/apps/webpeer/package.json
|
||||
RUN npm ci
|
||||
|
||||
COPY . .
|
||||
RUN npm run build -w self-hosted-livesync-cli
|
||||
|
||||
WORKDIR /workspace/src/apps/cli/testdeno
|
||||
|
||||
RUN deno cache --lock=deno.lock \
|
||||
bench-network-cases.ts \
|
||||
bench-latency-sweep.ts \
|
||||
bench-p2p-split-node.ts \
|
||||
bench-p2p.ts \
|
||||
bench-couchdb.ts
|
||||
|
||||
COPY test/bench-network/run-bench.sh /usr/local/bin/run-livesync-bench
|
||||
RUN chmod +x /usr/local/bin/run-livesync-bench
|
||||
|
||||
CMD ["run-livesync-bench"]
|
||||
@@ -0,0 +1,10 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
FROM alpine:3.22
|
||||
|
||||
RUN apk add --no-cache iproute2 socat
|
||||
|
||||
COPY test/bench-network/netem-tcp-shim.sh /usr/local/bin/livesync-netem-tcp-shim
|
||||
RUN chmod +x /usr/local/bin/livesync-netem-tcp-shim
|
||||
|
||||
CMD ["livesync-netem-tcp-shim"]
|
||||
@@ -0,0 +1,263 @@
|
||||
# Network benchmark package
|
||||
|
||||
This directory packages the CLI benchmark cases with Docker Compose. It is
|
||||
intended for reproducible local benchmark runs where CouchDB, the Nostr
|
||||
signalling relay, optional TURN, and the benchmark runner are fixed by the
|
||||
Compose file.
|
||||
|
||||
## Quick smoke run
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
docker compose -f test/bench-network/compose.yml run --rm bench-runner
|
||||
```
|
||||
|
||||
By default this runs:
|
||||
|
||||
- `couchdb-baseline`
|
||||
- `p2p-direct-local`
|
||||
|
||||
The dataset is intentionally small by default. Results are written to
|
||||
`test/bench-network/bench-results/`.
|
||||
|
||||
## GitHub Actions smoke run
|
||||
|
||||
`.github/workflows/cli-p2p-compose-smoke.yml` provides a manual
|
||||
`workflow_dispatch` smoke run for the same Compose package. It is intentionally
|
||||
not a required check yet, because WebRTC peer discovery can still be slow or
|
||||
environment-sensitive on GitHub-hosted runners. Keep the dataset small and use
|
||||
the uploaded JSON artefact to inspect whether failures are caused by peer
|
||||
discovery, synchronisation, CouchDB startup, or Docker networking.
|
||||
|
||||
## Select cases
|
||||
|
||||
```bash
|
||||
BENCH_CASES=couchdb-baseline,p2p-direct-local,p2p-user-turn \
|
||||
docker compose -f test/bench-network/compose.yml --profile turn run --rm bench-runner
|
||||
```
|
||||
|
||||
Available local cases:
|
||||
|
||||
- `couchdb-baseline`
|
||||
- `p2p-direct-local`
|
||||
- `couchdb-tethering-vpn-proxy`
|
||||
- `couchdb-netem-home-wifi`
|
||||
- `couchdb-netem-tethering-vpn`
|
||||
- `p2p-smartphone-vpn-direct`
|
||||
- `p2p-user-turn`
|
||||
|
||||
Set `BENCH_REPEAT_COUNT` to run each selected case more than once. Repeated
|
||||
results are written with suffixes such as `-r01`, `-r02`, and `-r03`, and the
|
||||
summary records the repeat index for each run.
|
||||
|
||||
`p2p-smartphone-vpn-direct` is a structural case name. When it is run inside
|
||||
this Compose package it is not a real smartphone tethering/VPN measurement; it
|
||||
uses the local Compose network. Use it only for wiring checks unless the runner
|
||||
is executed in an actual tethered/VPN environment.
|
||||
|
||||
## Comparison model
|
||||
|
||||
The primary local comparison is between a remote-database path and a direct P2P
|
||||
path:
|
||||
|
||||
| Case | Data path | What is measured | What is not measured |
|
||||
| ------------------ | ------------------------------------------- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------- |
|
||||
| `couchdb-baseline` | Device A -> CouchDB -> Device B | Two one-shot CLI synchronisation commands through a local HTTP latency proxy | Real WAN jitter, packet loss, bandwidth limits, VPN encapsulation, and server contention |
|
||||
| `p2p-direct-local` | Device A -> Device B after Nostr signalling | One CLI P2P synchronisation command over WebRTC DataChannel with TURN disabled | Public relay operation, mobile carrier behaviour, TURN relay throughput, and first-peer discovery latency |
|
||||
|
||||
Use the CouchDB result as the remote-store baseline and the P2P result as the
|
||||
direct-transfer comparison. The Nostr relay is used for signalling in the P2P
|
||||
case, but synchronised note content is transferred over the WebRTC DataChannel.
|
||||
The P2P result JSON records the selected WebRTC ICE candidate pair when the CLI
|
||||
can collect it from `RTCPeerConnection.getStats()`. Interpret P2P paths from
|
||||
the recorded candidate types rather than from TURN configuration alone. Do not
|
||||
report P2P runs as Tier 2 constrained-network measurements until host and
|
||||
client are captured under an equivalent shaped topology.
|
||||
|
||||
## Dataset and latency controls
|
||||
|
||||
```bash
|
||||
BENCH_MD_FILE_COUNT=100 \
|
||||
BENCH_MD_MIN_SIZE_BYTES=512 \
|
||||
BENCH_MD_MAX_SIZE_BYTES=2048 \
|
||||
BENCH_BIN_FILE_COUNT=25 \
|
||||
BENCH_BIN_SIZE_BYTES=8192 \
|
||||
BENCH_COUCHDB_RTT_MS=20 \
|
||||
BENCH_PEERS_TIMEOUT=60 \
|
||||
docker compose -f test/bench-network/compose.yml run --rm bench-runner
|
||||
```
|
||||
|
||||
The current CouchDB latency model is the existing HTTP proxy inside
|
||||
`bench-couchdb.ts`. It models a remote database path with additional request
|
||||
latency, but it does not model packet loss, jitter, MTU, bandwidth limits,
|
||||
bufferbloat, or VPN encapsulation.
|
||||
|
||||
For P2P runs, `BENCH_PEERS_TIMEOUT` is passed to `p2p-peers`. That command waits
|
||||
for the requested observation window before printing discovered peers, so the
|
||||
reported peer discovery command time should not be read as first-peer latency.
|
||||
|
||||
## Latency sweep
|
||||
|
||||
To run P2P once and CouchDB at several requested RTT values:
|
||||
|
||||
```bash
|
||||
BENCH_COMMAND=latency-sweep \
|
||||
BENCH_SWEEP_RTT_MS=20,50,100,150,300 \
|
||||
BENCH_MD_FILE_COUNT=100 \
|
||||
BENCH_MD_MIN_SIZE_BYTES=512 \
|
||||
BENCH_MD_MAX_SIZE_BYTES=2048 \
|
||||
BENCH_BIN_FILE_COUNT=25 \
|
||||
BENCH_BIN_SIZE_BYTES=8192 \
|
||||
BENCH_SYNC_TIMEOUT=300 \
|
||||
BENCH_PEERS_TIMEOUT=60 \
|
||||
docker compose -f test/bench-network/compose.yml run --rm bench-runner
|
||||
```
|
||||
|
||||
This sweep is useful for finding where the remote CouchDB path falls behind the
|
||||
local direct P2P path in the current HTTP-proxy latency model. It should not be
|
||||
presented as a full smartphone/VPN model.
|
||||
|
||||
## Network emulation smoke
|
||||
|
||||
The optional `netem` profile checks whether a Linux runner can apply traffic
|
||||
shaping inside a Compose-managed container. This is a fixture smoke test for a
|
||||
second-tier simulation design; it does not produce synchronisation performance
|
||||
results by itself.
|
||||
|
||||
```bash
|
||||
docker compose -f test/bench-network/compose.yml --profile netem run --rm netem-smoke
|
||||
```
|
||||
|
||||
The smoke writes `tc qdisc`, route, and interface details under
|
||||
`test/bench-network/bench-results/`. Profile parameters can be overridden:
|
||||
|
||||
```bash
|
||||
NETEM_PROFILE=tethering-vpn \
|
||||
NETEM_DELAY_MS=140 \
|
||||
NETEM_JITTER_MS=50 \
|
||||
NETEM_LOSS_PERCENT=1.0 \
|
||||
NETEM_BANDWIDTH_MBIT=10 \
|
||||
NETEM_MTU=1380 \
|
||||
docker compose -f test/bench-network/compose.yml --profile netem run --rm netem-smoke
|
||||
```
|
||||
|
||||
## Split-container P2P emulation
|
||||
|
||||
The optional `p2p-split` profile runs the P2P host and client in separate
|
||||
Compose services. Each service can apply `tc netem` to its own egress interface
|
||||
and the client result records the selected WebRTC ICE candidate pair.
|
||||
|
||||
```bash
|
||||
BENCH_MD_FILE_COUNT=2 \
|
||||
BENCH_BIN_FILE_COUNT=1 \
|
||||
BENCH_PEERS_TIMEOUT=10 \
|
||||
BENCH_SPLIT_RUN_ID="$(date -u +%Y%m%d%H%M%S)" \
|
||||
docker compose -f test/bench-network/compose.yml --profile p2p-split up \
|
||||
--abort-on-container-exit --exit-code-from p2p-split-client \
|
||||
p2p-split-host p2p-split-client
|
||||
```
|
||||
|
||||
By default this uses the `home-wifi` profile (`20 ms` delay, `5 ms` jitter,
|
||||
`0.1%` loss, `100 Mbit`, and `1500` MTU) on both P2P containers. Override the
|
||||
same `NETEM_*` variables used by the TCP shim to model a stricter profile.
|
||||
|
||||
```bash
|
||||
BENCH_MD_FILE_COUNT=100 \
|
||||
BENCH_MD_MIN_SIZE_BYTES=512 \
|
||||
BENCH_MD_MAX_SIZE_BYTES=2048 \
|
||||
BENCH_BIN_FILE_COUNT=25 \
|
||||
BENCH_BIN_SIZE_BYTES=8192 \
|
||||
BENCH_PEERS_TIMEOUT=60 \
|
||||
BENCH_SYNC_TIMEOUT=420 \
|
||||
BENCH_SPLIT_RUN_ID="$(date -u +%Y%m%d%H%M%S)" \
|
||||
BENCH_NETWORK_PROFILE=tethering-vpn \
|
||||
NETEM_PROFILE=tethering-vpn \
|
||||
NETEM_DELAY_MS=140 \
|
||||
NETEM_JITTER_MS=50 \
|
||||
NETEM_LOSS_PERCENT=1.0 \
|
||||
NETEM_BANDWIDTH_MBIT=10 \
|
||||
NETEM_MTU=1380 \
|
||||
docker compose -f test/bench-network/compose.yml --profile p2p-split up \
|
||||
--abort-on-container-exit --exit-code-from p2p-split-client \
|
||||
p2p-split-host p2p-split-client
|
||||
```
|
||||
|
||||
This is a Linux-only manual benchmark fixture, not a required pull-request CI
|
||||
job. It shapes each P2P container's egress path, including signalling traffic,
|
||||
and should be reported separately from the CouchDB TCP-shim measurements. The
|
||||
result JSON includes `ok: true` for completed runs; failed runs still write a
|
||||
summary with `ok: false` and a `failure` object before returning a non-zero
|
||||
exit code.
|
||||
|
||||
Remove the shared work volume between repeated manual runs when you do not use
|
||||
a unique `BENCH_SPLIT_RUN_ID`:
|
||||
|
||||
```bash
|
||||
docker compose -f test/bench-network/compose.yml --profile p2p-split down --volumes
|
||||
```
|
||||
|
||||
## P2P Signalling-Only Emulation
|
||||
|
||||
The optional `signalling-shim` profile shapes only the Nostr signalling relay
|
||||
path. The P2P host and client run in the benchmark runner as usual, and the
|
||||
configured relay URL points at a TCP netem shim in front of `nostr-relay`.
|
||||
This is the preferred fixture when evaluating the hypothesis that P2P avoids a
|
||||
constrained remote database data path while still depending on a signalling
|
||||
server for rendezvous.
|
||||
|
||||
```bash
|
||||
BENCH_CASES=p2p-signalling-netem-home-wifi \
|
||||
docker compose -f test/bench-network/compose.yml --profile signalling-shim run --rm \
|
||||
bench-runner-signalling-shim
|
||||
```
|
||||
|
||||
For a stricter signalling path:
|
||||
|
||||
```bash
|
||||
NETEM_PROFILE=tethering-vpn \
|
||||
NETEM_DELAY_MS=140 \
|
||||
NETEM_JITTER_MS=50 \
|
||||
NETEM_LOSS_PERCENT=1.0 \
|
||||
NETEM_BANDWIDTH_MBIT=10 \
|
||||
NETEM_MTU=1380 \
|
||||
BENCH_CASES=p2p-signalling-netem-tethering-vpn \
|
||||
docker compose -f test/bench-network/compose.yml --profile signalling-shim run --rm \
|
||||
bench-runner-signalling-shim
|
||||
```
|
||||
|
||||
Use this separately from `p2p-split`. The `p2p-split` profile shapes each peer's
|
||||
egress path, so it constrains both signalling and the selected WebRTC data
|
||||
path. The `signalling-shim` profile constrains only relay access, which keeps
|
||||
it focused on peer-to-signalling-server reachability rather than peer-to-peer
|
||||
note-data transfer.
|
||||
|
||||
## Shimmed CouchDB benchmark
|
||||
|
||||
The optional `shim` profile runs a CouchDB benchmark through a TCP forwarding
|
||||
container that applies `tc netem`. This is a manual Tier 2 synchronisation
|
||||
measurement path; it is intentionally separate from required pull-request CI.
|
||||
|
||||
```bash
|
||||
docker compose -f test/bench-network/compose.yml --profile shim run --rm bench-runner-shim
|
||||
```
|
||||
|
||||
The default profile is `home-wifi`. A smartphone/VPN-like profile can be
|
||||
requested by overriding both the shim parameters and the benchmark case:
|
||||
|
||||
```bash
|
||||
NETEM_PROFILE=tethering-vpn \
|
||||
NETEM_DELAY_MS=140 \
|
||||
NETEM_JITTER_MS=50 \
|
||||
NETEM_LOSS_PERCENT=1.0 \
|
||||
NETEM_BANDWIDTH_MBIT=10 \
|
||||
NETEM_MTU=1380 \
|
||||
BENCH_CASES=couchdb-netem-tethering-vpn \
|
||||
docker compose -f test/bench-network/compose.yml --profile shim run --rm bench-runner-shim
|
||||
```
|
||||
|
||||
The benchmark result records `simulationTier`, `networkProfile`, and
|
||||
`networkModel`. The shim also writes its applied `tc qdisc`, route, and
|
||||
interface state under `test/bench-network/bench-results/`.
|
||||
This shim currently measures the CouchDB path only. It does not shape or verify
|
||||
the WebRTC P2P data path.
|
||||
@@ -0,0 +1,329 @@
|
||||
services:
|
||||
couchdb:
|
||||
image: couchdb:3.5.0
|
||||
environment:
|
||||
COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin}
|
||||
COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword}
|
||||
COUCHDB_SINGLE_NODE: "true"
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"curl -fsS -u ${BENCH_COUCHDB_USER:-admin}:${BENCH_COUCHDB_PASSWORD:-testpassword} http://127.0.0.1:5984/_up >/dev/null",
|
||||
]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
nostr-relay:
|
||||
image: ghcr.io/hoytech/strfry:latest
|
||||
entrypoint: sh
|
||||
command:
|
||||
- -lc
|
||||
- |
|
||||
cat > /tmp/strfry.conf <<'EOF'
|
||||
db = "./strfry-db/"
|
||||
|
||||
relay {
|
||||
bind = "0.0.0.0"
|
||||
port = 7777
|
||||
nofiles = 65536
|
||||
|
||||
info {
|
||||
name = "livesync bench relay"
|
||||
description = "local relay for livesync compose benchmarks"
|
||||
}
|
||||
|
||||
maxWebsocketPayloadSize = 131072
|
||||
autoPingSeconds = 55
|
||||
|
||||
writePolicy {
|
||||
plugin = ""
|
||||
}
|
||||
}
|
||||
EOF
|
||||
exec /app/strfry --config /tmp/strfry.conf relay
|
||||
tmpfs:
|
||||
- /app/strfry-db:rw,size=256m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "nc -z 127.0.0.1 7777"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
coturn:
|
||||
image: coturn/coturn:latest
|
||||
command:
|
||||
- --log-file=stdout
|
||||
- --listening-port=3478
|
||||
- --user=${BENCH_TURN_USERNAME:-testuser}:${BENCH_TURN_CREDENTIAL:-testpass}
|
||||
- --realm=${BENCH_TURN_REALM:-livesync.test}
|
||||
profiles:
|
||||
- turn
|
||||
|
||||
bench-runner:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.runner
|
||||
depends_on:
|
||||
couchdb:
|
||||
condition: service_healthy
|
||||
nostr-relay:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BENCH_COMMAND: ${BENCH_COMMAND:-cases}
|
||||
BENCH_CASES: ${BENCH_CASES:-couchdb-baseline,p2p-direct-local}
|
||||
BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1}
|
||||
BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results
|
||||
BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results
|
||||
BENCH_SWEEP_RTT_MS: ${BENCH_SWEEP_RTT_MS:-20,50,100,150,300}
|
||||
BENCH_SWEEP_INCLUDE_P2P: ${BENCH_SWEEP_INCLUDE_P2P:-true}
|
||||
BENCH_COUCHDB_MANAGED: "false"
|
||||
BENCH_COUCHDB_BACKEND_URI: http://couchdb:5984
|
||||
BENCH_COUCHDB_URI: http://127.0.0.1:15989
|
||||
BENCH_COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin}
|
||||
BENCH_COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword}
|
||||
BENCH_RELAY: ws://nostr-relay:7777/
|
||||
BENCH_LOCAL_TURN_SERVERS: turn:coturn:3478
|
||||
BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20}
|
||||
BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512}
|
||||
BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
|
||||
BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
|
||||
BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
|
||||
BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-20}
|
||||
BENCH_TETHERING_VPN_RTT_MS: ${BENCH_TETHERING_VPN_RTT_MS:-120}
|
||||
BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
|
||||
BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
|
||||
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000}
|
||||
BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0}
|
||||
volumes:
|
||||
- ./bench-results:/workspace/src/apps/cli/testdeno/bench-results
|
||||
|
||||
couchdb-shim:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.shim
|
||||
profiles:
|
||||
- shim
|
||||
depends_on:
|
||||
couchdb:
|
||||
condition: service_healthy
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
environment:
|
||||
NETEM_PROFILE: ${NETEM_PROFILE:-home-wifi}
|
||||
NETEM_INTERFACE: ${NETEM_INTERFACE:-eth0}
|
||||
NETEM_DELAY_MS: ${NETEM_DELAY_MS:-20}
|
||||
NETEM_JITTER_MS: ${NETEM_JITTER_MS:-5}
|
||||
NETEM_LOSS_PERCENT: ${NETEM_LOSS_PERCENT:-0.1}
|
||||
NETEM_BANDWIDTH_MBIT: ${NETEM_BANDWIDTH_MBIT:-100}
|
||||
NETEM_MTU: ${NETEM_MTU:-1500}
|
||||
NETEM_RESULT_ROOT: /bench-results
|
||||
SHIM_LISTEN_PORT: 5984
|
||||
SHIM_TARGET_HOST: couchdb
|
||||
SHIM_TARGET_PORT: 5984
|
||||
volumes:
|
||||
- ./bench-results:/bench-results
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "nc -z 127.0.0.1 5984"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
bench-runner-shim:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.runner
|
||||
profiles:
|
||||
- shim
|
||||
depends_on:
|
||||
couchdb-shim:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BENCH_COMMAND: ${BENCH_COMMAND:-cases}
|
||||
BENCH_CASES: ${BENCH_CASES:-couchdb-netem-home-wifi}
|
||||
BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1}
|
||||
BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results
|
||||
BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results
|
||||
BENCH_COUCHDB_MANAGED: "false"
|
||||
BENCH_COUCHDB_BACKEND_URI: http://couchdb-shim:5984
|
||||
BENCH_SHIM_COUCHDB_URI: http://couchdb-shim:5984
|
||||
BENCH_COUCHDB_URI: http://127.0.0.1:15989
|
||||
BENCH_COUCHDB_USER: ${BENCH_COUCHDB_USER:-admin}
|
||||
BENCH_COUCHDB_PASSWORD: ${BENCH_COUCHDB_PASSWORD:-testpassword}
|
||||
BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20}
|
||||
BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512}
|
||||
BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
|
||||
BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
|
||||
BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
|
||||
BENCH_COUCHDB_RTT_MS: ${BENCH_COUCHDB_RTT_MS:-1}
|
||||
BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
|
||||
BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0}
|
||||
volumes:
|
||||
- ./bench-results:/workspace/src/apps/cli/testdeno/bench-results
|
||||
|
||||
p2p-signalling-shim:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.shim
|
||||
profiles:
|
||||
- signalling-shim
|
||||
depends_on:
|
||||
nostr-relay:
|
||||
condition: service_healthy
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
environment:
|
||||
NETEM_PROFILE: ${NETEM_PROFILE:-home-wifi}
|
||||
NETEM_INTERFACE: ${NETEM_INTERFACE:-eth0}
|
||||
NETEM_DELAY_MS: ${NETEM_DELAY_MS:-20}
|
||||
NETEM_JITTER_MS: ${NETEM_JITTER_MS:-5}
|
||||
NETEM_LOSS_PERCENT: ${NETEM_LOSS_PERCENT:-0.1}
|
||||
NETEM_BANDWIDTH_MBIT: ${NETEM_BANDWIDTH_MBIT:-100}
|
||||
NETEM_MTU: ${NETEM_MTU:-1500}
|
||||
NETEM_RESULT_ROOT: /bench-results
|
||||
SHIM_LISTEN_PORT: 7777
|
||||
SHIM_TARGET_HOST: nostr-relay
|
||||
SHIM_TARGET_PORT: 7777
|
||||
volumes:
|
||||
- ./bench-results:/bench-results
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "nc -z 127.0.0.1 7777"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
bench-runner-signalling-shim:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.runner
|
||||
profiles:
|
||||
- signalling-shim
|
||||
depends_on:
|
||||
p2p-signalling-shim:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BENCH_COMMAND: ${BENCH_COMMAND:-cases}
|
||||
BENCH_CASES: ${BENCH_CASES:-p2p-signalling-netem-home-wifi}
|
||||
BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1}
|
||||
BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results
|
||||
BENCH_SIGNAL_SHIM_RELAY: ws://p2p-signalling-shim:7777/
|
||||
BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20}
|
||||
BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512}
|
||||
BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
|
||||
BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
|
||||
BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
|
||||
BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
|
||||
BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
|
||||
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000}
|
||||
BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0}
|
||||
volumes:
|
||||
- ./bench-results:/workspace/src/apps/cli/testdeno/bench-results
|
||||
|
||||
p2p-split-host:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.runner
|
||||
profiles:
|
||||
- p2p-split
|
||||
depends_on:
|
||||
nostr-relay:
|
||||
condition: service_healthy
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
environment:
|
||||
BENCH_COMMAND: p2p-split-node
|
||||
BENCH_P2P_SPLIT_ROLE: host
|
||||
BENCH_SPLIT_RUN_ID: ${BENCH_SPLIT_RUN_ID:-bench-split-run}
|
||||
BENCH_SPLIT_WORK_ROOT: /p2p-work
|
||||
BENCH_SPLIT_RESULT_ROOT: /workspace/src/apps/cli/testdeno/bench-results
|
||||
BENCH_RELAY: ws://nostr-relay:7777/
|
||||
BENCH_APP_ID: ${BENCH_APP_ID:-self-hosted-livesync-cli-benchmark}
|
||||
BENCH_ROOM_ID: ${BENCH_ROOM_ID:-bench-split-room}
|
||||
BENCH_PASSPHRASE: ${BENCH_PASSPHRASE:-bench-split-passphrase}
|
||||
BENCH_TURN_SERVERS: ${BENCH_TURN_SERVERS:-}
|
||||
BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20}
|
||||
BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512}
|
||||
BENCH_MD_MAX_SIZE_BYTES: ${BENCH_MD_MAX_SIZE_BYTES:-2048}
|
||||
BENCH_BIN_FILE_COUNT: ${BENCH_BIN_FILE_COUNT:-5}
|
||||
BENCH_BIN_SIZE_BYTES: ${BENCH_BIN_SIZE_BYTES:-8192}
|
||||
BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
|
||||
BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
|
||||
BENCH_NETEM_ENABLED: ${BENCH_NETEM_ENABLED:-1}
|
||||
BENCH_NETWORK_PROFILE: ${BENCH_NETWORK_PROFILE:-home-wifi}
|
||||
NETEM_PROFILE: ${NETEM_PROFILE:-home-wifi}
|
||||
NETEM_INTERFACE: ${NETEM_INTERFACE:-eth0}
|
||||
NETEM_DELAY_MS: ${NETEM_DELAY_MS:-20}
|
||||
NETEM_JITTER_MS: ${NETEM_JITTER_MS:-5}
|
||||
NETEM_LOSS_PERCENT: ${NETEM_LOSS_PERCENT:-0.1}
|
||||
NETEM_BANDWIDTH_MBIT: ${NETEM_BANDWIDTH_MBIT:-100}
|
||||
NETEM_MTU: ${NETEM_MTU:-1500}
|
||||
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000}
|
||||
LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0}
|
||||
volumes:
|
||||
- p2p-split-work:/p2p-work
|
||||
- ./bench-results:/workspace/src/apps/cli/testdeno/bench-results
|
||||
|
||||
p2p-split-client:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.runner
|
||||
profiles:
|
||||
- p2p-split
|
||||
depends_on:
|
||||
nostr-relay:
|
||||
condition: service_healthy
|
||||
p2p-split-host:
|
||||
condition: service_started
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
environment:
|
||||
BENCH_COMMAND: p2p-split-node
|
||||
BENCH_P2P_SPLIT_ROLE: client
|
||||
BENCH_SPLIT_RUN_ID: ${BENCH_SPLIT_RUN_ID:-bench-split-run}
|
||||
BENCH_SPLIT_WORK_ROOT: /p2p-work
|
||||
BENCH_SPLIT_RESULT_ROOT: /workspace/src/apps/cli/testdeno/bench-results
|
||||
BENCH_RELAY: ws://nostr-relay:7777/
|
||||
BENCH_APP_ID: ${BENCH_APP_ID:-self-hosted-livesync-cli-benchmark}
|
||||
BENCH_ROOM_ID: ${BENCH_ROOM_ID:-bench-split-room}
|
||||
BENCH_PASSPHRASE: ${BENCH_PASSPHRASE:-bench-split-passphrase}
|
||||
BENCH_TURN_SERVERS: ${BENCH_TURN_SERVERS:-}
|
||||
BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300}
|
||||
BENCH_PEERS_TIMEOUT: ${BENCH_PEERS_TIMEOUT:-60}
|
||||
BENCH_NETEM_ENABLED: ${BENCH_NETEM_ENABLED:-1}
|
||||
BENCH_NETWORK_PROFILE: ${BENCH_NETWORK_PROFILE:-home-wifi}
|
||||
NETEM_PROFILE: ${NETEM_PROFILE:-home-wifi}
|
||||
NETEM_INTERFACE: ${NETEM_INTERFACE:-eth0}
|
||||
NETEM_DELAY_MS: ${NETEM_DELAY_MS:-20}
|
||||
NETEM_JITTER_MS: ${NETEM_JITTER_MS:-5}
|
||||
NETEM_LOSS_PERCENT: ${NETEM_LOSS_PERCENT:-0.1}
|
||||
NETEM_BANDWIDTH_MBIT: ${NETEM_BANDWIDTH_MBIT:-100}
|
||||
NETEM_MTU: ${NETEM_MTU:-1500}
|
||||
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000}
|
||||
LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0}
|
||||
volumes:
|
||||
- p2p-split-work:/p2p-work
|
||||
- ./bench-results:/workspace/src/apps/cli/testdeno/bench-results
|
||||
|
||||
netem-smoke:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: test/bench-network/Dockerfile.netem
|
||||
profiles:
|
||||
- netem
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
environment:
|
||||
NETEM_PROFILE: ${NETEM_PROFILE:-home-wifi}
|
||||
NETEM_INTERFACE: ${NETEM_INTERFACE:-eth0}
|
||||
NETEM_DELAY_MS: ${NETEM_DELAY_MS:-20}
|
||||
NETEM_JITTER_MS: ${NETEM_JITTER_MS:-5}
|
||||
NETEM_LOSS_PERCENT: ${NETEM_LOSS_PERCENT:-0.1}
|
||||
NETEM_BANDWIDTH_MBIT: ${NETEM_BANDWIDTH_MBIT:-100}
|
||||
NETEM_MTU: ${NETEM_MTU:-1500}
|
||||
NETEM_RESULT_ROOT: /bench-results
|
||||
volumes:
|
||||
- ./bench-results:/bench-results
|
||||
|
||||
volumes:
|
||||
p2p-split-work:
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
profile="${NETEM_PROFILE:-home-wifi}"
|
||||
iface="${NETEM_INTERFACE:-eth0}"
|
||||
delay_ms="${NETEM_DELAY_MS:-20}"
|
||||
jitter_ms="${NETEM_JITTER_MS:-5}"
|
||||
loss_percent="${NETEM_LOSS_PERCENT:-0.1}"
|
||||
bandwidth_mbit="${NETEM_BANDWIDTH_MBIT:-100}"
|
||||
mtu="${NETEM_MTU:-1500}"
|
||||
out_root="${NETEM_RESULT_ROOT:-/bench-results}"
|
||||
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
||||
out_dir="${out_root}/netem-smoke-${timestamp}"
|
||||
out_file="${out_dir}/summary.json"
|
||||
|
||||
mkdir -p "$out_dir"
|
||||
|
||||
if ! ip link show "$iface" >/dev/null 2>&1; then
|
||||
echo "Network interface '$iface' was not found" >&2
|
||||
ip addr >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
ip link set dev "$iface" mtu "$mtu"
|
||||
tc qdisc del dev "$iface" root >/dev/null 2>&1 || true
|
||||
tc qdisc add dev "$iface" root netem \
|
||||
delay "${delay_ms}ms" "${jitter_ms}ms" \
|
||||
loss "${loss_percent}%" \
|
||||
rate "${bandwidth_mbit}mbit"
|
||||
|
||||
json_lines() {
|
||||
awk '
|
||||
{
|
||||
gsub(/\\/, "\\\\");
|
||||
gsub(/"/, "\\\"");
|
||||
printf "%s \"%s\"", (NR == 1 ? "" : ",\n"), $0;
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
ip_addr="$(ip addr show "$iface" | json_lines)"
|
||||
ip_route="$(ip route | json_lines)"
|
||||
tc_qdisc="$(tc qdisc show dev "$iface" | json_lines)"
|
||||
|
||||
cat > "$out_file" <<EOF
|
||||
{
|
||||
"simulationTier": 2,
|
||||
"mode": "netem-smoke",
|
||||
"profile": "$profile",
|
||||
"interface": "$iface",
|
||||
"netem": {
|
||||
"delayMs": $delay_ms,
|
||||
"jitterMs": $jitter_ms,
|
||||
"lossPercent": $loss_percent,
|
||||
"bandwidthMbit": $bandwidth_mbit,
|
||||
"mtu": $mtu
|
||||
},
|
||||
"ipAddr": [
|
||||
$ip_addr
|
||||
],
|
||||
"ipRoute": [
|
||||
$ip_route
|
||||
],
|
||||
"tcQdisc": [
|
||||
$tc_qdisc
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
cat "$out_file"
|
||||
echo "[netem-smoke] result file: $out_file"
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
profile="${NETEM_PROFILE:-home-wifi}"
|
||||
iface="${NETEM_INTERFACE:-eth0}"
|
||||
delay_ms="${NETEM_DELAY_MS:-20}"
|
||||
jitter_ms="${NETEM_JITTER_MS:-5}"
|
||||
loss_percent="${NETEM_LOSS_PERCENT:-0.1}"
|
||||
bandwidth_mbit="${NETEM_BANDWIDTH_MBIT:-100}"
|
||||
mtu="${NETEM_MTU:-1500}"
|
||||
listen_port="${SHIM_LISTEN_PORT:-5984}"
|
||||
target_host="${SHIM_TARGET_HOST:-couchdb}"
|
||||
target_port="${SHIM_TARGET_PORT:-5984}"
|
||||
out_root="${NETEM_RESULT_ROOT:-/bench-results}"
|
||||
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
||||
out_dir="${out_root}/netem-shim-${profile}-${timestamp}"
|
||||
out_file="${out_dir}/summary.json"
|
||||
|
||||
json_lines() {
|
||||
awk '
|
||||
{
|
||||
gsub(/\\/, "\\\\");
|
||||
gsub(/"/, "\\\"");
|
||||
printf "%s \"%s\"", (NR == 1 ? "" : ",\n"), $0;
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
mkdir -p "$out_dir"
|
||||
|
||||
if ! ip link show "$iface" >/dev/null 2>&1; then
|
||||
echo "Network interface '$iface' was not found" >&2
|
||||
ip addr >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
ip link set dev "$iface" mtu "$mtu"
|
||||
tc qdisc del dev "$iface" root >/dev/null 2>&1 || true
|
||||
tc qdisc add dev "$iface" root netem \
|
||||
delay "${delay_ms}ms" "${jitter_ms}ms" \
|
||||
loss "${loss_percent}%" \
|
||||
rate "${bandwidth_mbit}mbit"
|
||||
|
||||
ip_addr="$(ip addr show "$iface" | json_lines)"
|
||||
ip_route="$(ip route | json_lines)"
|
||||
tc_qdisc="$(tc qdisc show dev "$iface" | json_lines)"
|
||||
|
||||
cat > "$out_file" <<EOF
|
||||
{
|
||||
"simulationTier": 2,
|
||||
"mode": "netem-tcp-shim",
|
||||
"profile": "$profile",
|
||||
"interface": "$iface",
|
||||
"listenPort": $listen_port,
|
||||
"targetHost": "$target_host",
|
||||
"targetPort": $target_port,
|
||||
"netem": {
|
||||
"delayMs": $delay_ms,
|
||||
"jitterMs": $jitter_ms,
|
||||
"lossPercent": $loss_percent,
|
||||
"bandwidthMbit": $bandwidth_mbit,
|
||||
"mtu": $mtu
|
||||
},
|
||||
"ipAddr": [
|
||||
$ip_addr
|
||||
],
|
||||
"ipRoute": [
|
||||
$ip_route
|
||||
],
|
||||
"tcQdisc": [
|
||||
$tc_qdisc
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
cat "$out_file"
|
||||
echo "[netem-shim] forwarding 0.0.0.0:${listen_port} to ${target_host}:${target_port}"
|
||||
echo "[netem-shim] result file: $out_file"
|
||||
|
||||
exec socat "TCP-LISTEN:${listen_port},fork,reuseaddr" "TCP:${target_host}:${target_port}"
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
case "${BENCH_COMMAND:-cases}" in
|
||||
cases)
|
||||
exec deno task bench:cases
|
||||
;;
|
||||
latency-sweep)
|
||||
exec deno task bench:latency-sweep
|
||||
;;
|
||||
p2p-split-node)
|
||||
exec deno task bench:p2p-split-node
|
||||
;;
|
||||
*)
|
||||
echo "Unknown BENCH_COMMAND: ${BENCH_COMMAND}" >&2
|
||||
echo "Expected one of: cases, latency-sweep, p2p-split-node" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user