Add scoped CLI benchmark metadata

src/apps/cli/testdeno/bench-p2p.ts

@@ -1,5 +1,9 @@
 import { TempDir } from "./helpers/temp.ts";
-import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
+import {
+    applyP2pSettings,
+    applyP2pTestTweaks,
+    initSettingsFile,
+} from "./helpers/settings.ts";
 import { startCliInBackground } from "./helpers/backgroundCli.ts";
 import {
     discoverPeer,
@@ -9,7 +13,10 @@ import {
     stopLocalRelayIfStarted,
 } from "./helpers/p2p.ts";
 import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
-import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
+import {
+    createDeterministicDataset,
+    type DatasetEntry,
+} from "./helpers/dataset.ts";
 
 type BenchmarkConfig = {
     caseName: string;
@@ -31,6 +38,8 @@ type BenchmarkConfig = {
     networkProfile: string;
     networkModel: string;
     candidatePathVerification: string;
+    measurementScope: string;
+    limitations: string[];
 };
 
 type P2PConnectionStats = {
@@ -83,6 +92,30 @@ function readEnvNumber(name: string, fallback: number): number {
     return parsed;
 }
 
+function readEnvStringArray(name: string, fallback: string[]): string[] {
+    const raw = Deno.env.get(name)?.trim();
+    if (!raw) {
+        return fallback;
+    }
+
+    try {
+        const parsed = JSON.parse(raw);
+        if (
+            Array.isArray(parsed) &&
+            parsed.every((item) => typeof item === "string")
+        ) {
+            return parsed;
+        }
+    } catch {
+        // Fall through to comma-separated parsing for hand-written invocations.
+    }
+
+    return raw
+        .split("|")
+        .map((item) => item.trim())
+        .filter((item) => item.length > 0);
+}
+
 function nowMs(): number {
     return performance.now();
 }
@@ -107,23 +140,45 @@ function buildConfig(): BenchmarkConfig {
     return {
         caseName: readEnvString("BENCH_CASE", "p2p-direct-local"),
         relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"),
-        appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
+        appId: readEnvString(
+            "BENCH_APP_ID",
+            "self-hosted-livesync-cli-benchmark",
+        ),
         roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`),
         passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
         turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
         datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
         datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
         mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
-        mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)),
-        mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)),
+        mdMinSizeBytes: Math.floor(
+            readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024),
+        ),
+        mdMaxSizeBytes: Math.floor(
+            readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024),
+        ),
         binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)),
-        binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
+        binSizeBytes: Math.floor(
+            readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024),
+        ),
         peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20),
         syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
         simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
         networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "local-direct"),
-        networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-runner-webrtc"),
-        candidatePathVerification: readEnvString("BENCH_P2P_CANDIDATE_PATH_VERIFICATION", "not-collected"),
+        networkModel: readEnvString(
+            "BENCH_NETWORK_MODEL",
+            "local-runner-webrtc",
+        ),
+        candidatePathVerification: readEnvString(
+            "BENCH_P2P_CANDIDATE_PATH_VERIFICATION",
+            "not-collected",
+        ),
+        measurementScope: readEnvString(
+            "BENCH_MEASUREMENT_SCOPE",
+            "One CLI P2P synchronisation phase over WebRTC DataChannel after signalling.",
+        ),
+        limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
+            "This benchmark result is scoped to the configured dataset, network model, and selected ICE path.",
+        ]),
     };
 }
 
@@ -152,7 +207,9 @@ function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
     return [...unique.values()];
 }
 
-async function readLatestP2PConnectionStats(statsPath: string): Promise<P2PConnectionStats | undefined> {
+async function readLatestP2PConnectionStats(
+    statsPath: string,
+): Promise<P2PConnectionStats | undefined> {
     try {
         const text = await Deno.readTextFile(statsPath);
         const lines = text
@@ -200,7 +257,7 @@ async function main(): Promise<void> {
                 config.appId,
                 config.relay,
                 "~.*",
-                config.turnServers
+                config.turnServers,
             ),
             applyP2pSettings(
                 clientSettings,
@@ -209,13 +266,21 @@ async function main(): Promise<void> {
                 config.appId,
                 config.relay,
                 "~.*",
-                config.turnServers
+                config.turnServers,
             ),
         ]);
 
         await Promise.all([
-            applyP2pTestTweaks(hostSettings, "p2p-bench-host", config.passphrase),
-            applyP2pTestTweaks(clientSettings, "p2p-bench-client", config.passphrase),
+            applyP2pTestTweaks(
+                hostSettings,
+                "p2p-bench-host",
+                config.passphrase,
+            ),
+            applyP2pTestTweaks(
+                clientSettings,
+                "p2p-bench-client",
+                config.passphrase,
+            ),
         ]);
 
         const seedFiles = await createDeterministicDataset({
@@ -233,15 +298,25 @@ async function main(): Promise<void> {
         await runCliOrFail(hostVault, "--settings", hostSettings, "mirror");
         const mirrorElapsed = nowMs() - mirrorStart;
 
-        const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
+        const host = startCliInBackground(
+            hostVault,
+            "--settings",
+            hostSettings,
+            "p2p-host",
+        );
         try {
             const hostReadyStart = nowMs();
             await host.waitUntilContains("P2P host is running", 20000);
             const hostReadyElapsed = nowMs() - hostReadyStart;
 
             const peerDiscoveryCommandStart = nowMs();
-            const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds);
-            const peerDiscoveryCommandElapsed = nowMs() - peerDiscoveryCommandStart;
+            const peer = await discoverPeer(
+                clientVault,
+                clientSettings,
+                config.peersTimeoutSeconds,
+            );
+            const peerDiscoveryCommandElapsed = nowMs() -
+                peerDiscoveryCommandStart;
 
             const syncStart = nowMs();
             await runCliOrFail(
@@ -250,22 +325,33 @@ async function main(): Promise<void> {
                 clientSettings,
                 "p2p-sync",
                 peer.id,
-                String(config.syncTimeoutSeconds)
+                String(config.syncTimeoutSeconds),
             );
             const syncElapsed = nowMs() - syncStart;
 
             const sampleFiles = pickSampleFiles(seedFiles.entries);
             for (const sample of sampleFiles) {
-                const pulledPath = workDir.join(`pulled-${sample.relativePath.replaceAll("/", "_")}`);
-                await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath);
+                const pulledPath = workDir.join(
+                    `pulled-${sample.relativePath.replaceAll("/", "_")}`,
+                );
+                await runCliOrFail(
+                    clientVault,
+                    "--settings",
+                    clientSettings,
+                    "pull",
+                    sample.relativePath,
+                    pulledPath,
+                );
                 await assertFilesEqual(
                     sample.absolutePath,
                     pulledPath,
-                    `sample file mismatch after sync: ${sample.relativePath}`
+                    `sample file mismatch after sync: ${sample.relativePath}`,
                 );
             }
 
-            const p2pConnectionStats = await readLatestP2PConnectionStats(p2pStatsPath);
+            const p2pConnectionStats = await readLatestP2PConnectionStats(
+                p2pStatsPath,
+            );
             const result = {
                 caseName: config.caseName,
                 mode: "p2p-cli-benchmark",
@@ -275,15 +361,19 @@ async function main(): Promise<void> {
                 simulationTier: config.simulationTier,
                 networkProfile: config.networkProfile,
                 networkModel: config.networkModel,
-                p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
-                p2pCandidatePathVerification: p2pConnectionStats?.candidatePathCollected
-                    ? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
-                    : config.candidatePathVerification,
+                measurementScope: config.measurementScope,
+                limitations: config.limitations,
+                p2pCandidatePathVerified:
+                    p2pConnectionStats?.candidatePathCollected === true,
+                p2pCandidatePathVerification:
+                    p2pConnectionStats?.candidatePathCollected
+                        ? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
+                        : config.candidatePathVerification,
                 p2pCandidatePathNote: p2pConnectionStats?.candidatePathCollected
                     ? "The selected ICE candidate pair was collected by the CLI benchmark. Interpret the path from the candidate types; do not infer TURN use from configuration alone."
                     : config.turnServers.trim().length > 0
-                      ? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run."
-                      : "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.",
+                    ? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run."
+                    : "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.",
                 p2pConnectionStats,
                 appId: config.appId,
                 roomId: config.roomId,
@@ -298,25 +388,39 @@ async function main(): Promise<void> {
                 mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
                 hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
                 peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
-                peerDiscoveryCommandElapsedMs: Number(peerDiscoveryCommandElapsed.toFixed(1)),
+                peerDiscoveryCommandElapsedMs: Number(
+                    peerDiscoveryCommandElapsed.toFixed(1),
+                ),
                 peerDiscoveryNote:
                     "p2p-peers waits for the requested timeout before printing discovered peers, so this is command duration, not first-peer latency.",
                 syncElapsedMs: Number(syncElapsed.toFixed(1)),
-                throughputBytesPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2)),
-                throughputMiBPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024).toFixed(4)),
+                throughputBytesPerSec: Number(
+                    (seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2),
+                ),
+                throughputMiBPerSec: Number(
+                    (seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024)
+                        .toFixed(
+                            4,
+                        ),
+                ),
             };
 
             if (resultPath) {
-                await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2));
+                await Deno.writeTextFile(
+                    resultPath,
+                    JSON.stringify(result, null, 2),
+                );
             }
 
             console.log(JSON.stringify(result, null, 2));
             console.error(
-                `[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes(
-                    seedFiles.totalBytes
-                )}) in ${formatMs(mirrorElapsed)}, ` +
+                `[Benchmark] mirrored ${seedFiles.totalFiles} files (${
+                    formatBytes(
+                        seedFiles.totalBytes,
+                    )
+                }) in ${formatMs(mirrorElapsed)}, ` +
                     `synced in ${formatMs(syncElapsed)} ` +
-                    `(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
+                    `(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`,
             );
         } finally {
             await host.stop();