From df390ac4561d0d4e4118efde6fd5291dac34aa4b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:02:11 +0000 Subject: [PATCH 01/15] test: fix deno test helpers --- src/apps/cli/testdeno/helpers/cli.ts | 72 +++++++++++-- src/apps/cli/testdeno/helpers/dataset.ts | 123 +++++++++++++++++++++++ src/apps/cli/testdeno/helpers/docker.ts | 58 +++++++---- 3 files changed, 228 insertions(+), 25 deletions(-) create mode 100644 src/apps/cli/testdeno/helpers/dataset.ts diff --git a/src/apps/cli/testdeno/helpers/cli.ts b/src/apps/cli/testdeno/helpers/cli.ts index 8c78b52..d2279cb 100644 --- a/src/apps/cli/testdeno/helpers/cli.ts +++ b/src/apps/cli/testdeno/helpers/cli.ts @@ -39,27 +39,73 @@ function concatChunks(chunks: Uint8Array[]): Uint8Array { return out; } +function formatTeeCommand(args: string[]): string { + return ["node", CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" "); +} + +function createLineTeeWriter( + pid: number, + streamName: "stdout" | "stderr", + writer: (chunk: Uint8Array) => void +): { write: (chunk: Uint8Array) => void; close: () => void } { + const enc = new TextEncoder(); + const dec = new TextDecoder(); + let pending = ""; + let headerWritten = false; + const emitLine = (line: string) => { + if (!headerWritten) { + writer(enc.encode(`[CLI tee pid=${pid}:${streamName}]\n`)); + headerWritten = true; + } + writer(enc.encode(`[CLI tee pid=${pid}:${streamName}] ${line}\n`)); + }; + + const flush = (final = false) => { + let index = pending.indexOf("\n"); + while (index >= 0) { + const line = pending.slice(0, index).replace(/\r$/, ""); + pending = pending.slice(index + 1); + emitLine(line); + index = pending.indexOf("\n"); + } + if (final && pending.length > 0) { + emitLine(pending.replace(/\r$/, "")); + pending = ""; + } + }; + + return { + write(chunk: Uint8Array) { + pending += dec.decode(chunk, { stream: true }); + flush(false); + }, + close() { + pending += dec.decode(); + flush(true); + }, + }; +} + async function collectStream( stream: ReadableStream, - teeTarget: WritableStream | null + teeTarget: { write: (chunk: Uint8Array) => void; close: () => void } | null ): Promise { const reader = stream.getReader(); const chunks: Uint8Array[] = []; - const writer = teeTarget?.getWriter(); try { while (true) { const { done, value } = await reader.read(); if (done) break; if (value) { chunks.push(value); - if (writer) { - await writer.write(value); + if (teeTarget) { + teeTarget.write(value); } } } } finally { - if (writer) { - writer.releaseLock(); + if (teeTarget) { + teeTarget.close(); } reader.releaseLock(); } @@ -76,8 +122,18 @@ async function runNodeCommand(args: string[], stdinData?: Uint8Array): Promise Deno.stdout.writeSync(chunk)) : null + ); + const stderrPromise = collectStream( + child.stderr, + TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => Deno.stderr.writeSync(chunk)) : null + ); if (stdinData) { const w = child.stdin.getWriter(); diff --git a/src/apps/cli/testdeno/helpers/dataset.ts b/src/apps/cli/testdeno/helpers/dataset.ts new file mode 100644 index 0000000..ad9af47 --- /dev/null +++ b/src/apps/cli/testdeno/helpers/dataset.ts @@ -0,0 +1,123 @@ +export type DeterministicDatasetConfig = { + rootDir: string; + datasetDirName: string; + seed: string; + mdCount: number; + mdMinSizeBytes: number; + mdMaxSizeBytes: number; + binCount: number; + binSizeBytes: number; +}; + +export type DatasetEntry = { + kind: "md" | "bin"; + relativePath: string; + absolutePath: string; + size: number; +}; + +export type DeterministicDataset = { + rootDir: string; + datasetDirName: string; + seed: string; + entries: DatasetEntry[]; + totalFiles: number; + totalBytes: number; + mdCount: number; + binCount: number; +}; + +function fnv1a32(input: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < input.length; i++) { + hash ^= input.charCodeAt(i) & 0xff; + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +function createXorshift32(seed: number): () => number { + let state = seed >>> 0; + if (state === 0) { + state = 0x9e3779b9; + } + return () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return state >>> 0; + }; +} + +function createTextBytes(size: number, fileIndex: number, seed: string): Uint8Array { + const template = + `# Bench file ${fileIndex}\n` + + `seed: ${seed}\n` + + "lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n"; + + const templateBytes = new TextEncoder().encode(template); + const out = new Uint8Array(size); + for (let i = 0; i < size; i++) { + out[i] = templateBytes[i % templateBytes.length]; + } + return out; +} + +function toPath(rootDir: string, relativePath: string): string { + return `${rootDir}/${relativePath}`; +} + +export async function createDeterministicDataset(config: DeterministicDatasetConfig): Promise { + if (config.mdCount < 0 || config.binCount < 0) { + throw new Error("mdCount and binCount must be non-negative"); + } + if (config.mdMinSizeBytes <= 0 || config.mdMaxSizeBytes <= 0 || config.binSizeBytes <= 0) { + throw new Error("all size values must be positive"); + } + if (config.mdMinSizeBytes > config.mdMaxSizeBytes) { + throw new Error("mdMinSizeBytes must be <= mdMaxSizeBytes"); + } + + const datasetRoot = toPath(config.rootDir, config.datasetDirName); + const mdDir = `${datasetRoot}/md`; + const binDir = `${datasetRoot}/bin`; + await Deno.mkdir(mdDir, { recursive: true }); + await Deno.mkdir(binDir, { recursive: true }); + + const nextRandom = createXorshift32(fnv1a32(config.seed)); + const mdRange = config.mdMaxSizeBytes - config.mdMinSizeBytes + 1; + const entries: DatasetEntry[] = []; + + for (let index = 0; index < config.mdCount; index++) { + const size = config.mdMinSizeBytes + (nextRandom() % mdRange); + const relativePath = `${config.datasetDirName}/md/file-${String(index).padStart(4, "0")}.md`; + const absolutePath = toPath(config.rootDir, relativePath); + const body = createTextBytes(size, index, config.seed); + await Deno.writeFile(absolutePath, body); + entries.push({ kind: "md", relativePath, absolutePath, size }); + } + + for (let index = 0; index < config.binCount; index++) { + const size = config.binSizeBytes; + const relativePath = `${config.datasetDirName}/bin/file-${String(index).padStart(4, "0")}.bin`; + const absolutePath = toPath(config.rootDir, relativePath); + const body = new Uint8Array(size); + for (let i = 0; i < size; i++) { + body[i] = nextRandom() & 0xff; + } + await Deno.writeFile(absolutePath, body); + entries.push({ kind: "bin", relativePath, absolutePath, size }); + } + + const totalBytes = entries.reduce((sum, e) => sum + e.size, 0); + return { + rootDir: config.rootDir, + datasetDirName: config.datasetDirName, + seed: config.seed, + entries, + totalFiles: entries.length, + totalBytes, + mdCount: config.mdCount, + binCount: config.binCount, + }; +} diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/src/apps/cli/testdeno/helpers/docker.ts index 5ecea1f..d04a1f9 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/src/apps/cli/testdeno/helpers/docker.ts @@ -27,29 +27,53 @@ function parseCommand(command: string): { bin: string; prefix: string[] } { return { bin: parts[0], prefix: parts.slice(1) }; } -async function runCommand(bin: string, args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { - const cmd = new Deno.Command(bin, { - args, - stdin: "null", - stdout: "piped", - stderr: "piped", - }); +async function collectStream( + stream: ReadableStream, + teeTarget: ((chunk: Uint8Array) => void) | null +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; try { - const { code, stdout, stderr } = await cmd.output(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + chunks.push(value); + if (teeTarget) { + teeTarget(value); + } + } + } finally { + reader.releaseLock(); + } + + const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + +async function runCommand(bin: string, args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { + try { + const child = new Deno.Command(bin, { + args, + stdin: "null", + stdout: "piped", + stderr: "piped", + }).spawn(); + const stdoutPromise = collectStream(child.stdout, DOCKER_TEE ? (chunk) => Deno.stdout.writeSync(chunk) : null); + const stderrPromise = collectStream(child.stderr, DOCKER_TEE ? (chunk) => Deno.stderr.writeSync(chunk) : null); + const [status, stdout, stderr] = await Promise.all([child.status, stdoutPromise, stderrPromise]); const dec = new TextDecoder(); const result = { - code, + code: status.code, stdout: dec.decode(stdout), stderr: dec.decode(stderr), }; - if (DOCKER_TEE) { - if (result.stdout.trim().length > 0) { - console.log(`[docker:${bin}] ${result.stdout.trimEnd()}`); - } - if (result.stderr.trim().length > 0) { - console.error(`[docker:${bin}] ${result.stderr.trimEnd()}`); - } - } return result; } catch (err) { if (err instanceof Deno.errors.NotFound) { From cc3c992b1deaab14348f13729c655f42a6e5d114 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:05:44 +0000 Subject: [PATCH 02/15] cli: add large-file-test and benchmark between couchdb and p2p --- src/apps/cli/.gitignore | 3 +- src/apps/cli/testdeno/bench-couchdb.ts | 310 +++++++++++++++++++++++ src/apps/cli/testdeno/bench-p2p.ts | 219 ++++++++++++++++ src/apps/cli/testdeno/bench-run-item1.sh | 45 ++++ src/apps/cli/testdeno/deno.json | 4 + 5 files changed, 580 insertions(+), 1 deletion(-) create mode 100644 src/apps/cli/testdeno/bench-couchdb.ts create mode 100644 src/apps/cli/testdeno/bench-p2p.ts create mode 100644 src/apps/cli/testdeno/bench-run-item1.sh diff --git a/src/apps/cli/.gitignore b/src/apps/cli/.gitignore index 24987cb..e775b86 100644 --- a/src/apps/cli/.gitignore +++ b/src/apps/cli/.gitignore @@ -5,4 +5,5 @@ test/test-init.local.sh node_modules .*.json *.env -!.test.env \ No newline at end of file +!.test.env +bench-results \ No newline at end of file diff --git a/src/apps/cli/testdeno/bench-couchdb.ts b/src/apps/cli/testdeno/bench-couchdb.ts new file mode 100644 index 0000000..678b236 --- /dev/null +++ b/src/apps/cli/testdeno/bench-couchdb.ts @@ -0,0 +1,310 @@ +import { TempDir } from "./helpers/temp.ts"; +import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; +import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; +import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; +import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts"; + +type BenchmarkConfig = { + couchdbBackendUri: string; + couchdbProxyUri: string; + couchdbUser: string; + couchdbPassword: string; + couchdbDbname: string; + datasetDirName: string; + datasetSeed: string; + mdFileCount: number; + mdMinSizeBytes: number; + mdMaxSizeBytes: number; + binFileCount: number; + binSizeBytes: number; + syncTimeoutSeconds: number; + requestedRttMs: number; + passphrase: string; + encrypt: boolean; +}; + +function readEnvString(name: string, fallback: string): string { + const value = Deno.env.get(name)?.trim(); + return value && value.length > 0 ? value : fallback; +} + +function readEnvNumber(name: string, fallback: number): number { + const raw = Deno.env.get(name); + if (raw === undefined || raw.trim() === "") { + return fallback; + } + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive number, got '${raw}'`); + } + return parsed; +} + +function readEnvBool(name: string, fallback: boolean): boolean { + const raw = Deno.env.get(name); + if (raw === undefined || raw.trim() === "") { + return fallback; + } + return /^(1|true|yes|on)$/i.test(raw.trim()); +} + +function nowMs(): number { + return performance.now(); +} + +function formatMs(value: number): string { + return `${value.toFixed(1)} ms`; +} + +function formatBytes(value: number): string { + if (value < 1024) { + return `${value} B`; + } + const kib = value / 1024; + if (kib < 1024) { + return `${kib.toFixed(1)} KiB`; + } + return `${(kib / 1024).toFixed(1)} MiB`; +} + +function buildConfig(): BenchmarkConfig { + return { + couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"), + couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"), + couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")), + couchdbPassword: readEnvString("BENCH_COUCHDB_PASSWORD", readEnvString("password", "password")), + couchdbDbname: readEnvString("BENCH_COUCHDB_DBNAME", `bench-couchdb-${Date.now()}`), + datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"), + datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), + mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)), + mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)), + mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)), + binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)), + binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)), + syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240), + requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)), + passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), + encrypt: readEnvBool("BENCH_ENCRYPT", true), + }; +} + +function readOptionalResultPath(): string | undefined { + const raw = Deno.env.get("BENCH_RESULT_JSON")?.trim(); + if (!raw) { + return undefined; + } + return raw; +} + +function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] { + if (entries.length === 0) { + return []; + } + const md = entries.find((e) => e.kind === "md"); + const bin = entries.find((e) => e.kind === "bin"); + const middle = entries[Math.floor(entries.length / 2)]; + const last = entries[entries.length - 1]; + const unique = new Map(); + for (const entry of [md, bin, middle, last]) { + if (entry) { + unique.set(entry.relativePath, entry); + } + } + return [...unique.values()]; +} + +type ProxyHandle = { + stop: () => Promise; + applied: boolean; + note: string; +}; + +function startCouchdbProxy(options: { + backendUri: string; + proxyUri: string; + requestedRttMs: number; +}): ProxyHandle { + const backend = new URL(options.backendUri); + const proxy = new URL(options.proxyUri); + const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2)); + const controller = new AbortController(); + + const listener = Deno.serve( + { + hostname: proxy.hostname, + port: Number(proxy.port), + signal: controller.signal, + onError(error) { + console.error(`[Proxy] ${String(error)}`); + return new Response("proxy error", { status: 502 }); + }, + }, + async (request) => { + await new Promise((resolve) => setTimeout(resolve, halfDelayMs)); + + const targetUrl = new URL(request.url); + targetUrl.protocol = backend.protocol; + targetUrl.host = backend.host; + + const headers = new Headers(request.headers); + headers.delete("host"); + headers.delete("content-length"); + + let requestBody: ArrayBuffer | undefined; + if (request.method !== "GET" && request.method !== "HEAD") { + try { + requestBody = await request.arrayBuffer(); + } catch { + requestBody = undefined; + } + } + + const upstream = await fetch(targetUrl, { + method: request.method, + headers, + body: requestBody, + redirect: "manual", + }); + + const responseHeaders = new Headers(upstream.headers); + responseHeaders.delete("content-length"); + const responseBody = await upstream.arrayBuffer(); + + return new Response(responseBody, { + status: upstream.status, + statusText: upstream.statusText, + headers: responseHeaders, + }); + } + ); + + return { + applied: true, + note: `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`, + stop: async () => { + controller.abort(); + await listener.finished.catch(() => {}); + }, + }; +} + +async function main(): Promise { + const config = buildConfig(); + const resultPath = readOptionalResultPath(); + + await using workDir = await TempDir.create("livesync-cli-couchdb-bench"); + const vaultA = workDir.join("vault-a"); + const vaultB = workDir.join("vault-b"); + const settingsA = workDir.join("settings-a.json"); + const settingsB = workDir.join("settings-b.json"); + await Deno.mkdir(vaultA, { recursive: true }); + await Deno.mkdir(vaultB, { recursive: true }); + + await initSettingsFile(settingsA); + await initSettingsFile(settingsB); + + await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname); + + const proxy = startCouchdbProxy({ + backendUri: config.couchdbBackendUri, + proxyUri: config.couchdbProxyUri, + requestedRttMs: config.requestedRttMs, + }); + + try { + await Promise.all([ + applyRemoteSyncSettings(settingsA, { + remoteType: "COUCHDB", + couchdbUri: config.couchdbProxyUri, + couchdbUser: config.couchdbUser, + couchdbPassword: config.couchdbPassword, + couchdbDbname: config.couchdbDbname, + encrypt: config.encrypt, + passphrase: config.passphrase, + }), + applyRemoteSyncSettings(settingsB, { + remoteType: "COUCHDB", + couchdbUri: config.couchdbProxyUri, + couchdbUser: config.couchdbUser, + couchdbPassword: config.couchdbPassword, + couchdbDbname: config.couchdbDbname, + encrypt: config.encrypt, + passphrase: config.passphrase, + }), + ]); + + const seedFiles = await createDeterministicDataset({ + rootDir: vaultA, + datasetDirName: config.datasetDirName, + seed: config.datasetSeed, + mdCount: config.mdFileCount, + mdMinSizeBytes: config.mdMinSizeBytes, + mdMaxSizeBytes: config.mdMaxSizeBytes, + binCount: config.binFileCount, + binSizeBytes: config.binSizeBytes, + }); + + const mirrorStart = nowMs(); + await runCliOrFail(vaultA, "--settings", settingsA, "mirror"); + const mirrorElapsed = nowMs() - mirrorStart; + + const syncAStart = nowMs(); + await runCliOrFail(vaultA, "--settings", settingsA, "sync"); + const syncAElapsed = nowMs() - syncAStart; + + const syncBStart = nowMs(); + await runCliOrFail(vaultB, "--settings", settingsB, "sync"); + const syncBElapsed = nowMs() - syncBStart; + + const sampleFiles = pickSampleFiles(seedFiles.entries); + for (const sample of sampleFiles) { + const pulledPath = workDir.join(`pulled-${sample.relativePath.split("/").join("_")}`); + await runCliOrFail(vaultB, "--settings", settingsB, "pull", sample.relativePath, pulledPath); + await assertFilesEqual(sample.absolutePath, pulledPath, `sample file mismatch after CouchDB sync: ${sample.relativePath}`); + } + + const result = { + mode: "couchdb-cli-benchmark", + couchdbBackendUri: config.couchdbBackendUri, + couchdbProxyUri: config.couchdbProxyUri, + couchdbDbname: config.couchdbDbname, + rttRequestedMs: config.requestedRttMs, + proxyApplied: proxy.applied, + proxyNote: proxy.note, + datasetSeed: config.datasetSeed, + datasetDirName: config.datasetDirName, + totalFiles: seedFiles.totalFiles, + totalBytes: seedFiles.totalBytes, + mdFileCount: seedFiles.mdCount, + binFileCount: seedFiles.binCount, + mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), + syncAElapsedMs: Number(syncAElapsed.toFixed(1)), + syncBElapsedMs: Number(syncBElapsed.toFixed(1)), + totalSyncElapsedMs: Number((syncAElapsed + syncBElapsed).toFixed(1)), + throughputBytesPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000)).toFixed(2)), + throughputMiBPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / 1024 / 1024).toFixed(4)), + }; + + if (resultPath) { + await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2)); + } + + console.log(JSON.stringify(result, null, 2)); + console.error( + `[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${formatBytes(seedFiles.totalBytes)}) in ${formatMs( + mirrorElapsed + )}, synced in ${formatMs(syncAElapsed + syncBElapsed)} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)` + ); + } finally { + await proxy.stop(); + await stopCouchdb().catch(() => {}); + } +} + +if (import.meta.main) { + main().catch((error) => { + console.error(`[Fatal Error]`, error); + Deno.exit(1); + }); +} \ No newline at end of file diff --git a/src/apps/cli/testdeno/bench-p2p.ts b/src/apps/cli/testdeno/bench-p2p.ts new file mode 100644 index 0000000..40be30a --- /dev/null +++ b/src/apps/cli/testdeno/bench-p2p.ts @@ -0,0 +1,219 @@ +import { TempDir } from "./helpers/temp.ts"; +import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; +import { startCliInBackground } from "./helpers/backgroundCli.ts"; +import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts"; +import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; +import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts"; + +type BenchmarkConfig = { + relay: string; + appId: string; + roomId: string; + passphrase: string; + datasetDirName: string; + datasetSeed: string; + mdFileCount: number; + mdMinSizeBytes: number; + mdMaxSizeBytes: number; + binFileCount: number; + binSizeBytes: number; + peersTimeoutSeconds: number; + syncTimeoutSeconds: number; +}; + +function readEnvString(name: string, fallback: string): string { + const value = Deno.env.get(name)?.trim(); + return value && value.length > 0 ? value : fallback; +} + +function readEnvNumber(name: string, fallback: number): number { + const raw = Deno.env.get(name); + if (raw === undefined || raw.trim() === "") { + return fallback; + } + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive number, got '${raw}'`); + } + return parsed; +} + +function nowMs(): number { + return performance.now(); +} + +function formatMs(value: number): string { + return `${value.toFixed(1)} ms`; +} + +function formatBytes(value: number): string { + if (value < 1024) { + return `${value} B`; + } + const kib = value / 1024; + if (kib < 1024) { + return `${kib.toFixed(1)} KiB`; + } + const mib = kib / 1024; + return `${mib.toFixed(1)} MiB`; +} + +function buildConfig(): BenchmarkConfig { + return { + relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"), + appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"), + roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`), + passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`), + datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"), + datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"), + mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)), + mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)), + mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)), + binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)), + binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)), + peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20), + syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240), + }; +} + +function readOptionalResultPath(): string | undefined { + const raw = Deno.env.get("BENCH_RESULT_JSON")?.trim(); + if (!raw) { + return undefined; + } + return raw; +} + +function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] { + if (entries.length === 0) { + return []; + } + const md = entries.find((e) => e.kind === "md"); + const bin = entries.find((e) => e.kind === "bin"); + const middle = entries[Math.floor(entries.length / 2)]; + const last = entries[entries.length - 1]; + const unique = new Map(); + for (const entry of [md, bin, middle, last]) { + if (entry) { + unique.set(entry.relativePath, entry); + } + } + return [...unique.values()]; +} + +async function main(): Promise { + const config = buildConfig(); + const resultPath = readOptionalResultPath(); + + const relayStarted = await maybeStartLocalRelay(config.relay); + await using workDir = await TempDir.create("livesync-cli-p2p-bench"); + + const hostVault = workDir.join("vault-host"); + const clientVault = workDir.join("vault-client"); + const hostSettings = workDir.join("settings-host.json"); + const clientSettings = workDir.join("settings-client.json"); + + await Promise.all([ + Deno.mkdir(hostVault, { recursive: true }), + Deno.mkdir(clientVault, { recursive: true }), + initSettingsFile(hostSettings), + initSettingsFile(clientSettings), + ]); + + await Promise.all([ + applyP2pSettings(hostSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"), + applyP2pSettings(clientSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"), + ]); + + await Promise.all([ + applyP2pTestTweaks(hostSettings, "p2p-bench-host", config.passphrase), + applyP2pTestTweaks(clientSettings, "p2p-bench-client", config.passphrase), + ]); + + const seedFiles = await createDeterministicDataset({ + rootDir: hostVault, + datasetDirName: config.datasetDirName, + seed: config.datasetSeed, + mdCount: config.mdFileCount, + mdMinSizeBytes: config.mdMinSizeBytes, + mdMaxSizeBytes: config.mdMaxSizeBytes, + binCount: config.binFileCount, + binSizeBytes: config.binSizeBytes, + }); + + const mirrorStart = nowMs(); + await runCliOrFail(hostVault, "--settings", hostSettings, "mirror"); + const mirrorElapsed = nowMs() - mirrorStart; + + const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host"); + try { + const hostReadyStart = nowMs(); + await host.waitUntilContains("P2P host is running", 20000); + const hostReadyElapsed = nowMs() - hostReadyStart; + + const peerDiscoveryStart = nowMs(); + const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds); + const peerDiscoveryElapsed = nowMs() - peerDiscoveryStart; + + const syncStart = nowMs(); + await runCliOrFail( + clientVault, + "--settings", + clientSettings, + "p2p-sync", + peer.id, + String(config.syncTimeoutSeconds) + ); + const syncElapsed = nowMs() - syncStart; + + const sampleFiles = pickSampleFiles(seedFiles.entries); + for (const sample of sampleFiles) { + const pulledPath = workDir.join(`pulled-${sample.relativePath.replaceAll("/", "_")}`); + await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath); + await assertFilesEqual(sample.absolutePath, pulledPath, `sample file mismatch after sync: ${sample.relativePath}`); + } + + const result = { + mode: "p2p-cli-benchmark", + relay: config.relay, + appId: config.appId, + roomId: config.roomId, + datasetSeed: config.datasetSeed, + datasetDirName: config.datasetDirName, + peerId: peer.id, + peerName: peer.name, + totalFiles: seedFiles.totalFiles, + totalBytes: seedFiles.totalBytes, + mdFileCount: seedFiles.mdCount, + binFileCount: seedFiles.binCount, + mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)), + hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)), + peerDiscoveryElapsedMs: Number(peerDiscoveryElapsed.toFixed(1)), + syncElapsedMs: Number(syncElapsed.toFixed(1)), + throughputBytesPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2)), + throughputMiBPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024).toFixed(4)), + }; + + if (resultPath) { + await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2)); + } + + console.log(JSON.stringify(result, null, 2)); + console.error( + `[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes(seedFiles.totalBytes)}) in ${formatMs(mirrorElapsed)}, ` + + `synced in ${formatMs(syncElapsed)} ` + + `(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)` + ); + } finally { + await host.stop(); + await stopLocalRelayIfStarted(relayStarted); + } +} + +if (import.meta.main) { + main().catch((error) => { + console.error(`[Fatal Error]`, error); + Deno.exit(1); + }); +} \ No newline at end of file diff --git a/src/apps/cli/testdeno/bench-run-item1.sh b/src/apps/cli/testdeno/bench-run-item1.sh new file mode 100644 index 0000000..53d2998 --- /dev/null +++ b/src/apps/cli/testdeno/bench-run-item1.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESULTS_ROOT="${SCRIPT_DIR}/bench-results" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +OUT_DIR="${RESULTS_ROOT}/${TIMESTAMP}" + +mkdir -p "${OUT_DIR}" + +echo "[bench-wrapper] output directory: ${OUT_DIR}" + +echo "[bench-wrapper] running p2p benchmark" +( + cd "${SCRIPT_DIR}" + BENCH_RESULT_JSON="${OUT_DIR}/p2p.json" deno task bench:p2p +) + +echo "[bench-wrapper] running couchdb benchmark with RTT ${BENCH_COUCHDB_RTT_MS:-default} ms (emulating HTTP network latency)" +( + cd "${SCRIPT_DIR}" + BENCH_RESULT_JSON="${OUT_DIR}/couchdb.json" deno task bench:couchdb +) + +cat > "${OUT_DIR}/README.txt" < Date: Fri, 22 May 2026 03:19:48 +0000 Subject: [PATCH 03/15] test: add port ready, container cleanup --- src/apps/cli/testdeno/helpers/cli.ts | 4 +- src/apps/cli/testdeno/helpers/docker.ts | 68 ++++++++++++++++++++--- src/apps/cli/testdeno/helpers/net.ts | 49 +++++++++++++++++ src/apps/cli/testdeno/helpers/p2p.ts | 72 +++++++++++++++++++------ 4 files changed, 171 insertions(+), 22 deletions(-) create mode 100644 src/apps/cli/testdeno/helpers/net.ts diff --git a/src/apps/cli/testdeno/helpers/cli.ts b/src/apps/cli/testdeno/helpers/cli.ts index d2279cb..a7e3a42 100644 --- a/src/apps/cli/testdeno/helpers/cli.ts +++ b/src/apps/cli/testdeno/helpers/cli.ts @@ -123,7 +123,9 @@ async function runNodeCommand(args: string[], stdinData?: Uint8Array): Promise | null = null; const DOCKER_TEE = Deno.env.get("LIVESYNC_DOCKER_TEE") === "1" || Deno.env.get("LIVESYNC_TEST_TEE") === "1"; +const trackedContainers = new Set(); +const CLEANUP_SIGNALS: Deno.Signal[] = ["SIGINT", "SIGTERM"]; +let signalCleanupHandlersInstalled = false; +let signalCleanupInProgress = false; // --------------------------------------------------------------------------- // Low-level docker wrapper @@ -183,6 +187,55 @@ async function dockerOrFail(...args: string[]): Promise { return r.stdout; } +async function stopAndRemoveContainer(container: string): Promise { + await docker("stop", container).catch(() => {}); + await docker("rm", container).catch(() => {}); +} + +async function cleanupTrackedContainers(reason: string): Promise { + const names = [...trackedContainers]; + if (names.length === 0) return; + + console.warn(`[WARN] cleaning up tracked containers on ${reason}: ${names.join(", ")}`); + for (const container of names.reverse()) { + await stopAndRemoveContainer(container); + trackedContainers.delete(container); + } +} + +async function handleSignalCleanup(signal: Deno.Signal): Promise { + if (signalCleanupInProgress) return; + signalCleanupInProgress = true; + try { + await cleanupTrackedContainers(`signal ${signal}`); + } finally { + Deno.exit(signal === "SIGINT" ? 130 : 143); + } +} + +function ensureSignalCleanupHandlers(): void { + if (signalCleanupHandlersInstalled) return; + signalCleanupHandlersInstalled = true; + for (const signal of CLEANUP_SIGNALS) { + try { + Deno.addSignalListener(signal, () => { + void handleSignalCleanup(signal); + }); + } catch { + // Unsupported signal on this platform. + } + } +} + +function trackContainer(container: string): void { + ensureSignalCleanupHandlers(); + trackedContainers.add(container); +} + +function untrackContainer(container: string): void { + trackedContainers.delete(container); +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -259,8 +312,8 @@ const MINIO_IMAGE = "minio/minio"; const MINIO_MC_IMAGE = "minio/mc"; export async function stopCouchdb(): Promise { - await docker("stop", COUCHDB_CONTAINER); - await docker("rm", COUCHDB_CONTAINER); + await stopAndRemoveContainer(COUCHDB_CONTAINER); + untrackContainer(COUCHDB_CONTAINER); } /** @@ -289,6 +342,7 @@ export async function startCouchdb(couchdbUri: string, user: string, password: s "COUCHDB_SINGLE_NODE=y", COUCHDB_IMAGE ); + trackContainer(COUCHDB_CONTAINER); console.log("[INFO] initialising CouchDB"); await initCouchdb(couchdbUri, user, password); @@ -389,8 +443,8 @@ function shQuote(value: string): string { } export async function stopMinio(): Promise { - await docker("stop", MINIO_CONTAINER); - await docker("rm", MINIO_CONTAINER); + await stopAndRemoveContainer(MINIO_CONTAINER); + untrackContainer(MINIO_CONTAINER); } async function initMinioBucket( @@ -470,6 +524,7 @@ export async function startMinio( "--console-address", ":9001" ); + trackContainer(MINIO_CONTAINER); console.log(`[INFO] initialising MinIO test bucket: ${bucket}`); let initialised = false; @@ -517,8 +572,8 @@ EOF exec /app/strfry --config /tmp/strfry.conf relay`; export async function stopP2pRelay(): Promise { - await docker("stop", P2P_RELAY_CONTAINER); - await docker("rm", P2P_RELAY_CONTAINER); + await stopAndRemoveContainer(P2P_RELAY_CONTAINER); + untrackContainer(P2P_RELAY_CONTAINER); } /** @@ -547,6 +602,7 @@ export async function startP2pRelay(): Promise { "-lc", STRFRY_BOOTSTRAP_SH ); + trackContainer(P2P_RELAY_CONTAINER); } export function isLocalP2pRelay(relayUrl: string): boolean { diff --git a/src/apps/cli/testdeno/helpers/net.ts b/src/apps/cli/testdeno/helpers/net.ts new file mode 100644 index 0000000..362c70d --- /dev/null +++ b/src/apps/cli/testdeno/helpers/net.ts @@ -0,0 +1,49 @@ +type WaitForPortOptions = { + timeoutMs?: number; + intervalMs?: number; + connectTimeoutMs?: number; +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise { + let timer: number | undefined; + try { + const connPromise = Deno.connect({ hostname, port }); + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`connect timeout after ${timeoutMs}ms`)), timeoutMs); + }); + const conn = await Promise.race([connPromise, timeoutPromise]); + conn.close(); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } +} + +export async function waitForPort(hostname: string, port: number, options: WaitForPortOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 15000; + const intervalMs = options.intervalMs ?? 250; + const connectTimeoutMs = options.connectTimeoutMs ?? 1000; + + const started = Date.now(); + let lastError: unknown; + + while (Date.now() - started < timeoutMs) { + try { + await connectWithTimeout(hostname, port, connectTimeoutMs); + return; + } catch (error) { + lastError = error; + await sleep(intervalMs); + } + } + + throw new Error( + `Port ${hostname}:${port} did not become ready within ${timeoutMs}ms` + + (lastError ? ` (last error: ${String(lastError)})` : "") + ); +} diff --git a/src/apps/cli/testdeno/helpers/p2p.ts b/src/apps/cli/testdeno/helpers/p2p.ts index 28741c4..efd8f7f 100644 --- a/src/apps/cli/testdeno/helpers/p2p.ts +++ b/src/apps/cli/testdeno/helpers/p2p.ts @@ -1,11 +1,26 @@ import { runCli } from "./cli.ts"; import { isLocalP2pRelay, startP2pRelay, stopP2pRelay } from "./docker.ts"; +import { waitForPort } from "./net.ts"; export type PeerEntry = { id: string; name: string; }; +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseRelayEndpoint(relay: string): { hostname: string; port: number } { + const url = new URL(relay); + const port = url.port ? Number(url.port) : url.protocol === "ws:" ? 80 : url.protocol === "wss:" ? 443 : NaN; + if (!Number.isFinite(port)) { + throw new Error(`Unsupported relay URL: ${relay}`); + } + const hostname = url.hostname === "localhost" ? "127.0.0.1" : url.hostname; + return { hostname, port }; +} + export function parsePeerLines(output: string): PeerEntry[] { return output .split(/\r?\n/) @@ -20,28 +35,55 @@ export async function discoverPeer( timeoutSeconds: number, targetPeer?: string ): Promise { - const result = await runCli(vaultDir, "--settings", settingsFile, "p2p-peers", String(timeoutSeconds)); - if (result.code !== 0) { - throw new Error(`p2p-peers failed\n${result.combined}`); - } - const peers = parsePeerLines(result.stdout); - if (targetPeer) { - const matched = peers.find((peer) => peer.id === targetPeer || peer.name === targetPeer); - if (matched) return matched; - } - if (peers.length === 0) { - const fallback = result.combined.match(/Advertisement from\s+([^\s]+)/); - if (fallback?.[1]) { - return { id: fallback[1], name: fallback[1] }; + const retries = Math.max(0, Number(Deno.env.get("LIVESYNC_P2P_PEERS_RETRY") ?? "3")); + let lastCombined = ""; + + for (let attempt = 0; attempt <= retries; attempt++) { + const result = await runCli(vaultDir, "--settings", settingsFile, "p2p-peers", String(timeoutSeconds)); + lastCombined = result.combined; + + if (result.code === 0) { + const peers = parsePeerLines(result.stdout); + if (targetPeer) { + const matched = peers.find((peer) => peer.id === targetPeer || peer.name === targetPeer); + if (matched) return matched; + } + if (peers.length > 0) { + return peers[0]; + } + + const fallback = result.combined.match(/Advertisement from\s+([^\s]+)/); + if (fallback?.[1]) { + return { id: fallback[1], name: fallback[1] }; + } } - throw new Error(`No peers discovered\n${result.combined}`); + + if (attempt < retries) { + const waitMs = 400 * (attempt + 1); + console.warn( + `[WARN] p2p-peers returned no usable peers, retrying (${attempt + 1}/${retries}) in ${waitMs}ms` + ); + await sleep(waitMs); + continue; + } + + throw new Error( + result.code !== 0 ? `p2p-peers failed\n${result.combined}` : `No peers discovered\n${result.combined}` + ); } - return peers[0]; + + throw new Error(`No peers discovered\n${lastCombined}`); } export async function maybeStartLocalRelay(relay: string): Promise { if (!isLocalP2pRelay(relay)) return false; await startP2pRelay(); + const endpoint = parseRelayEndpoint(relay); + await waitForPort(endpoint.hostname, endpoint.port, { + timeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000"), + intervalMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_INTERVAL_MS") ?? "250"), + connectTimeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_CONNECT_TIMEOUT_MS") ?? "1000"), + }); return true; } From c1f41910c4d75ab95c91371df25fd55c5b8dd9ea Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:20:11 +0000 Subject: [PATCH 04/15] test: add actions / caching --- .github/workflows/cli-deno-tests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 9230910..5bfd940 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -55,6 +55,8 @@ jobs: needs: prepare runs-on: ubuntu-latest timeout-minutes: 60 + env: + DENO_DIR: ~/.cache/deno strategy: fail-fast: false matrix: @@ -70,12 +72,21 @@ jobs: with: node-version: '24.x' cache: 'npm' + cache-dependency-path: package-lock.json - name: Setup Deno uses: denoland/setup-deno@v2 with: deno-version: v2.x + - name: Cache Deno dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/deno + key: ${{ runner.os }}-deno-${{ hashFiles('src/apps/cli/testdeno/deno.lock', 'src/apps/cli/testdeno/deno.json') }} + restore-keys: | + ${{ runner.os }}-deno- + - name: Install dependencies run: npm ci From 5fd85c71ca97a7b1db8e41fa3865f9e391be43ad Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:20:28 +0000 Subject: [PATCH 05/15] test: chore: prettify --- src/apps/cli/testdeno/bench-couchdb.ts | 18 ++++++++++-------- src/apps/cli/testdeno/bench-p2p.ts | 8 ++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/apps/cli/testdeno/bench-couchdb.ts b/src/apps/cli/testdeno/bench-couchdb.ts index 678b236..6ed6df7 100644 --- a/src/apps/cli/testdeno/bench-couchdb.ts +++ b/src/apps/cli/testdeno/bench-couchdb.ts @@ -120,11 +120,7 @@ type ProxyHandle = { note: string; }; -function startCouchdbProxy(options: { - backendUri: string; - proxyUri: string; - requestedRttMs: number; -}): ProxyHandle { +function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requestedRttMs: number }): ProxyHandle { const backend = new URL(options.backendUri); const proxy = new URL(options.proxyUri); const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2)); @@ -261,7 +257,11 @@ async function main(): Promise { for (const sample of sampleFiles) { const pulledPath = workDir.join(`pulled-${sample.relativePath.split("/").join("_")}`); await runCliOrFail(vaultB, "--settings", settingsB, "pull", sample.relativePath, pulledPath); - await assertFilesEqual(sample.absolutePath, pulledPath, `sample file mismatch after CouchDB sync: ${sample.relativePath}`); + await assertFilesEqual( + sample.absolutePath, + pulledPath, + `sample file mismatch after CouchDB sync: ${sample.relativePath}` + ); } const result = { @@ -283,7 +283,9 @@ async function main(): Promise { syncBElapsedMs: Number(syncBElapsed.toFixed(1)), totalSyncElapsedMs: Number((syncAElapsed + syncBElapsed).toFixed(1)), throughputBytesPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000)).toFixed(2)), - throughputMiBPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / 1024 / 1024).toFixed(4)), + throughputMiBPerSec: Number( + (seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / 1024 / 1024).toFixed(4) + ), }; if (resultPath) { @@ -307,4 +309,4 @@ if (import.meta.main) { console.error(`[Fatal Error]`, error); Deno.exit(1); }); -} \ No newline at end of file +} diff --git a/src/apps/cli/testdeno/bench-p2p.ts b/src/apps/cli/testdeno/bench-p2p.ts index 40be30a..cbc920f 100644 --- a/src/apps/cli/testdeno/bench-p2p.ts +++ b/src/apps/cli/testdeno/bench-p2p.ts @@ -171,7 +171,11 @@ async function main(): Promise { for (const sample of sampleFiles) { const pulledPath = workDir.join(`pulled-${sample.relativePath.replaceAll("/", "_")}`); await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath); - await assertFilesEqual(sample.absolutePath, pulledPath, `sample file mismatch after sync: ${sample.relativePath}`); + await assertFilesEqual( + sample.absolutePath, + pulledPath, + `sample file mismatch after sync: ${sample.relativePath}` + ); } const result = { @@ -216,4 +220,4 @@ if (import.meta.main) { console.error(`[Fatal Error]`, error); Deno.exit(1); }); -} \ No newline at end of file +} From 486fd15c6087c0bc128bc48864bf08543c8c7921 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:46:56 +0000 Subject: [PATCH 06/15] fix resouce handling --- src/apps/cli/testdeno/helpers/docker.ts | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/src/apps/cli/testdeno/helpers/docker.ts index 94d53e7..e9c91ea 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/src/apps/cli/testdeno/helpers/docker.ts @@ -18,6 +18,7 @@ const trackedContainers = new Set(); const CLEANUP_SIGNALS: Deno.Signal[] = ["SIGINT", "SIGTERM"]; let signalCleanupHandlersInstalled = false; let signalCleanupInProgress = false; +const signalCleanupHandlers = new Map void>(); // --------------------------------------------------------------------------- // Low-level docker wrapper @@ -217,16 +218,31 @@ function ensureSignalCleanupHandlers(): void { if (signalCleanupHandlersInstalled) return; signalCleanupHandlersInstalled = true; for (const signal of CLEANUP_SIGNALS) { + const listener = () => { + void handleSignalCleanup(signal); + }; try { - Deno.addSignalListener(signal, () => { - void handleSignalCleanup(signal); - }); + Deno.addSignalListener(signal, listener); + signalCleanupHandlers.set(signal, listener); } catch { // Unsupported signal on this platform. } } } +function removeSignalCleanupHandlers(): void { + if (!signalCleanupHandlersInstalled) return; + for (const [signal, listener] of signalCleanupHandlers) { + try { + Deno.removeSignalListener(signal, listener); + } catch { + // Ignore if already removed or unsupported. + } + } + signalCleanupHandlers.clear(); + signalCleanupHandlersInstalled = false; +} + function trackContainer(container: string): void { ensureSignalCleanupHandlers(); trackedContainers.add(container); @@ -234,6 +250,9 @@ function trackContainer(container: string): void { function untrackContainer(container: string): void { trackedContainers.delete(container); + if (trackedContainers.size === 0) { + removeSignalCleanupHandlers(); + } } function sleep(ms: number): Promise { From 8948bf2803d13456841f821c06287b65606e018c Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:48:02 +0000 Subject: [PATCH 07/15] test cli:p2p use nonce for peername --- .../testdeno/test-p2p-peers-local-relay.ts | 10 +++++--- src/apps/cli/testdeno/test-p2p-sync.ts | 9 +++++--- .../testdeno/test-p2p-three-nodes-conflict.ts | 23 +++++++++++++------ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts b/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts index f73fe35..c87c8a9 100644 --- a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts +++ b/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts @@ -9,6 +9,9 @@ Deno.test("p2p-peers: discovers host through local relay", async () => { const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; const timeoutSeconds = Number(Deno.env.get("TIMEOUT_SECONDS") ?? "8"); + const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; + const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; + const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; await using workDir = await TempDir.create("livesync-cli-p2p-peers-local-relay"); const hostVault = workDir.join("vault-host"); @@ -24,15 +27,16 @@ Deno.test("p2p-peers: discovers host through local relay", async () => { await initSettingsFile(clientSettings); await applyP2pSettings(hostSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); await applyP2pSettings(clientSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); - await applyP2pTestTweaks(hostSettings, "p2p-host", passphrase); - await applyP2pTestTweaks(clientSettings, "p2p-client", passphrase); + await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase); + await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase); const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host"); try { await host.waitUntilContains("P2P host is running", 20000); - const peer = await discoverPeer(clientVault, clientSettings, timeoutSeconds); + const peer = await discoverPeer(clientVault, clientSettings, timeoutSeconds, hostPeerName); assert(peer.id.length > 0); assert(peer.name.length > 0); + assert(peer.name === hostPeerName, `expected peer '${hostPeerName}', got '${peer.name}'`); } finally { await host.stop(); } diff --git a/src/apps/cli/testdeno/test-p2p-sync.ts b/src/apps/cli/testdeno/test-p2p-sync.ts index 970b4e9..2a50745 100644 --- a/src/apps/cli/testdeno/test-p2p-sync.ts +++ b/src/apps/cli/testdeno/test-p2p-sync.ts @@ -11,6 +11,9 @@ Deno.test("p2p-sync: discovers peer and completes sync", async () => { const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "12"); const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "15"); + const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; + const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; + const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; await using workDir = await TempDir.create("livesync-cli-p2p-sync"); const hostVault = workDir.join("vault-host"); @@ -26,8 +29,8 @@ Deno.test("p2p-sync: discovers peer and completes sync", async () => { await initSettingsFile(clientSettings); await applyP2pSettings(hostSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); await applyP2pSettings(clientSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); - await applyP2pTestTweaks(hostSettings, "p2p-host", passphrase); - await applyP2pTestTweaks(clientSettings, "p2p-client", passphrase); + await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase); + await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase); const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host"); try { @@ -36,7 +39,7 @@ Deno.test("p2p-sync: discovers peer and completes sync", async () => { clientVault, clientSettings, peersTimeout, - Deno.env.get("TARGET_PEER") ?? undefined + Deno.env.get("TARGET_PEER") ?? hostPeerName ); const syncResult = await runCli( clientVault, diff --git a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts b/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts index 744692e..6752f9b 100644 --- a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts +++ b/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts @@ -1,6 +1,6 @@ import { assert } from "@std/assert"; import { TempDir } from "./helpers/temp.ts"; -import { applyP2pSettings, initSettingsFile } from "./helpers/settings.ts"; +import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts"; import { jsonStringField, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; @@ -12,6 +12,10 @@ Deno.test("p2p: three nodes detect and resolve conflicts", async () => { const appId = Deno.env.get("APP_ID") ?? "self-hosted-livesync-cli-tests"; const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "10"); const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "15"); + const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; + const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; + const peerNameB = Deno.env.get("PEER_NAME_B") ?? `p2p-client-b-${nonce}`; + const peerNameC = Deno.env.get("PEER_NAME_C") ?? `p2p-client-c-${nonce}`; await using workDir = await TempDir.create("livesync-cli-p2p-3nodes"); const vaultA = workDir.join("vault-a"); @@ -26,16 +30,21 @@ Deno.test("p2p: three nodes detect and resolve conflicts", async () => { const relayStarted = await maybeStartLocalRelay(relay); try { - for (const settings of [settingsA, settingsB, settingsC]) { - await initSettingsFile(settings); - await applyP2pSettings(settings, roomId, passphrase, appId, relay); - } + await initSettingsFile(settingsA); + await initSettingsFile(settingsB); + await initSettingsFile(settingsC); + await applyP2pSettings(settingsA, roomId, passphrase, appId, relay); + await applyP2pSettings(settingsB, roomId, passphrase, appId, relay); + await applyP2pSettings(settingsC, roomId, passphrase, appId, relay); + await applyP2pTestTweaks(settingsA, hostPeerName, passphrase); + await applyP2pTestTweaks(settingsB, peerNameB, passphrase); + await applyP2pTestTweaks(settingsC, peerNameC, passphrase); const host = startCliInBackground(vaultA, "--settings", settingsA, "p2p-host"); try { await host.waitUntilContains("P2P host is running", 20000); - const peerFromB = await discoverPeer(vaultB, settingsB, peersTimeout); - const peerFromC = await discoverPeer(vaultC, settingsC, peersTimeout); + const peerFromB = await discoverPeer(vaultB, settingsB, peersTimeout, hostPeerName); + const peerFromC = await discoverPeer(vaultC, settingsC, peersTimeout, hostPeerName); const targetPath = "p2p/conflicted-from-two-clients.txt"; await runCliWithInputOrFail("from-client-b-v1\n", vaultB, "--settings", settingsB, "put", targetPath); From 3ab80190d63ba3068e4dcb99d24cec17fae354b5 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:48:41 +0000 Subject: [PATCH 08/15] test fix ci (Redundant test) --- .github/workflows/cli-deno-tests.yml | 4 ++-- src/apps/cli/testdeno/deno.json | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 5bfd940..60e8450 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -32,13 +32,13 @@ jobs: case "$SELECTED_TASK" in test) - TASK_MATRIX='["test:setup-put-cat","test:mirror","test:push-pull","test:sync-two-local","test:sync-locked-remote","test:p2p-host","test:p2p-peers","test:p2p-sync","test:p2p-three-nodes","test:p2p-upload-download","test:e2e-couchdb","test:e2e-matrix"]' + TASK_MATRIX='["test:setup-put-cat","test:mirror","test:push-pull","test:sync-two-local","test:sync-locked-remote","test:p2p-host","test:p2p-peers","test:p2p-sync","test:p2p-three-nodes","test:p2p-upload-download","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' ;; test:local) TASK_MATRIX='["test:setup-put-cat","test:mirror"]' ;; test:e2e-matrix) - TASK_MATRIX='["test:e2e-matrix"]' + TASK_MATRIX='["test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' ;; test:p2p-sync) TASK_MATRIX='["test:p2p-sync"]' diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json index 5c56614..8a273f8 100644 --- a/src/apps/cli/testdeno/deno.json +++ b/src/apps/cli/testdeno/deno.json @@ -17,7 +17,11 @@ "bench:item1": "bash ./bench-run-item1.sh", "bench:item1:full": "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh", "test:e2e-couchdb": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-couchdb.ts", - "test:e2e-matrix": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-matrix.ts" + "test:e2e-matrix": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-matrix.ts", + "test:e2e-matrix:couchdb-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc0' test-e2e-two-vaults-matrix.ts", + "test:e2e-matrix:couchdb-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc1' test-e2e-two-vaults-matrix.ts", + "test:e2e-matrix:minio-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc0' test-e2e-two-vaults-matrix.ts", + "test:e2e-matrix:minio-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc1' test-e2e-two-vaults-matrix.ts" }, "imports": { "@std/assert": "jsr:@std/assert@^1.0.13", From 37593bbee61d170066e4bd6618434d21fd093345 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 5 Jun 2026 09:07:38 +0100 Subject: [PATCH 09/15] Update CI to use deno --- .github/workflows/cli-deno-tests.yml | 43 +++++++++++++++++++++----- .github/workflows/cli-e2e.yml | 17 ---------- src/apps/cli/testdeno/helpers/p2p.ts | 3 ++ src/apps/cli/testdeno/test_dev_deno.md | 2 +- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 60e8450..2bf8753 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -1,17 +1,39 @@ name: cli-deno-tests on: + push: + branches: + - main + - beta + paths: + - '.github/workflows/cli-deno-tests.yml' + - 'src/apps/cli/**' + - 'src/lib/src/API/processSetting.ts' + - 'package.json' + - 'package-lock.json' + pull_request: + paths: + - '.github/workflows/cli-deno-tests.yml' + - 'src/apps/cli/**' + - 'src/lib/src/API/processSetting.ts' + - 'package.json' + - 'package-lock.json' workflow_dispatch: inputs: test_task: description: 'Deno test task to run' type: choice options: - - test + - test:ci + - test:p2p + - test:all - test:local - test:e2e-matrix - - test:p2p-sync - default: test + default: test:ci + enable_debug: + description: 'Enable verbose and debug logging' + type: boolean + default: false permissions: contents: read @@ -27,11 +49,17 @@ jobs: shell: bash run: | set -euo pipefail - SELECTED_TASK="${{ github.event_name == 'workflow_dispatch' && inputs.test_task || 'test' }}" + SELECTED_TASK="${{ github.event_name == 'workflow_dispatch' && inputs.test_task || 'test:ci' }}" echo "[INFO] Selected task set: $SELECTED_TASK" case "$SELECTED_TASK" in - test) + test:ci) + TASK_MATRIX='["test:setup-put-cat","test:mirror","test:push-pull","test:sync-two-local","test:sync-locked-remote","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' + ;; + test:p2p) + TASK_MATRIX='["test:p2p-host","test:p2p-peers","test:p2p-sync","test:p2p-three-nodes","test:p2p-upload-download"]' + ;; + test:all) TASK_MATRIX='["test:setup-put-cat","test:mirror","test:push-pull","test:sync-two-local","test:sync-locked-remote","test:p2p-host","test:p2p-peers","test:p2p-sync","test:p2p-three-nodes","test:p2p-upload-download","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' ;; test:local) @@ -40,9 +68,6 @@ jobs: test:e2e-matrix) TASK_MATRIX='["test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' ;; - test:p2p-sync) - TASK_MATRIX='["test:p2p-sync"]' - ;; *) echo "[ERROR] Unknown task set: $SELECTED_TASK" >&2 exit 1 @@ -113,6 +138,8 @@ jobs: env: LIVESYNC_DOCKER_MODE: native LIVESYNC_CLI_RETRY: 3 + LIVESYNC_CLI_DEBUG: ${{ inputs.enable_debug == true && '1' || '0' }} + LIVESYNC_CLI_VERBOSE: ${{ inputs.enable_debug == true && '1' || '0' }} run: | TASK="${{ matrix.task }}" echo "[INFO] Running Deno task: $TASK" diff --git a/.github/workflows/cli-e2e.yml b/.github/workflows/cli-e2e.yml index 0371a0d..1738860 100644 --- a/.github/workflows/cli-e2e.yml +++ b/.github/workflows/cli-e2e.yml @@ -12,23 +12,6 @@ on: - two-vaults-couchdb - two-vaults-minio default: two-vaults-matrix - push: - branches: - - main - - beta - paths: - - '.github/workflows/cli-e2e.yml' - - 'src/apps/cli/**' - - 'src/lib/src/API/processSetting.ts' - - 'package.json' - - 'package-lock.json' - pull_request: - paths: - - '.github/workflows/cli-e2e.yml' - - 'src/apps/cli/**' - - 'src/lib/src/API/processSetting.ts' - - 'package.json' - - 'package-lock.json' permissions: contents: read diff --git a/src/apps/cli/testdeno/helpers/p2p.ts b/src/apps/cli/testdeno/helpers/p2p.ts index efd8f7f..a7381d7 100644 --- a/src/apps/cli/testdeno/helpers/p2p.ts +++ b/src/apps/cli/testdeno/helpers/p2p.ts @@ -84,6 +84,9 @@ export async function maybeStartLocalRelay(relay: string): Promise { intervalMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_INTERVAL_MS") ?? "250"), connectTimeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_CONNECT_TIMEOUT_MS") ?? "1000"), }); + // Docker proxy accepts TCP connections instantly before the container's internal process is fully ready. + // Wait an additional few seconds to ensure strfry is actually accepting WebSockets. + await sleep(3000); return true; } diff --git a/src/apps/cli/testdeno/test_dev_deno.md b/src/apps/cli/testdeno/test_dev_deno.md index acc07ba..9ad15ae 100644 --- a/src/apps/cli/testdeno/test_dev_deno.md +++ b/src/apps/cli/testdeno/test_dev_deno.md @@ -281,7 +281,7 @@ deno task test:sync-two-local ## Continuous Integration -The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` is used to run these tests automatically on push and pull requests affecting the CLI. +The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` runs automatically on pushes and pull requests affecting the CLI, executing the non-P2P test suite (`test:ci`). P2P tests (`test:p2p`) are excluded from automatic execution and must be run via manual dispatch (`workflow_dispatch`). You can optionally check the "Enable verbose and debug logging" checkbox during a manual dispatch to produce detailed trace logs for troubleshooting. --- From 369e62ee8d15cac9f59637fd1c8d33d84011bedc Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 5 Jun 2026 09:27:19 +0100 Subject: [PATCH 10/15] Improved: we can set empty for turnServer explicitly. --- .../cli/testdeno/helpers/backgroundCli.ts | 26 ++++++++++++------- src/apps/cli/testdeno/helpers/cli.ts | 6 ++--- src/apps/cli/testdeno/helpers/settings.ts | 1 + src/lib | 2 +- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/apps/cli/testdeno/helpers/backgroundCli.ts b/src/apps/cli/testdeno/helpers/backgroundCli.ts index 1269d0f..ef81fea 100644 --- a/src/apps/cli/testdeno/helpers/backgroundCli.ts +++ b/src/apps/cli/testdeno/helpers/backgroundCli.ts @@ -1,4 +1,4 @@ -import { CLI_DIR } from "./cli.ts"; +import { CLI_DIR, TEE_ENABLED, formatTeeCommand, createLineTeeWriter } from "./cli.ts"; import { join } from "@std/path"; const CLI_DIST = join(CLI_DIR, "dist", "index.cjs"); @@ -12,10 +12,9 @@ function decorateArgs(args: string[]): string[] { async function pump( stream: ReadableStream, sink: (text: string) => void, - teeTarget: WritableStream | null + teeTarget: { write: (chunk: Uint8Array) => void; close: () => void } | null ): Promise { const reader = stream.getReader(); - const writer = teeTarget?.getWriter(); const dec = new TextDecoder(); try { while (true) { @@ -23,12 +22,12 @@ async function pump( if (done) break; if (!value) continue; sink(dec.decode(value, { stream: true })); - if (writer) { - await writer.write(value); + if (teeTarget) { + teeTarget.write(value); } } } finally { - if (writer) writer.releaseLock(); + if (teeTarget) teeTarget.close(); reader.releaseLock(); } } @@ -43,19 +42,20 @@ export class BackgroundCliProcess { readonly child: Deno.ChildProcess, readonly args: string[] ) { + const cliArgs = decorateArgs(args); this.#stdoutDone = pump( child.stdout, (text) => { this.#stdout += text; }, - null + TEE_ENABLED ? createLineTeeWriter(child.pid, "stdout", (chunk) => Deno.stdout.writeSync(chunk)) : null ); this.#stderrDone = pump( child.stderr, (text) => { this.#stderr += text; }, - null + TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => Deno.stderr.writeSync(chunk)) : null ); } @@ -101,12 +101,20 @@ export class BackgroundCliProcess { } export function startCliInBackground(...args: string[]): BackgroundCliProcess { + const cliArgs = decorateArgs(args); const child = new Deno.Command("node", { - args: [CLI_DIST, ...decorateArgs(args)], + args: [CLI_DIST, ...cliArgs], cwd: CLI_DIR, stdin: "null", stdout: "piped", stderr: "piped", }).spawn(); + + if (TEE_ENABLED) { + Deno.stdout.writeSync( + new TextEncoder().encode(`[CLI tee pid=${child.pid}] process(bg): ${formatTeeCommand(cliArgs)}\n`) + ); + } + return new BackgroundCliProcess(child, args); } diff --git a/src/apps/cli/testdeno/helpers/cli.ts b/src/apps/cli/testdeno/helpers/cli.ts index a7e3a42..c393c26 100644 --- a/src/apps/cli/testdeno/helpers/cli.ts +++ b/src/apps/cli/testdeno/helpers/cli.ts @@ -20,7 +20,7 @@ export interface CliResult { code: number; } -const TEE_ENABLED = Deno.env.get("LIVESYNC_TEST_TEE") === "1"; +export const TEE_ENABLED = Deno.env.get("LIVESYNC_TEST_TEE") === "1"; const VERBOSE_ENABLED = Deno.env.get("LIVESYNC_CLI_VERBOSE") === "1"; const DEBUG_ENABLED = Deno.env.get("LIVESYNC_CLI_DEBUG") === "1"; @@ -39,11 +39,11 @@ function concatChunks(chunks: Uint8Array[]): Uint8Array { return out; } -function formatTeeCommand(args: string[]): string { +export function formatTeeCommand(args: string[]): string { return ["node", CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" "); } -function createLineTeeWriter( +export function createLineTeeWriter( pid: number, streamName: "stdout" | "stderr", writer: (chunk: Uint8Array) => void diff --git a/src/apps/cli/testdeno/helpers/settings.ts b/src/apps/cli/testdeno/helpers/settings.ts index 6f88a03..7f114dd 100644 --- a/src/apps/cli/testdeno/helpers/settings.ts +++ b/src/apps/cli/testdeno/helpers/settings.ts @@ -184,6 +184,7 @@ export async function applyP2pSettings( data.P2P_relays = relays; data.P2P_AutoAcceptingPeers = autoAccept; data.P2P_AutoDenyingPeers = ""; + data.P2P_turnServers = "none"; data.P2P_IsHeadless = true; data.isConfigured = true; await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); diff --git a/src/lib b/src/lib index 76d9167..808efe1 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 76d91674c235c1ccf991a14802c737e82e144ef1 +Subproject commit 808efe19c8e94b32a039463d9bcd407550d335e6 From 6b7816d334328bb5d729ca31dbc5b90d0de520c7 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 5 Jun 2026 09:39:39 +0100 Subject: [PATCH 11/15] add coturn for test --- .github/workflows/cli-deno-tests.yml | 4 +- src/apps/cli/testdeno/helpers/docker.ts | 39 +++++++++++++++++++ src/apps/cli/testdeno/helpers/p2p.ts | 16 +++++++- src/apps/cli/testdeno/helpers/settings.ts | 7 +++- .../testdeno/test-p2p-peers-local-relay.ts | 31 +++++++++++++-- src/apps/cli/testdeno/test-p2p-sync.ts | 31 +++++++++++++-- .../testdeno/test-p2p-three-nodes-conflict.ts | 17 ++++++-- 7 files changed, 130 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 2bf8753..4d3790c 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -148,5 +148,5 @@ jobs: - name: Stop leftover containers if: always() run: | - docker stop couchdb-test minio-test relay-test >/dev/null 2>&1 || true - docker rm couchdb-test minio-test relay-test >/dev/null 2>&1 || true + docker stop couchdb-test minio-test relay-test coturn-test >/dev/null 2>&1 || true + docker rm couchdb-test minio-test relay-test coturn-test >/dev/null 2>&1 || true diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/src/apps/cli/testdeno/helpers/docker.ts index e9c91ea..c20ccd0 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/src/apps/cli/testdeno/helpers/docker.ts @@ -627,3 +627,42 @@ export async function startP2pRelay(): Promise { export function isLocalP2pRelay(relayUrl: string): boolean { return relayUrl === "ws://localhost:4000" || relayUrl === "ws://localhost:4000/"; } + +// --------------------------------------------------------------------------- +// Coturn (STUN/TURN) +// --------------------------------------------------------------------------- +const COTURN_CONTAINER = "coturn-test"; +const COTURN_IMAGE = "coturn/coturn:latest"; + +export async function stopCoturn(): Promise { + await stopAndRemoveContainer(COTURN_CONTAINER); + untrackContainer(COTURN_CONTAINER); +} + +export async function startCoturn( + port = 3478, + user = "testuser", + pass = "testpass", + realm = "livesync.test" +): Promise { + console.log("[INFO] stopping leftover Coturn container if present"); + await stopCoturn().catch(() => {}); + + console.log("[INFO] starting local Coturn container"); + await dockerOrFail( + "run", + "-d", + "--name", + COTURN_CONTAINER, + "-p", + `${port}:${port}`, + "-p", + `${port}:${port}/udp`, + COTURN_IMAGE, + "--log-file=stdout", + "--external-ip=127.0.0.1", + `--user=${user}:${pass}`, + `--realm=${realm}` + ); + trackContainer(COTURN_CONTAINER); +} diff --git a/src/apps/cli/testdeno/helpers/p2p.ts b/src/apps/cli/testdeno/helpers/p2p.ts index a7381d7..f3e0574 100644 --- a/src/apps/cli/testdeno/helpers/p2p.ts +++ b/src/apps/cli/testdeno/helpers/p2p.ts @@ -1,5 +1,5 @@ import { runCli } from "./cli.ts"; -import { isLocalP2pRelay, startP2pRelay, stopP2pRelay } from "./docker.ts"; +import { isLocalP2pRelay, startP2pRelay, stopP2pRelay, startCoturn, stopCoturn } from "./docker.ts"; import { waitForPort } from "./net.ts"; export type PeerEntry = { @@ -95,3 +95,17 @@ export async function stopLocalRelayIfStarted(started: boolean): Promise { await stopP2pRelay().catch(() => {}); } } + +export async function maybeStartCoturn(turnServers: string): Promise { + if (turnServers.includes("localhost") || turnServers.includes("127.0.0.1")) { + await startCoturn(); + return true; + } + return false; +} + +export async function stopCoturnIfStarted(started: boolean): Promise { + if (started) { + await stopCoturn().catch(() => {}); + } +} diff --git a/src/apps/cli/testdeno/helpers/settings.ts b/src/apps/cli/testdeno/helpers/settings.ts index 7f114dd..316025f 100644 --- a/src/apps/cli/testdeno/helpers/settings.ts +++ b/src/apps/cli/testdeno/helpers/settings.ts @@ -172,7 +172,8 @@ export async function applyP2pSettings( passphrase: string, appId = "self-hosted-livesync-cli-tests", relays = "ws://localhost:4000/", - autoAccept = "~.*" + autoAccept = "~.*", + turnServers = "turn:127.0.0.1:3478" ): Promise { const data = JSON.parse(await Deno.readTextFile(settingsFile)); data.P2P_Enabled = true; @@ -184,7 +185,9 @@ export async function applyP2pSettings( data.P2P_relays = relays; data.P2P_AutoAcceptingPeers = autoAccept; data.P2P_AutoDenyingPeers = ""; - data.P2P_turnServers = "none"; + data.P2P_turnServers = turnServers; + data.P2P_turnUsername = "testuser"; + data.P2P_turnCredential = "testpass"; data.P2P_IsHeadless = true; data.isConfigured = true; await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); diff --git a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts b/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts index c87c8a9..f1c481f 100644 --- a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts +++ b/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts @@ -2,7 +2,13 @@ import { assert } from "@std/assert"; import { TempDir } from "./helpers/temp.ts"; import { initSettingsFile, applyP2pSettings, applyP2pTestTweaks } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; -import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts"; +import { + discoverPeer, + maybeStartLocalRelay, + stopLocalRelayIfStarted, + maybeStartCoturn, + stopCoturnIfStarted, +} from "./helpers/p2p.ts"; Deno.test("p2p-peers: discovers host through local relay", async () => { const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; @@ -12,6 +18,7 @@ Deno.test("p2p-peers: discovers host through local relay", async () => { const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; + const turnServers = Deno.env.get("TURN_SERVERS") ?? "turn:127.0.0.1:3478"; await using workDir = await TempDir.create("livesync-cli-p2p-peers-local-relay"); const hostVault = workDir.join("vault-host"); @@ -22,11 +29,28 @@ Deno.test("p2p-peers: discovers host through local relay", async () => { await Deno.mkdir(clientVault, { recursive: true }); const relayStarted = await maybeStartLocalRelay(relay); + const coturnStarted = await maybeStartCoturn(turnServers); try { await initSettingsFile(hostSettings); await initSettingsFile(clientSettings); - await applyP2pSettings(hostSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); - await applyP2pSettings(clientSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); + await applyP2pSettings( + hostSettings, + roomId, + passphrase, + "self-hosted-livesync-cli-tests", + relay, + "~.*", + turnServers + ); + await applyP2pSettings( + clientSettings, + roomId, + passphrase, + "self-hosted-livesync-cli-tests", + relay, + "~.*", + turnServers + ); await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase); await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase); @@ -42,5 +66,6 @@ Deno.test("p2p-peers: discovers host through local relay", async () => { } } finally { await stopLocalRelayIfStarted(relayStarted); + await stopCoturnIfStarted(coturnStarted); } }); diff --git a/src/apps/cli/testdeno/test-p2p-sync.ts b/src/apps/cli/testdeno/test-p2p-sync.ts index 2a50745..9772232 100644 --- a/src/apps/cli/testdeno/test-p2p-sync.ts +++ b/src/apps/cli/testdeno/test-p2p-sync.ts @@ -2,7 +2,13 @@ import { assert } from "@std/assert"; import { TempDir } from "./helpers/temp.ts"; import { initSettingsFile, applyP2pSettings, applyP2pTestTweaks } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; -import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts"; +import { + discoverPeer, + maybeStartLocalRelay, + stopLocalRelayIfStarted, + maybeStartCoturn, + stopCoturnIfStarted, +} from "./helpers/p2p.ts"; import { runCli } from "./helpers/cli.ts"; Deno.test("p2p-sync: discovers peer and completes sync", async () => { @@ -14,6 +20,7 @@ Deno.test("p2p-sync: discovers peer and completes sync", async () => { const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; + const turnServers = Deno.env.get("TURN_SERVERS") ?? "turn:127.0.0.1:3478"; await using workDir = await TempDir.create("livesync-cli-p2p-sync"); const hostVault = workDir.join("vault-host"); @@ -24,11 +31,28 @@ Deno.test("p2p-sync: discovers peer and completes sync", async () => { await Deno.mkdir(clientVault, { recursive: true }); const relayStarted = await maybeStartLocalRelay(relay); + const coturnStarted = await maybeStartCoturn(turnServers); try { await initSettingsFile(hostSettings); await initSettingsFile(clientSettings); - await applyP2pSettings(hostSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); - await applyP2pSettings(clientSettings, roomId, passphrase, "self-hosted-livesync-cli-tests", relay); + await applyP2pSettings( + hostSettings, + roomId, + passphrase, + "self-hosted-livesync-cli-tests", + relay, + "~.*", + turnServers + ); + await applyP2pSettings( + clientSettings, + roomId, + passphrase, + "self-hosted-livesync-cli-tests", + relay, + "~.*", + turnServers + ); await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase); await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase); @@ -58,5 +82,6 @@ Deno.test("p2p-sync: discovers peer and completes sync", async () => { } } finally { await stopLocalRelayIfStarted(relayStarted); + await stopCoturnIfStarted(coturnStarted); } }); diff --git a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts b/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts index 6752f9b..ba6ed1a 100644 --- a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts +++ b/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts @@ -2,7 +2,13 @@ import { assert } from "@std/assert"; import { TempDir } from "./helpers/temp.ts"; import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; -import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts"; +import { + discoverPeer, + maybeStartLocalRelay, + stopLocalRelayIfStarted, + maybeStartCoturn, + stopCoturnIfStarted, +} from "./helpers/p2p.ts"; import { jsonStringField, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; Deno.test("p2p: three nodes detect and resolve conflicts", async () => { @@ -16,6 +22,7 @@ Deno.test("p2p: three nodes detect and resolve conflicts", async () => { const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; const peerNameB = Deno.env.get("PEER_NAME_B") ?? `p2p-client-b-${nonce}`; const peerNameC = Deno.env.get("PEER_NAME_C") ?? `p2p-client-c-${nonce}`; + const turnServers = Deno.env.get("TURN_SERVERS") ?? "turn:127.0.0.1:3478"; await using workDir = await TempDir.create("livesync-cli-p2p-3nodes"); const vaultA = workDir.join("vault-a"); @@ -29,13 +36,14 @@ Deno.test("p2p: three nodes detect and resolve conflicts", async () => { await Deno.mkdir(vaultC, { recursive: true }); const relayStarted = await maybeStartLocalRelay(relay); + const coturnStarted = await maybeStartCoturn(turnServers); try { await initSettingsFile(settingsA); await initSettingsFile(settingsB); await initSettingsFile(settingsC); - await applyP2pSettings(settingsA, roomId, passphrase, appId, relay); - await applyP2pSettings(settingsB, roomId, passphrase, appId, relay); - await applyP2pSettings(settingsC, roomId, passphrase, appId, relay); + await applyP2pSettings(settingsA, roomId, passphrase, appId, relay, "~.*", turnServers); + await applyP2pSettings(settingsB, roomId, passphrase, appId, relay, "~.*", turnServers); + await applyP2pSettings(settingsC, roomId, passphrase, appId, relay, "~.*", turnServers); await applyP2pTestTweaks(settingsA, hostPeerName, passphrase); await applyP2pTestTweaks(settingsB, peerNameB, passphrase); await applyP2pTestTweaks(settingsC, peerNameC, passphrase); @@ -123,5 +131,6 @@ Deno.test("p2p: three nodes detect and resolve conflicts", async () => { } } finally { await stopLocalRelayIfStarted(relayStarted); + await stopCoturnIfStarted(coturnStarted); } }); From 60f21eb9d2c31fdc03dcba58b91ff211625937d0 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 5 Jun 2026 09:44:17 +0100 Subject: [PATCH 12/15] detect loopback and coturn option --- .github/workflows/cli-deno-tests.yml | 5 +++++ src/apps/cli/testdeno/helpers/docker.ts | 9 ++++++--- src/apps/cli/testdeno/helpers/net.ts | 19 +++++++++++++++++++ src/apps/cli/testdeno/helpers/p2p.ts | 2 +- .../testdeno/test-p2p-peers-local-relay.ts | 9 +++++++-- src/apps/cli/testdeno/test-p2p-sync.ts | 9 +++++++-- .../testdeno/test-p2p-three-nodes-conflict.ts | 15 ++++++++++----- 7 files changed, 55 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 4d3790c..46a5e7a 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -34,6 +34,10 @@ on: description: 'Enable verbose and debug logging' type: boolean default: false + use_coturn: + description: 'Enable local coturn container for P2P tests' + type: boolean + default: false permissions: contents: read @@ -140,6 +144,7 @@ jobs: LIVESYNC_CLI_RETRY: 3 LIVESYNC_CLI_DEBUG: ${{ inputs.enable_debug == true && '1' || '0' }} LIVESYNC_CLI_VERBOSE: ${{ inputs.enable_debug == true && '1' || '0' }} + LIVESYNC_USE_COTURN: ${{ inputs.use_coturn == true && '1' || '0' }} run: | TASK="${{ matrix.task }}" echo "[INFO] Running Deno task: $TASK" diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/src/apps/cli/testdeno/helpers/docker.ts index c20ccd0..3c0e6ce 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/src/apps/cli/testdeno/helpers/docker.ts @@ -625,7 +625,7 @@ export async function startP2pRelay(): Promise { } export function isLocalP2pRelay(relayUrl: string): boolean { - return relayUrl === "ws://localhost:4000" || relayUrl === "ws://localhost:4000/"; + return relayUrl.includes("localhost") || relayUrl.includes("127.0.0.1") || relayUrl.includes("[::1]"); } // --------------------------------------------------------------------------- @@ -648,7 +648,10 @@ export async function startCoturn( console.log("[INFO] stopping leftover Coturn container if present"); await stopCoturn().catch(() => {}); - console.log("[INFO] starting local Coturn container"); + const { getOptimalLoopbackIp } = await import("./net.ts"); + const externalIp = await getOptimalLoopbackIp(); + + console.log(`[INFO] starting local Coturn container with external-ip ${externalIp}`); await dockerOrFail( "run", "-d", @@ -660,7 +663,7 @@ export async function startCoturn( `${port}:${port}/udp`, COTURN_IMAGE, "--log-file=stdout", - "--external-ip=127.0.0.1", + `--external-ip=${externalIp}`, `--user=${user}:${pass}`, `--realm=${realm}` ); diff --git a/src/apps/cli/testdeno/helpers/net.ts b/src/apps/cli/testdeno/helpers/net.ts index 362c70d..fa5debd 100644 --- a/src/apps/cli/testdeno/helpers/net.ts +++ b/src/apps/cli/testdeno/helpers/net.ts @@ -47,3 +47,22 @@ export async function waitForPort(hostname: string, port: number, options: WaitF (lastError ? ` (last error: ${String(lastError)})` : "") ); } + +export async function getOptimalLoopbackIp(): Promise { + const ipv4 = "127.0.0.1"; + const ipv6 = "::1"; + + try { + const l = Deno.listen({ hostname: ipv4, port: 0 }); + l.close(); + return ipv4; + } catch { + try { + const l = Deno.listen({ hostname: ipv6, port: 0 }); + l.close(); + return ipv6; + } catch { + return ipv4; // fallback to default + } + } +} diff --git a/src/apps/cli/testdeno/helpers/p2p.ts b/src/apps/cli/testdeno/helpers/p2p.ts index f3e0574..a04b080 100644 --- a/src/apps/cli/testdeno/helpers/p2p.ts +++ b/src/apps/cli/testdeno/helpers/p2p.ts @@ -97,7 +97,7 @@ export async function stopLocalRelayIfStarted(started: boolean): Promise { } export async function maybeStartCoturn(turnServers: string): Promise { - if (turnServers.includes("localhost") || turnServers.includes("127.0.0.1")) { + if (turnServers.includes("localhost") || turnServers.includes("127.0.0.1") || turnServers.includes("[::1]")) { await startCoturn(); return true; } diff --git a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts b/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts index f1c481f..36dbbf8 100644 --- a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts +++ b/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts @@ -9,16 +9,21 @@ import { maybeStartCoturn, stopCoturnIfStarted, } from "./helpers/p2p.ts"; +import { getOptimalLoopbackIp } from "./helpers/net.ts"; Deno.test("p2p-peers: discovers host through local relay", async () => { - const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; + const loopbackIp = await getOptimalLoopbackIp(); + const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; + + const relay = Deno.env.get("RELAY") ?? `ws://${loopbackHost}:4000/`; const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; const timeoutSeconds = Number(Deno.env.get("TIMEOUT_SECONDS") ?? "8"); const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; - const turnServers = Deno.env.get("TURN_SERVERS") ?? "turn:127.0.0.1:3478"; + const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; + const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-peers-local-relay"); const hostVault = workDir.join("vault-host"); diff --git a/src/apps/cli/testdeno/test-p2p-sync.ts b/src/apps/cli/testdeno/test-p2p-sync.ts index 9772232..53cefda 100644 --- a/src/apps/cli/testdeno/test-p2p-sync.ts +++ b/src/apps/cli/testdeno/test-p2p-sync.ts @@ -10,9 +10,13 @@ import { stopCoturnIfStarted, } from "./helpers/p2p.ts"; import { runCli } from "./helpers/cli.ts"; +import { getOptimalLoopbackIp } from "./helpers/net.ts"; Deno.test("p2p-sync: discovers peer and completes sync", async () => { - const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; + const loopbackIp = await getOptimalLoopbackIp(); + const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; + + const relay = Deno.env.get("RELAY") ?? `ws://${loopbackHost}:4000/`; const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "12"); @@ -20,7 +24,8 @@ Deno.test("p2p-sync: discovers peer and completes sync", async () => { const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; - const turnServers = Deno.env.get("TURN_SERVERS") ?? "turn:127.0.0.1:3478"; + const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; + const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-sync"); const hostVault = workDir.join("vault-host"); diff --git a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts b/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts index ba6ed1a..718367d 100644 --- a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts +++ b/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts @@ -10,19 +10,24 @@ import { stopCoturnIfStarted, } from "./helpers/p2p.ts"; import { jsonStringField, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; +import { getOptimalLoopbackIp } from "./helpers/net.ts"; Deno.test("p2p: three nodes detect and resolve conflicts", async () => { - const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; - const roomId = `${Deno.env.get("ROOM_ID_PREFIX") ?? "p2p-room"}-${Date.now()}`; - const passphrase = `${Deno.env.get("PASSPHRASE_PREFIX") ?? "p2p-pass"}-${Date.now()}`; - const appId = Deno.env.get("APP_ID") ?? "self-hosted-livesync-cli-tests"; + const loopbackIp = await getOptimalLoopbackIp(); + const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; + + const relay = Deno.env.get("RELAY") ?? `ws://${loopbackHost}:4000/`; + const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; + const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; + const appId = "self-hosted-livesync-cli-tests"; const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "10"); const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "15"); const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; const peerNameB = Deno.env.get("PEER_NAME_B") ?? `p2p-client-b-${nonce}`; const peerNameC = Deno.env.get("PEER_NAME_C") ?? `p2p-client-c-${nonce}`; - const turnServers = Deno.env.get("TURN_SERVERS") ?? "turn:127.0.0.1:3478"; + const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; + const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-3nodes"); const vaultA = workDir.join("vault-a"); From 2d8a285201d0da7ee558cc71f6ebe4a75ef27b1b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 8 Jun 2026 10:38:54 +0000 Subject: [PATCH 13/15] Port new tests --- src/apps/cli/testdeno/deno.json | 5 +- src/apps/cli/testdeno/test-daemon.ts | 116 ++++++++++++++++ src/apps/cli/testdeno/test-decoupled-vault.ts | 114 +++++++++++++++ src/apps/cli/testdeno/test-remote-commands.ts | 130 ++++++++++++++++++ src/apps/cli/testdeno/test_dev_deno.md | 22 +++ 5 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 src/apps/cli/testdeno/test-daemon.ts create mode 100644 src/apps/cli/testdeno/test-decoupled-vault.ts create mode 100644 src/apps/cli/testdeno/test-remote-commands.ts diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json index 8a273f8..2fd5183 100644 --- a/src/apps/cli/testdeno/deno.json +++ b/src/apps/cli/testdeno/deno.json @@ -1,7 +1,10 @@ { "tasks": { "test": "deno test --env-file=.test.env -A --no-check test-*.ts", - "test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts", + "test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts", + "test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts", + "test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts", + "test:remote-commands": "deno test --env-file=.test.env -A --no-check test-remote-commands.ts", "test:push-pull": "deno test --env-file=.test.env -A --no-check test-push-pull.ts", "test:setup-put-cat": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts", "test:mirror": "deno test --env-file=.test.env -A --no-check test-mirror.ts", diff --git a/src/apps/cli/testdeno/test-daemon.ts b/src/apps/cli/testdeno/test-daemon.ts new file mode 100644 index 0000000..e960dfc --- /dev/null +++ b/src/apps/cli/testdeno/test-daemon.ts @@ -0,0 +1,116 @@ +/** + * Deno port of test-daemon-linux.sh + * + * Tests daemon-related ignore rules behaviour. + * + * Tests that are runnable without a long-running daemon process are exercised + * here using the 'mirror' command, which calls the same 'isTargetFile' handler + * stack that the daemon uses. + * + * Covered cases: + * 1. .livesync/ignore with *.tmp pattern → ignored file is not synced to database + * 2. .livesync/ignore missing → no error, and normal synchronisation continues + * 3. import: .gitignore directive → patterns from .gitignore are merged + * + * Run: + * deno test -A test-daemon.ts + */ + +import { join } from "@std/path"; +import { assertEquals } from "@std/assert"; +import { TempDir } from "./helpers/temp.ts"; +import { runCliOrFail, runCli, assertContains, assertNotContains } from "./helpers/cli.ts"; +import { initSettingsFile, markSettingsConfigured } from "./helpers/settings.ts"; + +Deno.test("daemon: ignore rules behaviour", async (t) => { + // ------------------------------------------------------------------------- + // Case 1: .livesync/ignore with *.tmp → ignored file not synced to database + // ------------------------------------------------------------------------- + await t.step("case 1: .livesync/ignore *.tmp prevents sync", async () => { + await using workDir = await TempDir.create("livesync-cli-daemon-c1"); + const settingsFile = workDir.join("data.json"); + const vaultDir = workDir.join("vault"); + + await Deno.mkdir(join(vaultDir, ".livesync"), { recursive: true }); + await Deno.mkdir(join(vaultDir, "notes"), { recursive: true }); + + await initSettingsFile(settingsFile); + await markSettingsConfigured(settingsFile); + + await Deno.writeTextFile(join(vaultDir, ".livesync", "ignore"), "*.tmp\n"); + await Deno.writeTextFile(join(vaultDir, "notes", "normal.md"), "normal content\n"); + await Deno.writeTextFile(join(vaultDir, "notes", "scratch.tmp"), "tmp content\n"); + + console.log("[INFO] Running mirror for Case 1..."); + await runCliOrFail(vaultDir, "--settings", settingsFile, "mirror"); + + // The normal file should be in the database. + const resultNormal = workDir.join("case1-normal.txt"); + await runCliOrFail(vaultDir, "--settings", settingsFile, "pull", "notes/normal.md", resultNormal); + const normalContent = await Deno.readTextFile(resultNormal); + assertEquals(normalContent, "normal content\n", "normal.md content mismatch after mirror"); + + // The .tmp file should NOT be in the database. + const dbList = await runCliOrFail(vaultDir, "--settings", settingsFile, "ls"); + assertNotContains(dbList, "scratch.tmp", "scratch.tmp (ignored) was unexpectedly synced to database"); + assertContains(dbList, "normal.md", "normal.md was not found in database after mirror"); + console.log("[PASS] Case 1 verified successfully"); + }); + + // ------------------------------------------------------------------------- + // Case 2: .livesync/ignore absent → no error, and normal synchronisation continues + // ------------------------------------------------------------------------- + await t.step("case 2: .livesync/ignore absent does not cause failure", async () => { + await using workDir = await TempDir.create("livesync-cli-daemon-c2"); + const settingsFile = workDir.join("data2.json"); + const vaultDir = workDir.join("vault2"); + + await Deno.mkdir(join(vaultDir, "notes"), { recursive: true }); + + await initSettingsFile(settingsFile); + await markSettingsConfigured(settingsFile); + + // No .livesync directory at all. + await Deno.writeTextFile(join(vaultDir, "notes", "hello.md"), "hello\n"); + + console.log("[INFO] Running mirror for Case 2..."); + const result = await runCli(vaultDir, "--settings", settingsFile, "mirror"); + assertEquals(result.code, 0, "mirror exited non-zero when .livesync/ignore is absent"); + + // The normal file should have been synced. + const resultHello = workDir.join("case2-hello.txt"); + await runCliOrFail(vaultDir, "--settings", settingsFile, "pull", "notes/hello.md", resultHello); + const helloContent = await Deno.readTextFile(resultHello); + assertEquals(helloContent, "hello\n", "file content mismatch when .livesync/ignore is absent"); + console.log("[PASS] Case 2 verified successfully"); + }); + + // ------------------------------------------------------------------------- + // Case 3: import: .gitignore merges patterns + // ------------------------------------------------------------------------- + await t.step("case 3: import: .gitignore directive merges patterns", async () => { + await using workDir = await TempDir.create("livesync-cli-daemon-c3"); + const settingsFile = workDir.join("data3.json"); + const vaultDir = workDir.join("vault3"); + + await Deno.mkdir(join(vaultDir, ".livesync"), { recursive: true }); + await Deno.mkdir(join(vaultDir, "notes"), { recursive: true }); + + await initSettingsFile(settingsFile); + await markSettingsConfigured(settingsFile); + + await Deno.writeTextFile(join(vaultDir, ".livesync", "ignore"), "import: .gitignore\n"); + await Deno.writeTextFile(join(vaultDir, ".gitignore"), "# gitignore comment\n*.log\nbuild/\n"); + + await Deno.writeTextFile(join(vaultDir, "notes", "regular.md"), "regular note\n"); + await Deno.writeTextFile(join(vaultDir, "notes", "debug.log"), "log content\n"); + + console.log("[INFO] Running mirror for Case 3..."); + await runCliOrFail(vaultDir, "--settings", settingsFile, "mirror"); + + const dbList = await runCliOrFail(vaultDir, "--settings", settingsFile, "ls"); + assertNotContains(dbList, "debug.log", "debug.log (ignored via .gitignore import) was unexpectedly synced to database"); + assertContains(dbList, "regular.md", "regular.md was not synced normally alongside .gitignore import rules"); + console.log("[PASS] Case 3 verified successfully"); + }); +}); diff --git a/src/apps/cli/testdeno/test-decoupled-vault.ts b/src/apps/cli/testdeno/test-decoupled-vault.ts new file mode 100644 index 0000000..7e0f69a --- /dev/null +++ b/src/apps/cli/testdeno/test-decoupled-vault.ts @@ -0,0 +1,114 @@ +/** + * Deno port of test-decoupled-vault-linux.sh + * + * Tests push, pull, and mirror command behaviour when the vault directory is + * decoupled (separated) from the database directory. + * + * Run: + * deno test -A test-decoupled-vault.ts + */ + +import { join } from "@std/path"; +import { assertEquals } from "@std/assert"; +import { TempDir } from "./helpers/temp.ts"; +import { runCliOrFail } from "./helpers/cli.ts"; +import { applyCouchdbSettings, initSettingsFile, markSettingsConfigured } from "./helpers/settings.ts"; +import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; + +const REMOTE_PATH = Deno.env.get("REMOTE_PATH") ?? "test/push-pull-decoupled.txt"; + +Deno.test("decoupled database and vault", async () => { + await using workDir = await TempDir.create("livesync-cli-decoupled"); + + const settingsFile = workDir.join("data.json"); + const vaultDir = workDir.join("vault"); + const dbDir = workDir.join("db"); + + await Deno.mkdir(join(vaultDir, "test"), { recursive: true }); + await Deno.mkdir(dbDir, { recursive: true }); + + const uri = Deno.env.get("COUCHDB_URI") ?? "http://127.0.0.1:5989/"; + const user = Deno.env.get("COUCHDB_USER") ?? "admin"; + const password = Deno.env.get("COUCHDB_PASSWORD") ?? "testpassword"; + const dbname = Deno.env.get("COUCHDB_DBNAME") ?? `decoupled-${Date.now()}`; + + const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0"; + const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + + if (shouldStartDocker) { + await startCouchdb(uri, user, password, dbname); + } + + try { + await initSettingsFile(settingsFile); + + if (uri && user && password && dbname) { + console.log("[INFO] applying CouchDB environment variables to settings"); + await applyCouchdbSettings(settingsFile, uri, user, password, dbname); + } else { + console.warn( + "[WARN] CouchDB environment variables are not fully set. Push and pull operations may fail." + ); + await markSettingsConfigured(settingsFile); + } + + const srcFile = workDir.join("push-source.txt"); + const pulledFile = workDir.join("pull-result.txt"); + const content = `push-pull-decoupled-test ${new Date().toISOString()}\n`; + await Deno.writeTextFile(srcFile, content); + + // 1. Test push command with decoupled vault directory + console.log(`[INFO] push with decoupled vault -> ${REMOTE_PATH}`); + await runCliOrFail( + dbDir, + "--vault", + vaultDir, + "--settings", + settingsFile, + "push", + srcFile, + REMOTE_PATH + ); + + // 2. Test pull command with decoupled vault directory + console.log(`[INFO] pull with decoupled vault <- ${REMOTE_PATH}`); + await runCliOrFail( + dbDir, + "--vault", + vaultDir, + "--settings", + settingsFile, + "pull", + REMOTE_PATH, + pulledFile + ); + + const pulled = await Deno.readTextFile(pulledFile); + assertEquals(pulled, content, "push/pull roundtrip with decoupled vault content mismatch"); + console.log("[PASS] push/pull roundtrip with decoupled vault matched"); + + // 3. Clean up pulled file and vault test directory to verify mirror + await Deno.remove(pulledFile).catch(() => {}); + await Deno.remove(join(vaultDir, "test"), { recursive: true }).catch(() => {}); + + // 4. Test mirror command with decoupled vault directory + console.log("[INFO] mirror with decoupled vault"); + await runCliOrFail( + dbDir, + "--vault", + vaultDir, + "--settings", + settingsFile, + "mirror" + ); + + const restoredFile = join(vaultDir, REMOTE_PATH); + const restored = await Deno.readTextFile(restoredFile); + assertEquals(restored, content, "mirror with decoupled vault content mismatch"); + console.log("[PASS] mirror with decoupled vault matched"); + } finally { + if (shouldStartDocker && !keepDocker) { + await stopCouchdb().catch(() => {}); + } + } +}); diff --git a/src/apps/cli/testdeno/test-remote-commands.ts b/src/apps/cli/testdeno/test-remote-commands.ts new file mode 100644 index 0000000..9ed0afb --- /dev/null +++ b/src/apps/cli/testdeno/test-remote-commands.ts @@ -0,0 +1,130 @@ +/** + * Deno port of test-remote-commands-linux.sh + * + * Tests remote management commands: remote-status, lock-remote, unlock-remote, + * and mark-resolved. + * + * Scenario: + * 1. Start CouchDB, create a test database, and perform an initial sync. + * 2. Run remote-status and assert that the output contains the database name in JSON format. + * 3. Run lock-remote and verify that the remote database is locked. + * 4. Lock the remote database milestone manually, verify status, and run unlock-remote. + * Assert that the output of unlock-remote contains the unlocked verification status. + * 5. Lock the remote database milestone manually, run mark-resolved, and verify that the + * current device is accepted. + * + * Run: + * deno test -A test-remote-commands.ts + */ + +import { join } from "@std/path"; +import { TempDir } from "./helpers/temp.ts"; +import { runCli, assertContains } from "./helpers/cli.ts"; +import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; +import { startCouchdb, stopCouchdb, updateCouchdbDoc } from "./helpers/docker.ts"; + +async function runCliCombinedOrFail(...args: string[]): Promise { + const res = await runCli(...args); + if (res.code !== 0) { + throw new Error(`CLI exited with code ${res.code}\nstdout: ${res.stdout}\nstderr: ${res.stderr}`); + } + return res.combined; +} + +Deno.test("remote management commands", async () => { + await using workDir = await TempDir.create("livesync-cli-remote-cmds"); + + const settingsFile = workDir.join("settings.json"); + const vaultDir = workDir.join("vault"); + await Deno.mkdir(vaultDir, { recursive: true }); + + const uri = Deno.env.get("COUCHDB_URI") ?? "http://127.0.0.1:5989/"; + const user = Deno.env.get("COUCHDB_USER") ?? "admin"; + const password = Deno.env.get("COUCHDB_PASSWORD") ?? "testpassword"; + const dbSuffix = `${Date.now()}-${Math.floor(Math.random() * 10000)}`; + const dbname = Deno.env.get("COUCHDB_DBNAME") ?? `remotes-${dbSuffix}`; + + const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0"; + const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + + if (shouldStartDocker) { + await startCouchdb(uri, user, password, dbname); + } + + try { + await initSettingsFile(settingsFile); + await applyCouchdbSettings(settingsFile, uri, user, password, dbname, true); + + console.log("[INFO] Performing initial sync to create milestone document..."); + await runCliCombinedOrFail(vaultDir, "--settings", settingsFile, "sync"); + + // 1. remote-status outputs valid JSON with CouchDB details + console.log("[CASE] remote-status outputs valid JSON with CouchDB details"); + const statusOutput = await runCliCombinedOrFail(vaultDir, "--settings", settingsFile, "remote-status"); + assertContains( + statusOutput, + `"db_name": "${dbname}"`, + "remote-status should return JSON containing db_name" + ); + console.log("[PASS] remote-status verified"); + + // 2. lock-remote locks and verifies state + console.log("[CASE] lock-remote locks and verifies state"); + const lockOutput = await runCliCombinedOrFail(vaultDir, "--settings", settingsFile, "lock-remote"); + assertContains( + lockOutput, + "[Verification] Remote Database: LOCKED", + "lock-remote output should show that the remote database is locked" + ); + console.log("[PASS] lock-remote verified"); + + // 3. unlock-remote unlocks and verifies state + console.log("[CASE] unlock-remote unlocks and verifies state"); + // Manually lock milestone + console.log("[INFO] Manually locking milestone..."); + await updateCouchdbDoc(uri, user, password, `${dbname}/_local/obsydian_livesync_milestone`, (doc) => { + doc.locked = true; + doc.accepted_nodes = []; + return doc; + }); + + // Run unlock-remote and verify output contains verification message + const unlockOutput = await runCliCombinedOrFail(vaultDir, "--settings", settingsFile, "unlock-remote"); + assertContains( + unlockOutput, + "[Verification] Remote Database: UNLOCKED", + "unlock-remote output should contain verification status" + ); + console.log("[PASS] unlock-remote verified"); + + // 4. mark-resolved resolves and verifies state + console.log("[CASE] mark-resolved resolves and verifies state"); + // Manually lock milestone + console.log("[INFO] Manually locking milestone..."); + await updateCouchdbDoc(uri, user, password, `${dbname}/_local/obsydian_livesync_milestone`, (doc) => { + doc.locked = true; + doc.accepted_nodes = []; + return doc; + }); + + // Run mark-resolved and verify output contains verification messages + const resolvedOutput = await runCliCombinedOrFail(vaultDir, "--settings", settingsFile, "mark-resolved"); + assertContains( + resolvedOutput, + "[Verification] Remote Database: LOCKED", + "mark-resolved output should show that the remote database remains locked" + ); + assertContains( + resolvedOutput, + "ACCEPTED", + "mark-resolved output should show that the current device node is accepted" + ); + console.log("[PASS] mark-resolved verified"); + + console.log("[ALL PASS] All remote CLI commands verified successfully"); + } finally { + if (shouldStartDocker && !keepDocker) { + await stopCouchdb().catch(() => {}); + } + } +}); diff --git a/src/apps/cli/testdeno/test_dev_deno.md b/src/apps/cli/testdeno/test_dev_deno.md index 9ad15ae..8926b9c 100644 --- a/src/apps/cli/testdeno/test_dev_deno.md +++ b/src/apps/cli/testdeno/test_dev_deno.md @@ -39,6 +39,9 @@ src/apps/cli/testdeno/ test-mirror.ts test-sync-two-local-databases.ts test-sync-locked-remote.ts + test-daemon.ts + test-decoupled-vault.ts + test-remote-commands.ts ``` --- @@ -54,6 +57,9 @@ Main tasks: - `deno task test` - `deno task test:local` +- `deno task test:daemon` +- `deno task test:decoupled-vault` +- `deno task test:remote-commands` - `deno task test:push-pull` - `deno task test:setup-put-cat` - `deno task test:mirror` @@ -183,6 +189,19 @@ Both CouchDB and P2P relay flows are bash-independent. - `MINIO-enc0` - `MINIO-enc1` +### `test-daemon.ts` + +- Verifies daemon-related ignore rules behaviour. +- Exercises scenarios with `.livesync/ignore` wildcard rules, missing ignore rules, and imported `.gitignore` rules. + +### `test-decoupled-vault.ts` + +- Verifies push, pull, and mirror command behaviour when the vault directory is decoupled from the database directory. + +### `test-remote-commands.ts` + +- Verifies remote database management commands: `remote-status`, `lock-remote`, `unlock-remote`, and `mark-resolved`. + --- ## Running tests (PowerShell) @@ -198,11 +217,14 @@ deno task test:local # Individual tests deno task test:setup-put-cat deno task test:mirror +deno task test:daemon deno task test:push-pull deno task test:sync-locked-remote # CouchDB-based tests deno task test:sync-two-local +deno task test:decoupled-vault +deno task test:remote-commands deno task test:e2e-couchdb # P2P-based tests From a40929c9e4fe8e1274f310d2b52c36f892975fc6 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 8 Jun 2026 10:47:34 +0000 Subject: [PATCH 14/15] fixed: enhance conflict handling by adding settings check for document writes --- src/apps/cli/test/test-mirror-linux.sh | 9 +++++++++ src/apps/cli/testdeno/test-mirror.ts | 4 ++++ src/lib | 2 +- updates.md | 10 ++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/apps/cli/test/test-mirror-linux.sh b/src/apps/cli/test/test-mirror-linux.sh index 21a24d3..415548e 100755 --- a/src/apps/cli/test/test-mirror-linux.sh +++ b/src/apps/cli/test/test-mirror-linux.sh @@ -43,6 +43,15 @@ cli_test_init_settings_file "$SETTINGS_FILE" # isConfigured=true is required for mirror (canProceedScan checks this) cli_test_mark_settings_configured "$SETTINGS_FILE" +# Enable writeDocumentsIfConflicted to resolve unsynced conflicts during mirror +node -e ' +const fs = require("fs"); +const file = process.argv[1]; +const data = JSON.parse(fs.readFileSync(file, "utf-8")); +data.writeDocumentsIfConflicted = true; +fs.writeFileSync(file, JSON.stringify(data, null, 2)); +' "$SETTINGS_FILE" + # Preparation: Sync settings and files logic DB_SETTINGS="$DB_DIR/settings.json" cp "$SETTINGS_FILE" "$DB_SETTINGS" diff --git a/src/apps/cli/testdeno/test-mirror.ts b/src/apps/cli/testdeno/test-mirror.ts index b6eae4d..b4dc899 100644 --- a/src/apps/cli/testdeno/test-mirror.ts +++ b/src/apps/cli/testdeno/test-mirror.ts @@ -39,6 +39,10 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // isConfigured=true is required for canProceedScan in the mirror command. await markSettingsConfigured(settingsFile); + const data = JSON.parse(await Deno.readTextFile(settingsFile)); + data.writeDocumentsIfConflicted = true; + await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); + // Copy settings to the DB directory (separated-path mode) const dbSettings = workDir.join("db", "settings.json"); await Deno.copyFile(settingsFile, dbSettings); diff --git a/src/lib b/src/lib index 82e15f2..53804cb 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 82e15f2b9d99e0f595b45c864958a657d5d43bac +Subproject commit 53804cbaec7fed9591321e7fbe6dcc9092e51017 diff --git a/updates.md b/updates.md index fed04ca..5b292fb 100644 --- a/updates.md +++ b/updates.md @@ -3,6 +3,16 @@ Since 19th July, 2025 (beta1 in 0.25.0-beta1, 13th July, 2025) The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsidian-livesync/blob/main/updates_old.md). Because 0.25 got a lot of updates, thankfully, compatibility is kept and we do not need breaking changes! In other words, when get enough stabled. The next version will be v1.0.0. Even though it my hope. +## unreleased + +### Fixed (CLI, automated) + +- Fixed an issue where the mirror command could fail to apply updates when conflict preservation checks prevented overwriting unsynchronised local changes, even when the `force` parameter or `writeDocumentsIfConflicted` setting was enabled. + +### Improved + +- (CLI) Ported the remaining bash regression tests (`test-daemon-linux.sh`, `test-decoupled-vault-linux.sh`, and `test-remote-commands-linux.sh`) to Deno for cross-platform validation. + ## 0.25.74 8th June, 2026 From 5921a712278e462f5d68977be3a3013be0d8abc6 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 8 Jun 2026 10:52:00 +0000 Subject: [PATCH 15/15] Add CI --- .github/workflows/cli-deno-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 46a5e7a..eff411c 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -58,16 +58,16 @@ jobs: case "$SELECTED_TASK" in test:ci) - TASK_MATRIX='["test:setup-put-cat","test:mirror","test:push-pull","test:sync-two-local","test:sync-locked-remote","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' + TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon","test:push-pull","test:decoupled-vault","test:sync-two-local","test:sync-locked-remote","test:remote-commands","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' ;; test:p2p) TASK_MATRIX='["test:p2p-host","test:p2p-peers","test:p2p-sync","test:p2p-three-nodes","test:p2p-upload-download"]' ;; test:all) - TASK_MATRIX='["test:setup-put-cat","test:mirror","test:push-pull","test:sync-two-local","test:sync-locked-remote","test:p2p-host","test:p2p-peers","test:p2p-sync","test:p2p-three-nodes","test:p2p-upload-download","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' + TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon","test:push-pull","test:decoupled-vault","test:sync-two-local","test:sync-locked-remote","test:remote-commands","test:p2p-host","test:p2p-peers","test:p2p-sync","test:p2p-three-nodes","test:p2p-upload-download","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' ;; test:local) - TASK_MATRIX='["test:setup-put-cat","test:mirror"]' + TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon"]' ;; test:e2e-matrix) TASK_MATRIX='["test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]'