From 7c1c913f1de46957f0c3166c69d323d48d148f11 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 15 Sep 2026 12:05:24 +0000 Subject: [PATCH 01/15] fix(cli): enumerate current Vault files independently of cache --- .../cli/adapters/NodeFileSystemAdapter.ts | 15 ++- .../NodeFileSystemAdapter.unit.spec.ts | 118 ++++++++++++++++++ src/apps/cli/package.json | 2 +- updates.md | 4 + 4 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts diff --git a/src/apps/cli/adapters/NodeFileSystemAdapter.ts b/src/apps/cli/adapters/NodeFileSystemAdapter.ts index acc7596e..d386b7cf 100644 --- a/src/apps/cli/adapters/NodeFileSystemAdapter.ts +++ b/src/apps/cli/adapters/NodeFileSystemAdapter.ts @@ -92,10 +92,9 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { - if (this.fileCache.size === 0) { - await this.scanDirectory(); - } - return Array.from(this.fileCache.values()); + const files = new Map(); + await this.scanDirectoryInto("", files); + return Array.from(files.values()); } async renameFile(file: NodeFile, newPath: string): Promise { @@ -147,6 +146,10 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { + await this.scanDirectoryInto(relativePath, this.fileCache); + } + + private async scanDirectoryInto(relativePath: string, files: Map): Promise { const fullPath = this.resolvePath(relativePath); try { const directoryStat = await this.storage.stat(relativePath); @@ -160,10 +163,10 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { + const tempDirs: string[] = []; + const paths = ["a.md", "folder/b.md", "folder/sub/c.md"]; + + async function createVault() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-enumeration-")); + tempDirs.push(directory); + for (const file of paths) { + await fs.mkdir(path.dirname(path.join(directory, file)), { recursive: true }); + await fs.writeFile(path.join(directory, file), `content of ${file}`); + } + return { directory, adapter: new NodeFileSystemAdapter(directory) }; + } + + afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true }))); + }); + + it("lists every file when one file was refreshed before the first enumeration", async () => { + const { adapter } = await createVault(); + + expect(await adapter.refreshFile("folder/b.md")).not.toBeNull(); + + expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths); + }); + + it("lists every file after a path lookup without any replication", async () => { + const { adapter } = await createVault(); + + expect((await adapter.getAbstractFileByPath("folder/b.md"))?.path).toBe("folder/b.md"); + + expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths); + }); + + it("lists every file on the first enumeration without a prior path lookup", async () => { + const { adapter } = await createVault(); + + expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths); + }); + + it("excludes a deleted file after its cache entry is refreshed", async () => { + const { directory, adapter } = await createVault(); + await adapter.getFiles(); + + await fs.rm(path.join(directory, "folder/b.md")); + expect(await adapter.refreshFile("folder/b.md")).toBeNull(); + + expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(["a.md", "folder/sub/c.md"]); + }); + + it("reflects files added and deleted between enumerations", async () => { + const { directory, adapter } = await createVault(); + + expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths); + + await fs.rm(path.join(directory, "folder/b.md")); + const updatedContent = "updated content of a.md"; + await fs.writeFile(path.join(directory, "a.md"), updatedContent); + await fs.writeFile(path.join(directory, "later.md"), "content of later.md"); + + const files = await adapter.getFiles(); + expect(files.map((file) => file.path).sort()).toEqual(["a.md", "folder/sub/c.md", "later.md"]); + expect(files.find((file) => file.path === "a.md")?.stat.size).toBe(updatedContent.length); + }); + + it("returns complete listings from simultaneous calls", async () => { + const { adapter } = await createVault(); + + const originalStat = adapter.storage.stat.bind(adapter.storage); + let releaseFolderStat!: () => void; + const folderStatReleased = new Promise((resolve) => { + releaseFolderStat = resolve; + }); + let folderStatStarted!: () => void; + const folderStatStartedPromise = new Promise((resolve) => { + folderStatStarted = resolve; + }); + let pauseFolderStat = true; + const statSpy = vi.spyOn(adapter.storage, "stat").mockImplementation(async (relativePath) => { + const stat = await originalStat(relativePath); + if (pauseFolderStat && relativePath === "folder") { + pauseFolderStat = false; + folderStatStarted(); + await folderStatReleased; + } + return stat; + }); + + const firstListing = adapter.getFiles(); + let listings: Awaited>[] | undefined; + try { + await folderStatStartedPromise; + const secondListing = adapter.getFiles(); + const secondFiles = await secondListing; + releaseFolderStat(); + const firstFiles = await firstListing; + listings = [secondFiles, firstFiles]; + } finally { + releaseFolderStat(); + statSpy.mockRestore(); + } + + if (!listings) throw new Error("Expected both concurrent listings to complete"); + expect(listings.map((files) => files.map((file) => file.path).sort())).toEqual([paths, paths]); + }); + + it("returns an empty listing for an empty vault", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-enumeration-empty-")); + tempDirs.push(directory); + const adapter = new NodeFileSystemAdapter(directory); + + await expect(adapter.getFiles()).resolves.toEqual([]); + }); +}); diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 40c71da9..557e591a 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -12,7 +12,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", diff --git a/updates.md b/updates.md index 23b0dad4..a59de8a8 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,10 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Fixed + +- CLI: file enumeration now includes current files even after individual path lookups or earlier scans. + ## 1.0.28 9th September, 2026 From 70cfbd43a909faa71e903012cf6fa81631ca0e08 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 15 Sep 2026 12:06:35 +0000 Subject: [PATCH 02/15] fix(cli): prepare daemon and mirror Vaults during startup --- src/LiveSyncBaseCore.ts | 8 +- .../cli/commands/daemonCommand.unit.spec.ts | 39 +--- src/apps/cli/commands/runCommand.ts | 28 +-- src/apps/cli/main.bootstrap.unit.spec.ts | 195 ++++++++++++++++++ src/apps/cli/main.ts | 63 ++++-- src/apps/cli/package.json | 2 +- src/apps/cli/testdeno/deno.json | 1 + src/apps/cli/testdeno/run-ci-suite.ts | 1 + src/apps/cli/testdeno/test-daemon-startup.ts | 152 ++++++++++++++ src/modules/main/ModuleLiveSyncMain.ts | 8 +- .../main/ModuleLiveSyncMain.unit.spec.ts | 31 +++ updates.md | 1 + 12 files changed, 462 insertions(+), 67 deletions(-) create mode 100644 src/apps/cli/main.bootstrap.unit.spec.ts create mode 100644 src/apps/cli/testdeno/test-daemon-startup.ts diff --git a/src/LiveSyncBaseCore.ts b/src/LiveSyncBaseCore.ts index 743214fd..9870a448 100644 --- a/src/LiveSyncBaseCore.ts +++ b/src/LiveSyncBaseCore.ts @@ -38,6 +38,11 @@ export interface LiveSyncCoreFeatureViews { readonly replicationScheduling: ReplicationSchedulingControl; } +export interface StartupDatabaseOptions { + readonly ignoreSuspending?: boolean; + readonly continueOnFileFailure?: boolean; +} + type CompatibilityReplicatorView = ReplicatorInstance & Partial; export class LiveSyncBaseCore< @@ -78,7 +83,8 @@ export class LiveSyncBaseCore< ) => ServiceModules, extraModuleInitialiser: (core: LiveSyncBaseCore) => AbstractModule[], addOnsInitialiser: (core: LiveSyncBaseCore) => TCommands[], - featuresInitialiser: (core: LiveSyncBaseCore, coreFeatureViews: LiveSyncCoreFeatureViews) => void + featuresInitialiser: (core: LiveSyncBaseCore, coreFeatureViews: LiveSyncCoreFeatureViews) => void, + readonly startupDatabaseOptions: StartupDatabaseOptions = {} ) { this._services = serviceHub; this.registerReplicatorProviders(); diff --git a/src/apps/cli/commands/daemonCommand.unit.spec.ts b/src/apps/cli/commands/daemonCommand.unit.spec.ts index b99d2176..10dc251c 100644 --- a/src/apps/cli/commands/daemonCommand.unit.spec.ts +++ b/src/apps/cli/commands/daemonCommand.unit.spec.ts @@ -4,7 +4,7 @@ import { NO_INTERACTION } from "@vrtmrz/livesync-commonlib/replication"; import { runCommand } from "./runCommand"; import type { CLIOptions } from "./types"; -// Mock performFullScan so daemon tests don't require a real CouchDB connection. +// Track explicit scans: database preparation owns the daemon startup scan. vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", () => ({ performFullScan: vi.fn(async () => true), })); @@ -102,6 +102,7 @@ function createDaemonContext(core: ReturnType) { describe("daemon command", () => { beforeEach(() => { vi.restoreAllMocks(); + vi.mocked(offlineScanner.performFullScan).mockClear(); vi.useFakeTimers(); }); @@ -109,27 +110,16 @@ describe("daemon command", () => { vi.useRealTimers(); }); - it("calls performFullScan during startup", async () => { + it("does not repeat the startup scan after initial replication", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); - await runCommand(makeDaemonOptions(), createDaemonContext(core)); + expect(await runCommand(makeDaemonOptions(), createDaemonContext(core))).toBe(true); - expect(offlineScanner.performFullScan).toHaveBeenCalledTimes(1); - }); - - it("returns false when performFullScan fails", async () => { - const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(false); - - const result = await runCommand(makeDaemonOptions(), createDaemonContext(core)); - - expect(result).toBe(false); + expect(offlineScanner.performFullScan).not.toHaveBeenCalled(); }); it("polling mode: calls setTimeout when interval option is set", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); const context = createDaemonContext(core); @@ -143,7 +133,6 @@ describe("daemon command", () => { it("polling mode: applies settings with suspendFileWatching=false before setting interval", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(10), createDaemonContext(core)); @@ -156,7 +145,6 @@ describe("daemon command", () => { it("liveSync mode: calls applyPartial and applySettings", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(), createDaemonContext(core)); @@ -176,7 +164,6 @@ describe("daemon command", () => { liveSync: false, syncOnStart: false, })); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); const result = await runCommand(makeDaemonOptions(), createDaemonContext(core)); @@ -194,7 +181,6 @@ describe("daemon command", () => { liveSync: true, syncOnStart: false, })); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(), createDaemonContext(core)); @@ -205,22 +191,21 @@ describe("daemon command", () => { expect(warningCalls.length).toBe(0); }); - it("calls replicate before performFullScan", async () => { + it("completes initial replication before restoring automatic synchronisation", async () => { const core = createCoreMock(); const callOrder: string[] = []; core.services.replication.replicateUnattended = vi.fn(async () => { callOrder.push("replicate"); return { status: "completed" as const }; }); - vi.mocked(offlineScanner.performFullScan).mockImplementation(async () => { - callOrder.push("performFullScan"); - return true; + core.services.control.applySettings.mockImplementation(async () => { + callOrder.push("restoreSettings"); }); const context = createDaemonContext(core); await runCommand(makeDaemonOptions(), context); - expect(callOrder).toEqual(["replicate", "performFullScan"]); + expect(callOrder).toEqual(["replicate", "restoreSettings"]); expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({ trigger: "daemon", interaction: NO_INTERACTION, @@ -234,12 +219,11 @@ describe("daemon command", () => { status: "failed" as const, error: new Error("initial replication failed"), })); - vi.mocked(offlineScanner.performFullScan).mockClear(); const result = await runCommand(makeDaemonOptions(), createDaemonContext(core)); expect(result).toBe(false); - // performFullScan should NOT have been called + expect(core.services.control.applySettings).not.toHaveBeenCalled(); expect(offlineScanner.performFullScan).not.toHaveBeenCalled(); expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({ trigger: "daemon", @@ -249,7 +233,6 @@ describe("daemon command", () => { it("polling mode: registers onUnload handler that clears timeout", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(10), createDaemonContext(core)); @@ -265,7 +248,6 @@ describe("daemon command", () => { it("polling backoff: interval escalates on failure, caps at 300000ms, then halves on recovery", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); // startup replicate (call 1) succeeds; poll calls 2–7 fail; call 8 succeeds. let callCount = 0; @@ -320,7 +302,6 @@ describe("daemon command", () => { it("polling error handling: replicate rejection is caught and written to standard error", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); // Make replicate succeed on the initial call (startup), then fail on the poll. let callCount = 0; diff --git a/src/apps/cli/commands/runCommand.ts b/src/apps/cli/commands/runCommand.ts index c2328c7e..8f27b870 100644 --- a/src/apps/cli/commands/runCommand.ts +++ b/src/apps/cli/commands/runCommand.ts @@ -15,11 +15,6 @@ import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_b import type { CLICommandContext, CLIOptions } from "./types"; import { toArrayBuffer, toDatabaseRelativePath } from "./utils"; import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p"; -import { - performFullScan, - VaultScanResults, -} from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; -import { UnresolvedErrorManager } from "@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node"; import { writeStderrLine, writeStdoutLine } from "@/apps/cli/cliOutput"; @@ -59,9 +54,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext // accept whatever configuration the remote has. await core.services.setting.applyPartial({ disableCheckingConfigMismatch: true }, true); - // 1. Replicate the configured remote into the local database so the - // mirror scan has content to work with. - log("Replicating from remote..."); + // Database preparation has already reconciled the local database and Vault. + // Replicate before restoring automatic synchronisation. + log("Replicating with remote..."); const replResult = await core.services.replication.replicateUnattended({ trigger: "daemon", interaction: NO_INTERACTION, @@ -73,17 +68,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext replicationScheduling.markInitialOneShotSatisfied(); log("Initial replication complete"); - // 2. Mirror scan to reconcile PouchDB ↔ local filesystem. - const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events); - log("Running mirror scan..."); - const scanOk = await performFullScan(core, log, errorManager, false, true); - if (!scanOk) { - writeStderrLine(standardIo, "[Daemon] Mirror scan failed, cannot continue"); - return false; - } - log("Mirror scan complete"); - - // 3. Re-enable sync. + // Re-enable sync. const restoreSyncSettings = async () => { await core.services.setting.applyPartial( { @@ -530,9 +515,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext if (options.command === "mirror") { writeStderrLine(standardIo, "[Command] mirror"); - const log = (msg: unknown) => writeStderrLine(standardIo, `[Mirror] ${String(msg)}`); - const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events); - return (await performFullScan(core, log, errorManager, false, true)) === VaultScanResults.COMPLETED; + // Database preparation has already completed the mirror scan. + return true; } if (options.command === "remote-add") { diff --git a/src/apps/cli/main.bootstrap.unit.spec.ts b/src/apps/cli/main.bootstrap.unit.spec.ts new file mode 100644 index 00000000..91679968 --- /dev/null +++ b/src/apps/cli/main.bootstrap.unit.spec.ts @@ -0,0 +1,195 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as chokidar from "chokidar"; +import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; +import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { ServiceFileHandler } from "@/serviceModules/FileHandler"; +import { ServiceFileAccessCLI } from "./serviceModules/ServiceFileAccessImpl"; +import { runCommand } from "./commands/runCommand"; +import { createDefaultCliSettings } from "./cliSettingsDefaults"; +import { main, type CliCommandRunner } from "./main"; + +vi.mock("chokidar", { spy: true }); + +function createStandardIoMock() { + return { + readStdin: vi.fn(async () => ""), + prompt: vi.fn(async () => ""), + writeStdout: vi.fn(), + writeStderr: vi.fn(), + }; +} + +describe("CLI database preparation", () => { + const originalArgv = process.argv.slice(); + const originalExitCode = process.exitCode; + let directory: string; + let vaultPath: string; + let settingsPath: string; + let signalHandlers: Map<"SIGINT" | "SIGTERM", Set>; + let standardIo: ReturnType; + + beforeEach(async () => { + vi.mocked(chokidar.watch).mockClear(); + directory = await mkdtemp(join(tmpdir(), "livesync-cli-bootstrap-")); + vaultPath = join(directory, "vault"); + settingsPath = join(directory, "settings.json"); + await mkdir(join(vaultPath, "notes"), { recursive: true }); + await writeFile(join(vaultPath, "notes/local.md"), "local content"); + await writeFile(settingsPath, JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true })); + standardIo = createStandardIoMock(); + signalHandlers = new Map( + (["SIGINT", "SIGTERM"] as const).map((signal) => [signal, new Set(process.listeners(signal))]) + ); + process.exitCode = undefined; + vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`__EXIT__:${code ?? 0}`); + }); + }); + + afterEach(async () => { + for (const [signal, originalHandlers] of signalHandlers) { + for (const handler of process.listeners(signal)) { + if (!originalHandlers.has(handler)) process.removeListener(signal, handler); + } + } + process.argv = originalArgv.slice(); + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + await rm(directory, { recursive: true, force: true }); + }); + + async function start(command: "daemon" | "mirror" | "ls", runner: CliCommandRunner, exitCode = 1) { + process.argv = ["node", "livesync-cli", directory, "--vault", vaultPath, "--settings", settingsPath, command]; + // Daemon probes return false so the real core unloads without keeping a daemon alive. + await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`); + } + + it.each([ + { command: "daemon" as const, suspendFileWatching: false }, + { command: "mirror" as const, suspendFileWatching: false }, + { command: "mirror" as const, suspendFileWatching: true }, + ])( + "prepares the Vault before $command (watching suspended: $suspendFileWatching)", + async ({ command, suspendFileWatching }) => { + await writeFile( + settingsPath, + JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching }) + ); + await mkdir(join(vaultPath, ".livesync")); + await writeFile(join(vaultPath, ".livesync/ignore"), "*.tmp\n"); + await writeFile(join(vaultPath, "notes/ignored.tmp"), "ignored"); + const storedPaths: string[] = []; + let content: string | undefined; + const runner = vi.fn(async (_options, { core }) => { + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + const file = await core.serviceModules.databaseFileAccess.fetch("notes/local.md" as FilePathWithPrefix); + content = file ? await file.body.text() : undefined; + return false; + }); + + await start(command, runner); + + expect(runner).toHaveBeenCalledOnce(); + expect(storedPaths).toEqual(["notes/local.md"]); + expect(content).toBe("local content"); + expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content"); + } + ); + + it("runs the mirror scan once and exits without starting file watching", async () => { + const enumerate = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles"); + const watch = vi.mocked(chokidar.watch); + const runner = vi.fn(runCommand); + + await start("mirror", runner, 0); + + expect(runner).toHaveBeenCalledOnce(); + expect(enumerate).toHaveBeenCalledOnce(); + expect(watch).not.toHaveBeenCalled(); + }); + + it.each([ + { command: "daemon" as const, commandRuns: true }, + { command: "mirror" as const, commandRuns: false }, + ])("handles an individual file failure during $command preparation", async ({ command, commandRuns }) => { + const store = vi + .spyOn(ServiceFileHandler.prototype, "storeFileToDB") + .mockRejectedValue(new Error("file failed")); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + + await start(command, runner); + + expect(store).toHaveBeenCalledOnce(); + expect(runner).toHaveBeenCalledTimes(commandRuns ? 1 : 0); + expect(unload).toHaveBeenCalledOnce(); + expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content"); + }); + + it("does not import vault files for standalone database commands", async () => { + const storedPaths: string[] = []; + const runner = vi.fn(async (_options, { core }) => { + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + return false; + }); + + await start("ls", runner); + + expect(runner).toHaveBeenCalledOnce(); + expect(storedPaths).toEqual([]); + }); + + it("unloads without starting the command when database preparation fails", async () => { + vi.spyOn(ControlService.prototype, "onReady").mockResolvedValue(false); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + const settingsBefore = await readFile(settingsPath, "utf-8"); + + await start("daemon", runner); + + expect(runner).not.toHaveBeenCalled(); + expect(unload).toHaveBeenCalledOnce(); + expect(unload.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(process.exit).mock.invocationCallOrder[0]); + expect(process.exit).toHaveBeenCalledWith(1); + expect(await readFile(settingsPath, "utf-8")).toBe(settingsBefore); + }); + + it("stops the daemon when the startup scanner refuses a suspended Vault scan", async () => { + await writeFile( + settingsPath, + JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching: true }) + ); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + + await start("daemon", runner); + + expect(runner).not.toHaveBeenCalled(); + expect(unload).toHaveBeenCalledOnce(); + expect(process.exit).toHaveBeenCalledWith(1); + }); + + it("unloads when database preparation throws", async () => { + const ready = vi.spyOn(ControlService.prototype, "onReady").mockRejectedValue(new Error("scan failed")); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + + try { + await start("daemon", runner); + + expect(runner).not.toHaveBeenCalled(); + expect(unload).toHaveBeenCalledOnce(); + expect(standardIo.writeStderr.mock.calls.flat().join("")).toContain("scan failed"); + } finally { + const control = ready.mock.contexts[0]; + if (unload.mock.calls.length === 0 && control instanceof ControlService) await control.onUnload(); + } + }); +}); diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 895b5a16..1b2bff53 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -1,6 +1,6 @@ import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub"; import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage"; -import { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; +import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore"; import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules"; import { LOG_LEVEL_VERBOSE, @@ -24,6 +24,7 @@ import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/ import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { IgnoreRules } from "./serviceModules/IgnoreRules"; import { useP2PReplicatorFeature, type UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/p2p"; +import { useOfflineScanner } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; import type { ReplicationSchedulingControl } from "@/serviceFeatures/replicationScheduling"; import { createNodeStandardIo, fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node"; import type { StandardIo } from "@vrtmrz/livesync-commonlib/context"; @@ -41,6 +42,27 @@ import { } from "./settingsPersistence"; const SETTINGS_FILE = ".livesync/settings.json"; + +interface CLIVaultSyncMode { + readonly watchFiles: boolean; + readonly reflectReplicationResults: boolean; + readonly startupDatabaseOptions: StartupDatabaseOptions; +} + +// Commands which synchronise a physical Vault with the local database. +const VAULT_SYNC_MODES: Readonly>> = { + daemon: { + watchFiles: true, + reflectReplicationResults: true, + startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: true }, + }, + mirror: { + watchFiles: false, + reflectReplicationResults: false, + startupDatabaseOptions: { ignoreSuspending: true, continueOnFileFailure: false }, + }, +}; + ensureGlobalNodeLocalStorage(); defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG; @@ -296,6 +318,7 @@ export async function main( commandRunner: CliCommandRunner = runCommand ) { const options = parseArgs(standardIo); + const vaultSyncMode = VAULT_SYNC_MODES[options.command]; if (options.interval && options.command !== "daemon") { writeStderrLine( standardIo, @@ -357,9 +380,6 @@ export async function main( // Resolve vault path: mirror positional argument takes priority, // then --vault flag, otherwise fall back to databasePath. - // For daemon mode, enable chokidar file watching so the _changes feed picks up events. - // mirror runs a single full scan and doesn't need continuous watching. - const watchEnabled = options.command === "daemon"; const vaultPath = options.command === "mirror" && options.commandArgs[0] ? path.resolve(options.commandArgs[0]) @@ -385,7 +405,7 @@ export async function main( infoLog(`Settings: ${settingsPath}`); infoLog(""); let ignoreRules: IgnoreRules | undefined; - if (options.command === "daemon" || options.command === "mirror") { + if (vaultSyncMode) { ignoreRules = new IgnoreRules(vaultPath, (message, detail) => { if (detail === undefined) { writeStderrLine(standardIo, message); @@ -426,9 +446,8 @@ export async function main( } writeStderrLine(standardIo, prefix, message); }, true); - // Prevent replication result from being processed automatically in non-daemon commands. - // In daemon mode the default handler must run so changes are applied to the filesystem. - if (options.command !== "daemon") { + // Only modes which reflect replication results use the default filesystem handler. + if (!vaultSyncMode?.reflectReplicationResults) { serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => { writeStderrLine( standardIo, @@ -489,12 +508,21 @@ export async function main( const core = new LiveSyncBaseCore( serviceHubInstance, (core: LiveSyncBaseCore, serviceHub: InjectableServiceHub) => { - return initialiseServiceModulesCLI(vaultPath, core, serviceHub, ignoreRules, watchEnabled); + return initialiseServiceModulesCLI( + vaultPath, + core, + serviceHub, + ignoreRules, + vaultSyncMode?.watchFiles ?? false + ); }, (core) => [], () => [], // No add-ons (core, coreFeatureViews) => { replicationScheduling = coreFeatureViews.replicationScheduling; + if (vaultSyncMode) { + useOfflineScanner(core); + } // Register P2P replicator feature. p2pReplicator = useP2PReplicatorFeature(core); // Add target filter to prevent internal files are handled @@ -512,7 +540,7 @@ export async function main( return await Promise.resolve(true); }, -1 /* highest priority */); - // Apply user-defined ignore rules for daemon mode (lower priority, runs after dotfile check). + // Apply user-defined ignore rules after the dotfile check. if (ignoreRules) { const rules = ignoreRules; core.services.vault.isTargetFile.addHandler(async (target) => { @@ -524,7 +552,8 @@ export async function main( return true; }, 0); } - } + }, + vaultSyncMode?.startupDatabaseOptions ); if (!replicationScheduling) { throw new Error("Replication scheduling was not provided during core feature composition."); @@ -577,7 +606,7 @@ export async function main( : originalSettingsText; // Capture sync settings before suspendAllSync() clobbers them. - // Used by daemon mode to restore the correct sync behaviour after the mirror scan. + // Used by daemon mode to restore sync behaviour after initial replication. const settingsBeforeSuspend = cloneSettings(core.services.setting.currentSettings()); const durableSettingsBeforeSuspend = cloneSettings(settingsBeforeSuspend); applyStoredSetting(durableSettingsBeforeSuspend, settingsAfterLoadText, "useIndexedDBAdapter"); @@ -592,7 +621,15 @@ export async function main( }; await core.services.setting.suspendAllSync(); const settingsAfterSuspend = cloneSettings(core.services.setting.currentSettings()); - await core.services.control.onReady(); + let readyResult = false; + try { + readyResult = await core.services.control.onReady(); + } finally { + if (!readyResult) await core.services.control.onUnload(); + } + if (!readyResult) { + throw new Error("Failed to initialise LiveSync."); + } const settingsBeforeCommand = cloneSettings(core.services.setting.currentSettings()); const transientSettingKeys = changedSettingKeys(settingsBeforeSuspend, settingsAfterSuspend); for (const key of CLI_RUNTIME_ONLY_SETTING_KEYS) { diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 557e591a..e82ae069 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -12,7 +12,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json index b68a120a..30262fa6 100644 --- a/src/apps/cli/testdeno/deno.json +++ b/src/apps/cli/testdeno/deno.json @@ -5,6 +5,7 @@ "test:p2p:compose": "deno run -A --no-check run-compose-p2p.ts", "test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts", "test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts", + "test:daemon-startup": "deno test --env-file=.test.env -A --no-check test-daemon-startup.ts", "test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts", "test:remote-commands": "deno test --env-file=.test.env -A --no-check test-remote-commands.ts", "test:settings-writeback": "deno test -A --no-check test-settings-writeback.ts", diff --git a/src/apps/cli/testdeno/run-ci-suite.ts b/src/apps/cli/testdeno/run-ci-suite.ts index 9b5b0c60..7c391ce9 100644 --- a/src/apps/cli/testdeno/run-ci-suite.ts +++ b/src/apps/cli/testdeno/run-ci-suite.ts @@ -4,6 +4,7 @@ const TASKS = [ "test:setup-put-cat", "test:mirror", "test:daemon", + "test:daemon-startup", "test:push-pull", "test:decoupled-vault", "test:sync-two-local", diff --git a/src/apps/cli/testdeno/test-daemon-startup.ts b/src/apps/cli/testdeno/test-daemon-startup.ts new file mode 100644 index 00000000..8810faa1 --- /dev/null +++ b/src/apps/cli/testdeno/test-daemon-startup.ts @@ -0,0 +1,152 @@ +import { assertEquals } from "@std/assert"; +import { join } from "@std/path"; +import { TempDir } from "./helpers/temp.ts"; +import { runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts"; +import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; +import { startCliInBackground, type BackgroundCliProcess } from "./helpers/backgroundCli.ts"; +import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; + +function envOrDefault(keys: string[], fallback: string): string { + for (const key of keys) { + const value = Deno.env.get(key)?.trim(); + if (value) return value; + } + return fallback; +} + +function waitForTick(): Promise { + return new Promise((resolve) => setTimeout(resolve, 100)); +} + +async function waitForText(filePath: string, expected: string, timeoutMs = 45_000): Promise { + const deadline = Date.now() + timeoutMs; + let actual = ""; + while (Date.now() < deadline) { + try { + actual = await Deno.readTextFile(filePath); + if (actual === expected) return; + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error; + } + await waitForTick(); + } + throw new Error( + `Timed out waiting for ${filePath} to contain ${JSON.stringify(expected)}; actual=${JSON.stringify(actual)}` + ); +} + +async function waitForMissing(filePath: string, timeoutMs = 45_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + await Deno.stat(filePath); + } catch (error) { + if (error instanceof Deno.errors.NotFound) return; + throw error; + } + await waitForTick(); + } + throw new Error(`Timed out waiting for ${filePath} to be removed`); +} + +async function stopDaemon(daemon: BackgroundCliProcess | undefined): Promise { + if (!daemon) return; + await daemon.stop().catch(() => {}); +} + +Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files", async () => { + await using workDir = await TempDir.create("livesync-cli-daemon-startup"); + + const couchdbUri = envOrDefault(["COUCHDB_URI", "hostname"], "http://127.0.0.1:5989").replace(/\/$/, ""); + const couchdbUser = envOrDefault(["COUCHDB_USER", "username"], "admin"); + const couchdbPassword = envOrDefault(["COUCHDB_PASSWORD", "password"], "testpassword"); + const dbPrefix = envOrDefault(["COUCHDB_DBNAME", "dbname"], "livesync-test-db-ci"); + const dbname = `${dbPrefix}-daemon-startup-${Date.now()}-${Math.floor(Math.random() * 1_000_000)}`.toLowerCase(); + + const databaseA = workDir.join("database-a"); + const databaseB = workDir.join("database-b"); + const databaseC = workDir.join("database-c"); + const vaultA = workDir.join("vault-a"); + const vaultB = workDir.join("vault-b"); + const vaultC = workDir.join("vault-c"); + const settingsA = workDir.join("settings-a.json"); + const settingsB = workDir.join("settings-b.json"); + const settingsC = workDir.join("settings-c.json"); + + await Promise.all([ + Deno.mkdir(databaseA, { recursive: true }), + Deno.mkdir(databaseB, { recursive: true }), + Deno.mkdir(databaseC, { recursive: true }), + Deno.mkdir(vaultA, { recursive: true }), + Deno.mkdir(vaultB, { recursive: true }), + Deno.mkdir(vaultC, { recursive: true }), + ]); + + const startupPath = "notes/present-before-start.md"; + const deletePath = "notes/deleted-while-stopped.md"; + const remoteOnlyPath = "notes/remote-only.md"; + const startupFileA = join(vaultA, startupPath); + const deleteFileA = join(vaultA, deletePath); + const startupFileB = join(vaultB, startupPath); + const deleteFileB = join(vaultB, deletePath); + const remoteOnlyFileB = join(vaultB, remoteOnlyPath); + + await Deno.mkdir(join(vaultA, "notes"), { recursive: true }); + await Deno.writeTextFile(startupFileA, "created before daemon startup\n"); + const initialTime = new Date(Date.now() - 10_000); + await Deno.utime(startupFileA, initialTime, initialTime); + await Deno.writeTextFile(deleteFileA, "delete this after the first run\n"); + + let daemonA: BackgroundCliProcess | undefined; + let daemonB: BackgroundCliProcess | undefined; + try { + await startCouchdb(couchdbUri, couchdbUser, couchdbPassword, dbname); + for (const settings of [settingsA, settingsB, settingsC]) { + await initSettingsFile(settings); + await applyCouchdbSettings(settings, couchdbUri, couchdbUser, couchdbPassword, dbname, true); + } + + // A pre-existing local file must be uploaded by the daemon's startup scan. + daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon"); + await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000); + + // A separate daemon proves that the first startup replication reached CouchDB + // and that remote files are reflected into its filesystem. + daemonB = startCliInBackground(databaseB, "--vault", vaultB, "--settings", settingsB, "daemon"); + await daemonB.waitUntilContains("[Daemon] Initial replication complete", 45_000); + await waitForText(startupFileB, "created before daemon startup\n"); + await waitForText(deleteFileB, "delete this after the first run\n"); + + // Changes made while A is stopped must be found by its next startup scan. + assertEquals(await daemonA.stop(), 0, daemonA.combined); + daemonA = undefined; + await Deno.writeTextFile(startupFileA, "edited while daemon was stopped\n"); + await Deno.remove(deleteFileA); + + daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon"); + await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000); + await waitForText(startupFileB, "edited while daemon was stopped\n"); + await waitForMissing(deleteFileB); + + // Seed a file into a third local database without creating it in vault C. + // After C's finite sync, it exists only remotely from B's point of view. + await runCliWithInputOrFail( + "created in a different local database\n", + databaseC, + "--vault", + vaultC, + "--settings", + settingsC, + "put", + remoteOnlyPath + ); + await runCliOrFail(databaseC, "--vault", vaultC, "--settings", settingsC, "sync"); + await waitForText(remoteOnlyFileB, "created in a different local database\n"); + assertEquals(await Deno.readTextFile(startupFileB), "edited while daemon was stopped\n"); + assertEquals((await Deno.stat(remoteOnlyFileB)).isFile, true); + } finally { + await stopDaemon(daemonB); + await stopDaemon(daemonA); + await stopCouchdb().catch(() => {}); + } +}); diff --git a/src/modules/main/ModuleLiveSyncMain.ts b/src/modules/main/ModuleLiveSyncMain.ts index 613dd119..7cdc656a 100644 --- a/src/modules/main/ModuleLiveSyncMain.ts +++ b/src/modules/main/ModuleLiveSyncMain.ts @@ -45,7 +45,13 @@ export class ModuleLiveSyncMain extends AbstractModule { } // Ordinary start-up may continue when individual files could not be // processed. Explicit Fetch and Rebuild flows retain the strict default. - const initialisationResult = await this.services.databaseEvents.initialiseDatabase(false, false, false, true); + const { ignoreSuspending = false, continueOnFileFailure = true } = this.core.startupDatabaseOptions; + const initialisationResult = await this.services.databaseEvents.initialiseDatabase( + false, + false, + ignoreSuspending, + continueOnFileFailure + ); if (initialisationResult === VaultScanResults.FAILED) { this._log($msg("Ui.Common.LocalDatabaseInitialisationFailed"), LOG_LEVEL_NOTICE); //TODO:stop all sync. diff --git a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts index d21b8e8d..1fc7a139 100644 --- a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts +++ b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts @@ -25,6 +25,7 @@ describe("ModuleLiveSyncMain", () => { const log = vi.fn(); const host = { core: { + startupDatabaseOptions: {}, services: { appLifecycle: { onLayoutReady: vi.fn(async () => true), @@ -58,6 +59,7 @@ describe("ModuleLiveSyncMain", () => { }; const host = { core: { + startupDatabaseOptions: {}, services: { appLifecycle }, }, services: { @@ -77,4 +79,33 @@ describe("ModuleLiveSyncMain", () => { expect(result).toBe(true); expect(log).toHaveBeenCalledWith("Ui.Common.SomeFilesCouldNotBeSynchronised", LOG_LEVEL_NOTICE); }); + + it("passes strict startup database options to initialisation", async () => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: { + ignoreSuspending: true, + continueOnFileFailure: false, + }, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + databaseEvents: { initialiseDatabase }, + }, + settings: { + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false); + }); }); diff --git a/updates.md b/updates.md index a59de8a8..0cd2e254 100644 --- a/updates.md +++ b/updates.md @@ -14,6 +14,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Fixed +- CLI: daemon and mirror now scan the Vault during database initialisation, following the Obsidian startup sequence. The daemon completes this scan before replication; mirror runs the scan once and still exits with an error if any file cannot be processed. - CLI: file enumeration now includes current files even after individual path lookups or earlier scans. ## 1.0.28 From 6f0892a825c1b7707c3c13391dc489a47e5d8a21 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 15 Sep 2026 12:36:08 +0000 Subject: [PATCH 03/15] ci(cli): run the daemon startup regression in CI --- .github/workflows/cli-deno-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 889436fe..98423fcd 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -56,7 +56,7 @@ jobs: case "$SELECTED_TASK" in test:ci) - TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon","test:push-pull","test:decoupled-vault","test:sync-two-local","test:sync-locked-remote","test:remote-commands","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' + TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon","test:daemon-startup","test:push-pull","test:decoupled-vault","test:sync-two-local","test:sync-locked-remote","test:remote-commands","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' ;; test:local) TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon"]' From d082b409c08ab5c140a24db72f93db3c03a05f1e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 15 Sep 2026 13:32:55 +0000 Subject: [PATCH 04/15] test: use RustFS for S3 service fixtures --- .github/workflows/unit-ci.yml | 2 +- devs.md | 4 +- src/apps/cli/test/test-helpers.sh | 37 ++++++++--- src/apps/cli/testdeno/helpers/docker.ts | 60 ++++++++++------- .../testdeno/test-e2e-two-vaults-couchdb.ts | 26 ++++++++ src/apps/cli/testdeno/test_dev_deno.md | 7 +- src/apps/cli/util/minio-init.sh | 65 ++++++------------- src/apps/cli/util/minio-start.sh | 8 ++- src/apps/cli/util/minio-stop.sh | 2 +- test/e2e-obsidian/README.md | 4 +- test/shell/minio-init.sh | 65 ++++++------------- test/shell/minio-start.sh | 8 ++- test/shell/minio-stop.sh | 2 +- 13 files changed, 157 insertions(+), 133 deletions(-) diff --git a/.github/workflows/unit-ci.yml b/.github/workflows/unit-ci.yml index 4efee26b..a2169d2d 100644 --- a/.github/workflows/unit-ci.yml +++ b/.github/workflows/unit-ci.yml @@ -199,7 +199,7 @@ jobs: if: ${{ steps.integration_tests.outputs.present == 'true' }} run: npm run test:docker-couchdb:start - - name: Start MinIO container + - name: Start RustFS container if: ${{ steps.integration_tests.outputs.present == 'true' }} run: npm run test:docker-s3:start diff --git a/devs.md b/devs.md index 57303b12..21b1f2e4 100644 --- a/devs.md +++ b/devs.md @@ -87,9 +87,9 @@ Regression tests remain in the suite owned by the implementation under test. Plu - **CLI E2E** (`src/apps/cli/testdeno/`): Host-independent consumer workflows. The canonical Compose P2P suite covers ordinary two-peer synchronisation, replacement of the current Replicator followed by transfer with the same peer, and explicit relay disconnection followed by paused and resumed reconnection. Its lifecycle entry point is included only in the Docker test build and does not add a public CLI command. Run `npm run test:e2e:cli` for the ordinary suite or `npm run test:e2e:cli:p2p` for P2P validation. - **Self-hosted setup tools** (`utils/couchdb/`, `utils/setup/`, and `utils/flyio/`): Deno contract tests consume the exact locked Commonlib registry package, verify current CouchDB, Object Storage, and random-room P2P Setup URI defaults and remote profiles, and keep CouchDB administration separate from package-owned LiveSync database-version negotiation. `unit-ci` also provisions a real temporary CouchDB database and verifies its version document against the installed Commonlib package. Run `npm run test:setup-tools` for the local contract gate. -- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. Run focused scripts such as `npm run test:e2e:obsidian:two-vault-sync`, or use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and MinIO fixtures managed by the wrapper. +- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. Run focused scripts such as `npm run test:e2e:obsidian:two-vault-sync`, or use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and RustFS fixtures managed by the wrapper. -- **Docker Services**: Service-backed tests use CouchDB and MinIO (S3). Canonical P2P validation owns its relay through the CLI Compose runner: +- **Docker Services**: Service-backed tests use CouchDB and RustFS (S3). Canonical P2P validation owns its relay through the CLI Compose runner: ```bash npm run test:docker-all:start # Start all test services diff --git a/src/apps/cli/test/test-helpers.sh b/src/apps/cli/test/test-helpers.sh index 1e38a045..9ed34a51 100644 --- a/src/apps/cli/test/test-helpers.sh +++ b/src/apps/cli/test/test-helpers.sh @@ -307,10 +307,19 @@ cli_test_wait_for_minio_bucket() { local delay_sec=2 local i for ((i = 1; i <= retries; i++)); do - if docker run --rm --network host --entrypoint=/bin/sh minio/mc -c "mc alias set myminio $minio_endpoint $minio_access_key $minio_secret_key >/dev/null 2>&1 && mc ls myminio/$minio_bucket >/dev/null 2>&1"; then + if docker run --rm --network host --entrypoint=/bin/sh \ + rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914 \ + -c 'set -e +rc alias set myminio "$1" "$2" "$3" >/dev/null 2>&1 +rc ls "myminio/$4" >/dev/null 2>&1 +' sh "$minio_endpoint" "$minio_access_key" "$minio_secret_key" "$minio_bucket"; then return 0 fi - bucketName="$minio_bucket" bash "$CLI_DIR/util/minio-init.sh" >/dev/null 2>&1 || true + minioEndpoint="$minio_endpoint" \ + accessKey="$minio_access_key" \ + secretKey="$minio_secret_key" \ + bucketName="$minio_bucket" \ + bash "$CLI_DIR/util/minio-init.sh" >/dev/null 2>&1 || true sleep "$delay_sec" done return 1 @@ -323,26 +332,34 @@ cli_test_start_minio() { local minio_bucket="$4" local minio_init_ok=0 - echo "[INFO] stopping leftover MinIO container if present" + echo "[INFO] stopping leftover RustFS container if present" cli_test_stop_minio - echo "[INFO] starting MinIO test container" - bucketName="$minio_bucket" bash "$CLI_DIR/util/minio-start.sh" + echo "[INFO] starting RustFS test container" + minioEndpoint="$minio_endpoint" \ + accessKey="$minio_access_key" \ + secretKey="$minio_secret_key" \ + bucketName="$minio_bucket" \ + bash "$CLI_DIR/util/minio-start.sh" - echo "[INFO] initialising MinIO test bucket: $minio_bucket" + echo "[INFO] initialising RustFS test bucket: $minio_bucket" for _ in 1 2 3 4 5; do - if bucketName="$minio_bucket" bash "$CLI_DIR/util/minio-init.sh"; then + if minioEndpoint="$minio_endpoint" \ + accessKey="$minio_access_key" \ + secretKey="$minio_secret_key" \ + bucketName="$minio_bucket" \ + bash "$CLI_DIR/util/minio-init.sh"; then minio_init_ok=1 break fi sleep 2 done if [[ "$minio_init_ok" != "1" ]]; then - echo "[FAIL] could not initialise MinIO bucket after retries: $minio_bucket" >&2 + echo "[FAIL] could not initialise RustFS bucket after retries: $minio_bucket" >&2 exit 1 fi if ! cli_test_wait_for_minio_bucket "$minio_endpoint" "$minio_access_key" "$minio_secret_key" "$minio_bucket"; then - echo "[FAIL] MinIO bucket not ready: $minio_bucket" >&2 + echo "[FAIL] RustFS bucket not ready: $minio_bucket" >&2 exit 1 fi } @@ -359,4 +376,4 @@ display_test_info(){ if [[ "${LIVESYNC_TEST_DOCKER:-0}" == "1" ]]; then # shellcheck source=/dev/null source "$(dirname "${BASH_SOURCE[0]}")/test-helpers-docker.sh" -fi \ No newline at end of file +fi diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/src/apps/cli/testdeno/helpers/docker.ts index 3c0e6ce1..159d6e60 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/src/apps/cli/testdeno/helpers/docker.ts @@ -190,7 +190,7 @@ async function dockerOrFail(...args: string[]): Promise { async function stopAndRemoveContainer(container: string): Promise { await docker("stop", container).catch(() => {}); - await docker("rm", container).catch(() => {}); + await docker("rm", "-v", container).catch(() => {}); } async function cleanupTrackedContainers(reason: string): Promise { @@ -327,8 +327,22 @@ const COUCHDB_CONTAINER = "couchdb-test"; const COUCHDB_IMAGE = "couchdb:3.5.0"; const MINIO_CONTAINER = "minio-test"; -const MINIO_IMAGE = "minio/minio"; -const MINIO_MC_IMAGE = "minio/mc"; +// RustFS provides the S3 backend for the existing MINIO test mode. +const S3_IMAGE = "rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035"; +const S3_CLIENT_IMAGE = "rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914"; +const S3_BUCKET_CORS = ` + + * + GET + PUT + POST + DELETE + HEAD + * + authorization + ETag + +`; export async function stopCouchdb(): Promise { await stopAndRemoveContainer(COUCHDB_CONTAINER); @@ -454,7 +468,7 @@ export async function updateCouchdbDoc( } // --------------------------------------------------------------------------- -// MinIO +// S3 (RustFS) // --------------------------------------------------------------------------- function shQuote(value: string): string { @@ -473,9 +487,10 @@ async function initMinioBucket( bucket: string ): Promise { const cmd = - `mc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` + - `mc mb --ignore-existing myminio/${shQuote(bucket)} >/dev/null 2>&1`; - const r = await docker("run", "--rm", "--network", "host", "--entrypoint", "/bin/sh", MINIO_MC_IMAGE, "-c", cmd); + `rc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` + + `rc mb --ignore-existing myminio/${shQuote(bucket)} >/dev/null 2>&1 && ` + + `printf %s ${shQuote(S3_BUCKET_CORS)} | rc cors set myminio/${shQuote(bucket)} - >/dev/null 2>&1`; + const r = await docker("run", "--rm", "--network", "host", "--entrypoint", "/bin/sh", S3_CLIENT_IMAGE, "-c", cmd); return r.code === 0; } @@ -487,8 +502,8 @@ async function waitForMinioBucket( ): Promise { for (let i = 0; i < 30; i++) { const checkCmd = - `mc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` + - `mc ls myminio/${shQuote(bucket)} >/dev/null 2>&1`; + `rc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` + + `rc ls myminio/${shQuote(bucket)} >/dev/null 2>&1`; const check = await docker( "run", "--rm", @@ -498,7 +513,7 @@ async function waitForMinioBucket( "host", "--entrypoint", "/bin/sh", - MINIO_MC_IMAGE, + S3_CLIENT_IMAGE, "-c", checkCmd ); @@ -508,7 +523,7 @@ async function waitForMinioBucket( await initMinioBucket(minioEndpoint, accessKey, secretKey, bucket); await sleep(2000); } - throw new Error(`MinIO bucket not ready: ${bucket}`); + throw new Error(`S3 bucket not ready: ${bucket}`); } export async function startMinio( @@ -517,10 +532,10 @@ export async function startMinio( secretKey: string, bucket: string ): Promise { - console.log("[INFO] stopping leftover MinIO container if present"); + console.log("[INFO] stopping leftover S3 test container if present"); await stopMinio().catch(() => {}); - console.log("[INFO] starting MinIO test container"); + console.log("[INFO] starting RustFS test container"); await dockerOrFail( "run", "-d", @@ -532,20 +547,19 @@ export async function startMinio( "-p", "9001:9001", "-e", - `MINIO_ROOT_USER=${accessKey}`, + `RUSTFS_ACCESS_KEY=${accessKey}`, "-e", - `MINIO_ROOT_PASSWORD=${secretKey}`, + `RUSTFS_SECRET_KEY=${secretKey}`, "-e", - `MINIO_SERVER_URL=${minioEndpoint}`, - MINIO_IMAGE, - "server", - "/data", - "--console-address", - ":9001" + "RUSTFS_CONSOLE_ENABLE=true", + "-e", + "RUSTFS_CORS_ALLOWED_ORIGINS=*", + S3_IMAGE, + "/data" ); trackContainer(MINIO_CONTAINER); - console.log(`[INFO] initialising MinIO test bucket: ${bucket}`); + console.log(`[INFO] initialising S3 test bucket: ${bucket}`); let initialised = false; for (let i = 0; i < 5; i++) { if (await initMinioBucket(minioEndpoint, accessKey, secretKey, bucket)) { @@ -555,7 +569,7 @@ export async function startMinio( await sleep(2000); } if (!initialised) { - throw new Error(`Could not initialise MinIO bucket after retries: ${bucket}`); + throw new Error(`Could not initialise S3 bucket after retries: ${bucket}`); } await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket); diff --git a/src/apps/cli/testdeno/test-e2e-two-vaults-couchdb.ts b/src/apps/cli/testdeno/test-e2e-two-vaults-couchdb.ts index 0c0151ae..c112b702 100644 --- a/src/apps/cli/testdeno/test-e2e-two-vaults-couchdb.ts +++ b/src/apps/cli/testdeno/test-e2e-two-vaults-couchdb.ts @@ -60,6 +60,32 @@ export async function runScenario(remoteType: RemoteType, encrypt: boolean): Pro } try { + if (remoteType === "MINIO") { + // The shared S3 fixture also serves the browser and real Obsidian tests. + const origin = "app://obsidian.md"; + const requestedHeaders = ["authorization", "content-type", "x-amz-date", "x-amz-content-sha256"]; + const preflight = await fetch(`${minioEndpoint}/${minioBucket}`, { + method: "OPTIONS", + headers: { + Origin: origin, + "Access-Control-Request-Method": "PUT", + "Access-Control-Request-Headers": requestedHeaders.join(","), + }, + }); + await preflight.body?.cancel(); + assert(preflight.ok, "The S3 fixture must accept browser preflight requests"); + const allowedOrigin = preflight.headers.get("access-control-allow-origin"); + assert(allowedOrigin === "*" || allowedOrigin === origin, "The S3 fixture must allow the Obsidian origin"); + const allowedHeaders = (preflight.headers.get("access-control-allow-headers") ?? "") + .toLowerCase() + .split(",") + .map((header) => header.trim()); + assert(allowedHeaders.includes("authorization"), "S3 CORS must explicitly allow the Authorization header"); + assert( + preflight.headers.get("access-control-allow-methods")?.split(/,\s*/).includes("PUT"), + "S3 CORS must allow browser uploads" + ); + } await initSettingsFile(settingsA); await initSettingsFile(settingsB); await applyRemoteSyncSettings(settingsA, { diff --git a/src/apps/cli/testdeno/test_dev_deno.md b/src/apps/cli/testdeno/test_dev_deno.md index da809ce2..889a4fe2 100644 --- a/src/apps/cli/testdeno/test_dev_deno.md +++ b/src/apps/cli/testdeno/test_dev_deno.md @@ -99,11 +99,12 @@ This file corresponds to settings helpers in `test-helpers.sh`. ### `helpers/docker.ts` -- Starts, stops, and initialises CouchDB directly from Deno. +- Starts, stops, and initialises CouchDB and RustFS directly from Deno. - Configures CouchDB via `fetch + retry`. +- Initialises S3 buckets using the RustFS `rc` client, including CORS for signed browser requests. - Starts and stops the P2P relay through the same Docker runner. -Both CouchDB and P2P relay flows are bash-independent. +These flows do not require Bash on the host. The S3 matrix tasks, environment variables, and container name retain their existing `minio` names for compatibility; RustFS provides the test backend. The RustFS server and `rc` client images are pinned by version and digest. ### `helpers/backgroundCli.ts` @@ -328,7 +329,7 @@ The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` runs automati ## Current limitations -- MinIO startup and matrix coverage are ported. Current limits are elsewhere, not setup URI generation. +- S3 startup and matrix coverage use RustFS. Current limits are elsewhere, not setup URI generation. --- diff --git a/src/apps/cli/util/minio-init.sh b/src/apps/cli/util/minio-init.sh index 353832bd..98e5da31 100755 --- a/src/apps/cli/util/minio-init.sh +++ b/src/apps/cli/util/minio-init.sh @@ -1,47 +1,24 @@ #!/bin/bash set -e -cat >/tmp/mybucket-rw.json < -# -# http://localhost:63315 -# http://localhost:63316 -# http://localhost -# GET -# PUT -# POST -# DELETE -# HEAD -# * -# -# " > /tmp/cors.xml -# docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c " -# mc alias set myminio $minioEndpoint $username $password -# mc mb --ignore-existing myminio/$bucketName -# mc admin policy create myminio my-custom-policy /tmp/mybucket-rw.json -# echo 'Creating service account for user $username with access key $accessKey' -# mc admin user svcacct add --access-key '$accessKey' --secret-key '$secretKey' myminio '$username' -# mc admin policy attach myminio my-custom-policy --user '$accessKey' -# echo 'Verifying policy and user creation:' -# mc admin user svcacct info myminio '$accessKey' -# " -docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c " - mc alias set myminio $minioEndpoint $accessKey $secretKey - mc mb --ignore-existing myminio/$bucketName -" \ No newline at end of file +docker run --rm --network host --entrypoint=/bin/sh \ + rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914 \ + -c 'set -e +rc alias set myminio "$1" "$2" "$3" +rc mb --ignore-existing "myminio/$4" +rc cors set "myminio/$4" - < + + * + GET + PUT + POST + DELETE + HEAD + * + authorization + ETag + + +CORS +' sh "$minioEndpoint" "$accessKey" "$secretKey" "$bucketName" diff --git a/src/apps/cli/util/minio-start.sh b/src/apps/cli/util/minio-start.sh index 45547866..ecaaa453 100755 --- a/src/apps/cli/util/minio-start.sh +++ b/src/apps/cli/util/minio-start.sh @@ -1,2 +1,8 @@ #!/bin/bash -docker run -d --name minio-test -p 9000:9000 -p 9001:9001 -e MINIO_ROOT_USER=$accessKey -e MINIO_ROOT_PASSWORD=$secretKey -e MINIO_SERVER_URL=$minioEndpoint minio/minio server /data --console-address ':9001' \ No newline at end of file +docker run -d --name minio-test \ + -p 9000:9000 -p 9001:9001 \ + -e "RUSTFS_ACCESS_KEY=$accessKey" \ + -e "RUSTFS_SECRET_KEY=$secretKey" \ + -e "RUSTFS_CONSOLE_ENABLE=true" \ + -e 'RUSTFS_CORS_ALLOWED_ORIGINS=*' \ + rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035 /data diff --git a/src/apps/cli/util/minio-stop.sh b/src/apps/cli/util/minio-stop.sh index 08703b72..07aca610 100755 --- a/src/apps/cli/util/minio-stop.sh +++ b/src/apps/cli/util/minio-stop.sh @@ -1,3 +1,3 @@ #!/bin/bash docker stop minio-test -docker rm minio-test \ No newline at end of file +docker rm -v minio-test diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 066f097a..bacb0cad 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -125,7 +125,7 @@ The mobile pass uses Obsidian's `app.emulateMobile(true)`, a 390 by 844 CSS-pixe `test:e2e:obsidian:p2p-pane` starts one configured CouchDB-only session with no P2P profile and separate configured P2P sessions for desktop and mobile. It proves that the command remains registered while the retired command, automatic pane, and ribbon entry without a P2P configuration are absent. For the configured P2P profiles, it verifies that the desktop ribbon is available, the current status command reaches the pane without it opening at start-up, checks its connection control and horizontal layout, and captures unobstructed desktop and mobile screenshots. The mobile session uses a fresh Vault, profile, and Obsidian process, enters `app.emulateMobile(true)` through `lifecycle.beforePluginStart`, and requires the P2P view to belong to the right drawer rather than inheriting desktop workspace state. It deliberately uses no relay or peer: replacement of the active replicator is covered by focused unit tests, the Deno and Compose CLI P2P lifecycle suite covers the headless transport, and `p2p-setup-uri-workflow` owns the visible transfer path between two real Obsidian sessions. -`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Object Storage upload and Setup URI round-trip, P2P Setup URI round-trip, startup scan, provisioned CouchDB Setup URI, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB, MinIO, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run. +`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Object Storage upload and Setup URI round-trip, P2P Setup URI round-trip, startup scan, provisioned CouchDB Setup URI, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB, RustFS, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run. `test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, starts from configured plug-in data without the device-local compatibility marker, and verifies the copied-or-restored Vault explanation in the actual compatibility dialogue. It captures the summary and details, resumes explicitly, confirms that the marker was recorded, creates a note in real Obsidian, commits the note into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the metadata document and its chunk documents. @@ -160,7 +160,7 @@ LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) -- `test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service and starts with isolated Object Storage settings and the device-local compatibility acknowledgement already in place, keeping the scenario focused on upload rather than unconfigured start-up or setup. It confirms those settings through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. Adapter tests separately observe an in-progress SDK command, while this real-runtime workflow verifies the resulting request counters advance and rebalance. -`test:e2e:obsidian:object-storage-setup-uri-workflow` uses the public Commonlib-backed tool to generate the initial Setup URI for a unique MinIO prefix, completes visible initialisation on the first device, and then asks that working real Obsidian device to create a new Setup URI through the registered command. A second real Obsidian device imports only the device-generated URI. The workflow verifies the A-to-B note through explicit replication, then verifies that the B-to-A note arrives through `syncOnStart` after restarting the first device, without requesting manual replication. It captures the documented onboarding choices, and removes the Object Storage prefix only after both sessions have stopped. +`test:e2e:obsidian:object-storage-setup-uri-workflow` uses the public Commonlib-backed tool to generate the initial Setup URI for a unique Object Storage prefix, completes visible initialisation on the first device, and then asks that working real Obsidian device to create a new Setup URI through the registered command. A second real Obsidian device imports only the device-generated URI. The workflow verifies the A-to-B note through explicit replication, then verifies that the B-to-A note arrives through `syncOnStart` after restarting the first device, without requesting manual replication. It captures the documented onboarding choices, and removes the Object Storage prefix only after both sessions have stopped. `test:e2e:obsidian:p2p-setup-uri-workflow` runs two concurrent isolated real Obsidian sessions against the local Compose Nostr relay fixture. The first device imports a generated initial Setup URI and completes its signalling test with zero peers, creates a Setup URI for the second device through the registered command, and remains online while the second device imports it. The second device must select the expected online source before Fetch can rebuild its local database. The workflow accepts each connection request visibly on the receiving device, verifies the initial A-to-B fetch, checks that the menu for the three persistent per-peer actions remains within the viewport, reconnects both P2P sessions in join order, and verifies the B-to-A return journey. Every started session remains tracked until teardown completes. diff --git a/test/shell/minio-init.sh b/test/shell/minio-init.sh index 353832bd..98e5da31 100755 --- a/test/shell/minio-init.sh +++ b/test/shell/minio-init.sh @@ -1,47 +1,24 @@ #!/bin/bash set -e -cat >/tmp/mybucket-rw.json < -# -# http://localhost:63315 -# http://localhost:63316 -# http://localhost -# GET -# PUT -# POST -# DELETE -# HEAD -# * -# -# " > /tmp/cors.xml -# docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c " -# mc alias set myminio $minioEndpoint $username $password -# mc mb --ignore-existing myminio/$bucketName -# mc admin policy create myminio my-custom-policy /tmp/mybucket-rw.json -# echo 'Creating service account for user $username with access key $accessKey' -# mc admin user svcacct add --access-key '$accessKey' --secret-key '$secretKey' myminio '$username' -# mc admin policy attach myminio my-custom-policy --user '$accessKey' -# echo 'Verifying policy and user creation:' -# mc admin user svcacct info myminio '$accessKey' -# " -docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c " - mc alias set myminio $minioEndpoint $accessKey $secretKey - mc mb --ignore-existing myminio/$bucketName -" \ No newline at end of file +docker run --rm --network host --entrypoint=/bin/sh \ + rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914 \ + -c 'set -e +rc alias set myminio "$1" "$2" "$3" +rc mb --ignore-existing "myminio/$4" +rc cors set "myminio/$4" - < + + * + GET + PUT + POST + DELETE + HEAD + * + authorization + ETag + + +CORS +' sh "$minioEndpoint" "$accessKey" "$secretKey" "$bucketName" diff --git a/test/shell/minio-start.sh b/test/shell/minio-start.sh index 45547866..ecaaa453 100755 --- a/test/shell/minio-start.sh +++ b/test/shell/minio-start.sh @@ -1,2 +1,8 @@ #!/bin/bash -docker run -d --name minio-test -p 9000:9000 -p 9001:9001 -e MINIO_ROOT_USER=$accessKey -e MINIO_ROOT_PASSWORD=$secretKey -e MINIO_SERVER_URL=$minioEndpoint minio/minio server /data --console-address ':9001' \ No newline at end of file +docker run -d --name minio-test \ + -p 9000:9000 -p 9001:9001 \ + -e "RUSTFS_ACCESS_KEY=$accessKey" \ + -e "RUSTFS_SECRET_KEY=$secretKey" \ + -e "RUSTFS_CONSOLE_ENABLE=true" \ + -e 'RUSTFS_CORS_ALLOWED_ORIGINS=*' \ + rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035 /data diff --git a/test/shell/minio-stop.sh b/test/shell/minio-stop.sh index 08703b72..07aca610 100755 --- a/test/shell/minio-stop.sh +++ b/test/shell/minio-stop.sh @@ -1,3 +1,3 @@ #!/bin/bash docker stop minio-test -docker rm minio-test \ No newline at end of file +docker rm -v minio-test From 28884c3fd28f93b380ae49a6834d8a862dd9dbfd Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 15 Sep 2026 14:02:41 +0000 Subject: [PATCH 05/15] docs: credit CLI scan fix contributors --- updates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/updates.md b/updates.md index 0cd2e254..2132e944 100644 --- a/updates.md +++ b/updates.md @@ -15,7 +15,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Fixed - CLI: daemon and mirror now scan the Vault during database initialisation, following the Obsidian startup sequence. The daemon completes this scan before replication; mirror runs the scan once and still exits with an error if any file cannot be processed. -- CLI: file enumeration now includes current files even after individual path lookups or earlier scans. +- CLI: file enumeration now includes current files even after individual path lookups or earlier scans. This incorporates an adapted version of the fix proposed in PR #1188. Thank you to @YakupEmreYerli for the fix and regression tests, and to @nsanitas for the detailed report and analysis in #1143! ## 1.0.28 From 93bc161f203c5c4f2750e2ea52df80c9b3cd026a Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 15 Sep 2026 16:16:46 +0000 Subject: [PATCH 06/15] Add optional Cloudflare TURN credentials and secure profile sharing --- devs.md | 2 + .../2026_08_p2p_transport_compatibility.md | 10 +- .../design_docs/renewable_turn_credentials.md | 508 ++++++++++++++++++ docs/p2p.md | 42 +- docs/settings.md | 22 +- eslint.community.config.mjs | 7 + eslint.config.mjs | 7 + package-lock.json | 8 +- package.json | 2 +- .../BrowserP2PTransportSettings.svelte | 144 ++--- src/apps/cli/commands/runCommand.ts | 4 +- src/apps/cli/commands/runCommand.unit.spec.ts | 27 +- src/apps/cli/main.ts | 5 +- src/apps/webapp/WebAppRuntime.ts | 5 +- src/apps/webpeer/src/WebPeerRuntime.ts | 6 +- .../messages/LiveSyncProvisionalMessages.ts | 31 +- src/common/reportTool.ts | 2 + src/common/reportTool.unit.spec.ts | 41 ++ src/common/turnSettingsPrivacy.ts | 52 ++ src/common/turnSettingsPrivacy.unit.spec.ts | 71 +++ src/common/types.ts | 2 +- src/features/P2PSync/TurnConfiguration.svelte | 83 +++ .../cloudflare/iceServerSource.ts | 384 +++++++++++++ .../cloudflare/iceServerSource.unit.spec.ts | 164 ++++++ src/integrations/cloudflare/settings.ts | 87 +++ src/integrations/iceServerSources.ts | 85 +++ .../iceServerSources.unit.spec.ts | 39 ++ src/main.ts | 4 +- .../ModuleObsidianSettingAsMarkdown.ts | 13 + .../SettingDialogue/PaneRemoteConfig.ts | 11 + .../SetupWizard/dialogs/SetupRemoteP2P.svelte | 55 +- .../SetupWizard/dialogs/UseSetupURI.svelte | 14 +- src/serviceFeatures/setupObsidian/qrCode.ts | 10 +- .../setupObsidian/qrCode.unit.spec.ts | 14 + .../setupObsidian/setupProtocol.ts | 11 +- .../setupObsidian/setupProtocol.unit.spec.ts | 14 + src/serviceFeatures/setupObsidian/setupUri.ts | 9 +- src/serviceFeatures/useIceServerSources.ts | 17 + test/apps/webapp/WebAppRuntime.unit.spec.ts | 2 +- .../webpeer/browser-smoke.test.ts | 23 + 40 files changed, 1855 insertions(+), 182 deletions(-) create mode 100644 docs/design_docs/renewable_turn_credentials.md create mode 100644 src/common/reportTool.unit.spec.ts create mode 100644 src/common/turnSettingsPrivacy.ts create mode 100644 src/common/turnSettingsPrivacy.unit.spec.ts create mode 100644 src/features/P2PSync/TurnConfiguration.svelte create mode 100644 src/integrations/cloudflare/iceServerSource.ts create mode 100644 src/integrations/cloudflare/iceServerSource.unit.spec.ts create mode 100644 src/integrations/cloudflare/settings.ts create mode 100644 src/integrations/iceServerSources.ts create mode 100644 src/integrations/iceServerSources.unit.spec.ts create mode 100644 src/serviceFeatures/useIceServerSources.ts diff --git a/devs.md b/devs.md index 21b1f2e4..d95d0a68 100644 --- a/devs.md +++ b/devs.md @@ -189,6 +189,8 @@ steps required to add a built-in provider. Commonlib owns one stable `LiveSyncP2PService`, its `P2PRoomSessionOwner`, and the replaceable Trystero room session. Host commands, event handlers, and views consume the focused transport, connection-probe admission, directory, peer-admission, transfer, change-relay, configuration, and diagnostic views returned by the service feature. They must not retain the deprecated compatibility Replicator as an ordinary service locator, close Trystero-owned raw peers, or install another Trystero transport generation at the application root. The exact implemented ownership and shutdown boundaries are recorded in Commonlib's [P2P transport lifecycle](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/p2p-transport-lifecycle.md) design document. +The proposed [TURN credential sources design](docs/design_docs/renewable_turn_credentials.md) covers credential expiry in the existing room reuse decision, replication continuation after room replacement, persisted and shared provider tokens, report redaction, and optional integrations on the device. It records the Commonlib work and compatibility boundaries before implementation. + ### Conflict Merge Policy Markdown conflict auto-merge should behave like a conservative three-way merge. The guiding rule is to merge changes when they touch non-overlapping regions, and to keep a manual conflict when the edits overlap semantically. diff --git a/docs/adr/2026_08_p2p_transport_compatibility.md b/docs/adr/2026_08_p2p_transport_compatibility.md index c9691c28..004c6e5d 100644 --- a/docs/adr/2026_08_p2p_transport_compatibility.md +++ b/docs/adr/2026_08_p2p_transport_compatibility.md @@ -60,7 +60,15 @@ The first settings revision retains the existing storage and dialogue contract o A future interface may present the existing comma-separated value as ordered `turn:` and `turns:` URL rows without changing its serialised representation. A structured list of multiple credential profiles is deferred until a provider or self-hosted use case requires different credentials in the same P2P profile. -Static long-term credentials are the supported first stage. Managed providers may return short-lived credentials, but LiveSync must not store a provider API token or a Coturn shared authentication secret. A future managed-credential design needs a separately trusted HTTPS endpoint, expiry handling, refresh behaviour, failure reporting, and a clear Setup URI policy. It is not represented as another static password field. +Static long-term credentials are the supported first stage. Managed credentials use an optional source implementation on the device, behind a service-independent acquisition contract. Service-specific requests and settings belong under `src/integrations/`; Commonlib owns acquisition coordination and the P2P lifecycle. A separately operated HTTPS credential endpoint is an optional future source, not a prerequisite. + +A user-supplied provider API token is persisted as a sensitive P2P profile setting and included in encrypted Setup URI sharing, so that participating devices can use the same configuration without repeated token entry. Optional configuration encryption must cover every saved copy. Reports and logs redact the complete provider configuration and issued credentials, including inactive profiles and settings projections. Coturn's server-side shared authentication secret remains outside client settings. + +Issued short-lived TURN credentials and their expiry remain in memory. The existing room reuse decision checks both the effective connection settings and credential validity. When reconciliation finds expired credentials, it uses the normal room retirement and replacement path with newly acquired credentials. Replacement may cancel an in-progress transfer; the next replication attempt uses stored checkpoints and revision comparison to retain received progress. Whether that next attempt starts automatically follows the existing synchronisation policy. + +Time passing alone does not trigger acquisition or disconnection. This design adds no renewal timer, per-peer acquisition hook, configuration update on raw peers, or credential-driven ICE restart. Internal peer reconnection within an unchanged room does not guarantee fresh issuance. Acquisition failure is reported without changing the selected source or route policy. See [TURN credential sources](../design_docs/renewable_turn_credentials.md) for the proposed contract, persistence and sharing formats, room replacement, and verified replication continuation behaviour. + +When managed sources are introduced, relay-only validation accepts a valid managed TURN source configuration as well as the existing manual URL list. Failure to acquire usable TURN entries keeps relay-only mode selected and reports the connection failure; it does not restore `Automatic` silently. ### TURN allocation check and route diagnostics diff --git a/docs/design_docs/renewable_turn_credentials.md b/docs/design_docs/renewable_turn_credentials.md new file mode 100644 index 00000000..474142a0 --- /dev/null +++ b/docs/design_docs/renewable_turn_credentials.md @@ -0,0 +1,508 @@ +--- +date: 2026-09-15 +commonlib-version: "0.1.25-dev.turn-credentials.3" +self-hosted-livesync-version: "1.0.28" +status: unreleased +--- + +# TURN credential sources + +## Purpose and decisions + +This developer design addresses [Issue #1182](https://github.com/vrtmrz/obsidian-livesync/issues/1182) +through a service-independent interface for acquiring TURN credentials. +The [P2P transport compatibility ADR](../adr/2026_08_p2p_transport_compatibility.md) +records the accepted policy. The contract, lifecycle, settings, and host +integration are implemented locally. Real provider issuance, relay-only +Obsidian synchronisation, and synchronisation after explicit reconnection have +been verified. Expiry-driven TURN reconnection remains release validation work. + +The design uses these decisions: + +- Acquire credentials on the device through an optional service integration. +- Persist the user-supplied provider API token with the P2P profile and include + it in encrypted Setup URI sharing for additional devices. +- Redact provider configuration and issued credentials from reports and logs. +- Keep issued short-lived credentials in memory only. +- Keep a local expiry alongside issued credentials and check it in the + existing room reuse decision. +- When that decision finds expired credentials, acquire a new configuration + and use the existing room replacement lifecycle. Replacement may cancel + an in-progress transfer; the next replication attempt reuses stored progress. +- Check expiry when the room lifecycle is reconciled. Add no renewal timer, + per-peer acquisition hook, `setConfiguration()`, or credential-driven ICE + restart. + +Manual TURN configuration remains supported without a provider account. +Cloudflare is the first optional integration. A separate credential endpoint, +a general authentication framework, runtime extension loading, and migration +of existing service integrations are outside the first delivery. + +## Ownership and composition + +An **ICE server source**, represented by `IceServerSource`, supplies ICE server +URLs, access credentials, and their expiry. This is developer vocabulary for +the acquisition contract; it is separate from a Replicator provider. + +| Component | Responsibility | Owner | +| --- | --- | --- | +| Source contract | Acquisition result, validation, and safe failure categories | Commonlib | +| Credential state and room reuse | Memory cache, expiry check, acquisition, cancellation, and room replacement | Commonlib `P2PRoomSessionOwner` | +| Physical peer creation | Use the configuration supplied when joining the room | Existing Trystero implementation | +| Source catalogue and settings | Explicit source selection and host dependencies | LiveSync | +| Cloudflare source | Provider request, response conversion, and configuration validation | LiveSync `src/integrations/cloudflare/` | + +Implementation placement: + +```text +Commonlib + P2P source contract and private credential cache + Expiry check in the existing room owner and session construction + +LiveSync + src/integrations/iceServerSources.ts + src/integrations/cloudflare/iceServerSource.ts + src/integrations/cloudflare/settings.ts + src/serviceFeatures/useIceServerSources.ts +``` + +`integrations/` groups code which connects external services to the common +contract. It does not imply a hosted project service or a public extension +marketplace. The service feature composes a closed catalogue of source +factories with explicit dependencies, following +[Service feature and legacy Module boundaries](service_feature_and_legacy_module_boundaries.md). +An integration receives neither `LiveSyncBaseCore` nor ownership of replication. + +Supply the catalogue through an optional composition argument to +`useP2PReplicatorFeature`, preserving its manual-only default for existing +Commonlib consumers. Factories validate settings without network access; +acquisition runs only when requested by the P2P owner. Unsupported sources +produce an explicit configuration error. + +```mermaid +flowchart LR + R["Existing room lifecycle reconciliation"] --> D{"Same binding and valid credentials?"} + D -->|"Yes"| K["Keep current room"] + D -->|"No"| C["Retire current room, if present"] + C --> A["Reuse valid cached credentials or acquire"] + A --> O["Open room with resolved ICE configuration"] +``` + +## Settings and dependencies + +Present a `TURN configuration` choice with `Manual` and `Cloudflare`. +The catalogue supplies each integration's label and fields; the common P2P +engine does not branch on a service name. + +| Input | Manual | Cloudflare | +| --- | --- | --- | +| TURN server URLs | Existing field | Supplied by the API | +| TURN username and credential | Existing fields | Issued in memory | +| TURN Key ID | Unused | Required and persisted | +| TURN Key API Token | Unused | Required, masked in the dialogue, and persisted | + +The first Cloudflare implementation requests a 24-hour lifetime internally. +It needs no account ID, email address, custom endpoint URL, or renewal interval +setting. This lifetime is a design default, not a provider default. + +Dependencies are an injected HTTP operation, a clock, cancellation/deadline +handling, and the existing settings and P2P lifecycle services. No Cloudflare +SDK, credential broker, or new operating-system secret-store dependency is +required. + +Retain `P2P_turnServers`, `P2P_turnUsername`, and `P2P_turnCredential` for manual +configuration. An absent source selection means manual. Add a versioned P2P +profile descriptor, `P2P_iceServerSource`: + +```json +{ + "version": 1, + "id": "cloudflare", + "configuration": { + "turnKeyId": "user-supplied-key-id", + "apiToken": "user-supplied-turn-key-api-token" + } +} +``` + +Commonlib owns the JSON envelope; each source owns validation of its +configuration. Unsupported identifiers and versions remain preserved in +storage and produce an explicit unsupported result when selected. Loading an +inactive profile performs no acquisition. + +The selected source configuration, including token changes, participates in +the effective P2P configuration identity. Issued credentials and their expiry +are separate runtime state. Room reuse requires both a matching identity and +usable credentials. Under managed selection, unused manual credentials do +not affect that identity; manual selection preserves the existing projection. +Keep the identity opaque and absent from diagnostics. Apply source changes +and expired runtime credentials through the existing room replacement policy. + +## Persistence, sharing, and redaction + +The API token is an ordinary sensitive connection setting. Persist it with +the profile so that restarting a device and configuring another device do +not require re-entry. This does not claim operating-system keychain storage. +When optional configuration encryption is enabled, cover both the saved +profile URI and any top-level settings projection containing the source. +Failure to encrypt either copy must leave the prior saved settings intact +and report a safe error; it must not silently save a plaintext replacement. + +| Destination | Provider API token | Issued TURN username and credential | +| --- | --- | --- | +| Saved P2P profile | Included | Omitted | +| Encrypted Setup URI | Included with the source and Key ID | Omitted | +| Runtime room configuration | Available only to the source | Cached in memory and passed to WebRTC | +| General report or diagnostic log | Redacted | Redacted | + +Encrypted Setup URI sharing is the complete sharing route for managed +profiles. Preserve the independent main-remote and P2P selections and the +receiving device's own peer name. Raw profile and unencrypted QR copy actions +should offer encrypted Setup URI sharing when their output includes a managed +source, including one in an inactive profile. Never substitute a temporary +TURN password or silently export a profile missing its API token. + +Markdown settings export must not leak tokens through either the top-level +source or a profile URI. For this first delivery, omit the profile collection, +its selections, and the source projection together when managed profiles are +present, and explain that connection sharing uses the encrypted Setup URI. +Importing Markdown without that group preserves the local profiles and +selections rather than replacing them with a filtered collection. + +Reports expose only safe source labels and acquisition state. Redact the +entire opaque source configuration, including unknown source configurations, +and every stored or projected copy. Preserve the existing scheme-only +redaction of profile URIs in `src/common/reportTool.ts`. Do not log request +headers, raw API bodies, source identity values, or HTTP errors which embed +credentials. Use one redaction policy across report and diagnostic paths; +cover inactive profiles and encoded values in tests. + +## Acquisition contract + +Commonlib exports the acquisition contract from `/p2p`: + +```typescript +type IceServerConfiguration = { + iceServers: readonly RTCIceServer[]; + expiresAt: number | null; +}; + +declare class IceServerSourceError extends Error { + constructor( + code: "configuration" | "authentication" | "unavailable" | "invalid-response", + message: string, + retryable: boolean + ); +} + +interface IceServerSource { + acquire(signal: AbortSignal): Promise; +} +``` + +`expiresAt` is a local Unix timestamp in milliseconds. `null` represents +non-expiring manual configuration; managed results require a finite expiry. +Sources throw a typed, safe failure or propagate cancellation. The room owner +calls the same operation when it needs an initial or replacement credential +set. A source does not save settings, schedule renewal, mutate peers, or +start replication. + +Validate supported `stun:`, `stuns:`, `turn:`, and `turns:` URLs, complete TURN +credentials, bounded response size and entry count, and enough remaining +lifetime for connection establishment. A managed TURN source must return at +least one usable TURN entry. Copy the validated result before handing it to +WebRTC; unknown fields never become arbitrary `RTCConfiguration` options. +Preserve ordinary STUN behaviour and the selected connection-path policy. + +### Cloudflare request + +The source calls the fixed provider API: + +```http +POST https://rtc.live.cloudflare.com/v1/turn/keys/{TURN_KEY_ID}/credentials/generate-ice-servers +Authorization: Bearer {TURN_KEY_API_TOKEN} +Content-Type: application/json + +{"ttl":86400} +``` + +Cloudflare returns an `iceServers` array. Its documented maximum lifetime is +48 hours, and the returned ICE server structure has no TTL. Derive the local +expiry from the requested TTL and the time before the request started, +allowing for request duration and a connection-establishment margin. Reject a +response which has already become too old. See +[credential generation](https://developers.cloudflare.com/realtime/turn/generate-credentials/) +and [the TURN FAQ](https://developers.cloudflare.com/realtime/turn/faq/). + +Only the Key ID, API token, and requested lifetime go to the provider. The +source has no need for a Vault passphrase, Group ID, peer name, or file data. +Use a TURN Key API Token, not an account-wide API key. Cloudflare documents a +server-side secret model; this design explicitly permits users to place and +share their own issuance token on their participating devices. Whoever +receives that token can issue credentials under its authority. + +All maintained hosts inject `API.webCompatFetch`, using standard fetch +cancellation and redirect controls. The source refuses redirects, omits cookies, +requests `no-store`, and applies a 15-second deadline. It bounds the response to +32 KiB, 16 ICE entries, and 32 URLs. Commonlib independently validates the +result and requires at least 30 seconds of remaining lifetime before use. + +A read-only CORS preflight on 15 September 2026 returned HTTP 204 and allowed +POST, `Authorization`, and `Content-Type` from the requested origin. This +establishes preflight support, not successful authenticated issuance. Obsidian's +`nativeFetch` adapter is not used here because its `requestUrl` path does not +forward all required fetch controls. Provider HTTP behaviour is covered by +fixtures; operator-owned credentials are still required for real issuance and +TURN allocation validation. + +## Room reuse and credential expiry + +### Runtime state and decision + +Keep one private cached result for the effective source configuration in the +P2P room owner. It contains the validated ICE servers, `expiresAt`, and the +source identity which produced them. Reuse it while that source still matches +and its remaining lifetime is sufficient. Clear it on source change, explicit +disconnect, suspension, or owner disposal. Neither the credentials nor the +expiry becomes a persisted setting. + +`expiresAt` is derived from issuance time and the requested TTL. A fixed TTL +value alone cannot identify whether an earlier issuance has expired. Keep the +expiry check separate from the stable settings signature rather than making +wall-clock time an ordinary configuration field. + +The existing `reconcileTransport()` reuse decision becomes conceptually: + +```typescript +const reusable = + current?.host.isServing && + bindingsMatch(activeBinding, desiredBinding) && + credentialsRemainUsable(activeCredentials, now); +``` + +Manual configuration has no managed expiry and preserves the existing +behaviour. For a managed source, a missing or expired result makes the room +ineligible for reuse even if the saved settings have not changed. + +When reuse is unavailable, use the existing lifecycle queue: + +1. Retire the current session, if present. Its cancellation and settlement + path also handles any in-progress transfers. +2. Resolve valid cached credentials for the desired source, or await a new + `acquire()` result. Serialised reconciliation shares this work rather than + issuing a request for each physical peer. +3. Construct the replacement session with a temporary, resolved ICE + configuration. Keep that configuration separate from persisted manual + fields and the settings projection used for policy changes. +4. Before publishing the session, recheck the source identity, expiry, + enabled state, and room demand. Discard obsolete results and candidates. + +Acquisition and room opening have bounded deadlines. A result which expires +before publication is unusable. Each reconciliation makes one acquisition +attempt; a later explicit retry or existing reconciliation can try again. A credential test uses its own result and does not replace +the active room's cache. + +### When the check runs + +Use existing reconciliation opportunities, including explicit connection, +changes to room demand, and applicable settings/lifecycle events. Time passing +alone does not run reconciliation or close a room. If reconciliation runs +after expiry, ordinary replacement may interrupt a transfer; no additional +idle wait or transfer-preservation mechanism is required. + +Not every operation passes this decision. A transfer admitted directly by an +existing session, a signalling WebSocket reconnect, and Trystero's internal +physical-peer reconnection can proceed without owner reconciliation. This +scope checks credential validity during room reconciliation and acquires a +new set when needed. Individual physical connection attempts use the room's +existing configuration. +A room which remains open beyond expiry may require an explicit reconnect +before new TURN-dependent peers can connect. + +### Existing transport boundary + +The inspected baseline is Commonlib `0.1.24` and Trystero `0.25.3`, as pinned +in the LiveSync lockfile: + +| Package boundary | Relevant behaviour | +| --- | --- | +| Commonlib `P2PRoomSessionOwner.reconcileTransport()` | Reuses an equivalent serving room; otherwise retires it and constructs another session. | +| Commonlib `P2PRoomSession.retire()` | Rejects new work, cancels current finite operations, waits for settlement, and disposes the room. | +| Commonlib `TrysteroReplicatorP2PServer.start()` | Supplies resolved options to Trystero when joining the room. | +| Trystero `dist/strategy.mjs` and `dist/offer-pool.mjs` | The final room leave destroys the outgoing offer pool; a later join can use new options. | +| Trystero `dist/shared-peer.mjs` | Live physical peers may survive logical room leave/rejoin under Trystero ownership. | + +Use the normal retire-before-open path. LiveSync does not close raw peers or +create another transport generation. The design requires no Trystero peer +factory extension, eager-pool change, or existing-peer configuration update. +Verify fresh TURN allocation after normal room replacement in the maintained +host topology; a still-connected shared peer can remain usable and is not +proof that a fresh allocation used the new credentials. This assumes one +active P2P room per host; pool replacement while another room remains open +needs separate validation. + +### Replication after interruption + +Commonlib `0.1.24` uses `replicateShim()` for P2P transfer. Its checkpoint is +stored in database-local documents, using the source and destination database +names and a source-side marker. The Trystero peer ID is not the checkpoint +identity. Rejoining the same databases with a new peer ID therefore retains +replication progress. + +For each batch, the shim reads changes, compares destination revisions with +`revsDiff`, fetches missing revisions, writes them with `new_edits: false`, +and invokes the processing callback before advancing the checkpoint. Room +retirement does not delete the database documents or replication checkpoints. + +Consequently, the next replication attempt starts at the last committed +checkpoint. If interruption or a lost response leaves writes beyond that +checkpoint, it may scan that batch again; revision comparison avoids fetching +already stored revisions again. Missing or incomplete document revisions are +retried. This preserves received Metadata and Chunks, but does not resume a +partially received network message at its last byte. Normal P2P calls use +`rewind: false`; database replacement, removed checkpoint state, or an explicit +rewind can require an earlier scan. + +Starting that next attempt follows existing synchronisation policy. An +unfinished AutoSync baseline remains eligible when an accepted matching peer +is advertised again: `P2PAutomationCoordinator` only records completed +baselines. A cancelled manual transfer does not automatically restart merely +because the room reconnects; the next requested synchronisation uses the +same stored progress. This feature adds no universal transfer retry loop and +does not report a cancelled attempt as successful. + +A focused check executed the pinned `ReplicatorShim.js` with in-memory +database boundaries and confirmed both cancellation after a committed batch +and loss of completion after writes but before the checkpoint. Both subsequent +attempts fetched only missing revisions. The pinned automation coordinator +also allowed another attempt after a cancelled baseline. These checks verify +the algorithms; they do not establish real WebRTC reconnection or file +reflection behaviour, which remains part of implementation validation. + +### Failure and cancellation + +Acquisition failure leaves the attempted room opening unavailable and reports +a safe, actionable state. Do not fall back to saved manual credentials, +choose another provider, or relax relay-only mode. Authentication and +configuration errors wait for correction or an explicit retry. Transient +failures are marked retryable for the existing lifecycle or an explicit retry; +this source adds no automatic acquisition or reconnect loop. + +Explicit disconnect, source changes, and application suspension invalidate +pending acquisition. A late HTTP result cannot publish a room or restore an +obsolete source. Cancellation must take effect while room opening awaits +acquisition rather than waiting behind it in the lifecycle queue. The owner +rechecks current demand and configuration before exposing a replacement. + +## Compatibility and verification + +### Stored settings and sharing formats + +Update Commonlib's P2P setting type, `pickP2PSyncSettings`, connection-string +parser, Setup URI processing, and settings encryption together. Update the +LiveSync Setup dialogue, import handler, profile export, Markdown settings, +and report paths. Existing fixed-field serialisers would otherwise discard +the source. Generated credentials never populate the manual fields. + +Managed profile strings need a distinguishable format, +`sls+p2p-v2://`. Commonlib `0.1.24` rejects that scheme, whereas it silently +drops unknown fields in ordinary `sls+p2p://` strings. Manual profiles retain +their current format. Validate the source before activation; unknown sources +must not become manual connections. + +Full encrypted Setup URIs also need a distinguishable outer format, +`obsidian://setuplivesync-v2?settings=`, and a versioned encrypted +envelope when managed profiles are included. The old import path decrypts and +merges arbitrary JSON, so a nested profile version alone is insufficient. +Validate the new envelope before applying settings in every maintained host. +Apply stored settings schema checks on load and import, including downgrades; +older clients must not activate a managed profile after dropping its source. +Document any minimum-client and downgrade requirements with the implementation. + +For a selected managed source, save the complete P2P connection in its +versioned profile and disable the persisted legacy P2P projection: clear its +Group ID and passphrase, and save `P2P_Enabled` and `P2P_AutoStart` as false. +A compatible client restores those runtime values from the selected profile. +This prevents an older client which rejects the profile URI from connecting +through leftover manual fields. Source-only settings without a configured +room can remain disabled until setup is complete. The live settings and +setting-saved notifications retain their usable runtime values. A selected +manual profile retains its established persisted representation, even when +another saved profile has a managed source. + +The P2P data protocol and Group ID remain unchanged. A peer using manually +configured TURN can communicate with one using issued credentials; validate +that interoperability without requiring both peers to use the same issuer. + +### Real-provider verification + +On 15 September 2026, the local LiveSync build with Commonlib +`0.1.25-dev.turn-credentials.3` passed a real Cloudflare TURN check in two +isolated Obsidian 1.12.7 instances on one Linux host. Both instances used the +Cloudflare source and `P2P_connectionPath: "relay"`, with a local Nostr relay +used only for signalling. + +- The source received HTTP 201 responses and acquired credentials with a + requested 24-hour lifetime. The Obsidian instances also received successful + issuance responses through their own HTTP integration. +- Both endpoints reported selected local and remote candidates of type + `relay`, using UDP, before transferring a note. The receiving Vault contained + the expected note content after replication completed. +- Explicitly disconnecting one instance removed its peer advertisement from + the other. Reconnecting issued credentials again and established a new + relay-only connection. A second note then travelled in the reverse direction + and appeared with the expected content in the receiving Vault. + +This check covers initial provider issuance, real relayed replication, and +credential reacquisition after an explicit disconnect. It does not establish +natural TTL expiry, interruption within a replication batch, mobile operating +system behaviour, mixed manual/managed peers, or connectivity between different +networks. Those cases retain their separate validation requirements. The +results contain no provider token, TURN username, or TURN credential. + +### Acceptance criteria for implementation + +- Manual configuration, default STUN, and existing Setup URIs retain their + behaviour. Unsupported managed sources fail explicitly. +- Provider tokens survive restart, profile selection, optional configuration + encryption, and encrypted Setup URI sharing. Reports and logs reveal no + tokens or issued credentials, including inactive and encoded copies. +- Issued credentials never enter persisted settings, exports, or reports. +- Equivalent settings and valid credentials reuse the room. Expired + credentials cause the next owner reconciliation to acquire and replace + through the existing lifecycle; manual settings retain their behaviour. +- Concurrent reconciliation does not duplicate acquisition. Late responses + after disconnect, source change, or suspension cannot publish a room. + Expiry tests cover delayed responses and clock changes. +- Time passing alone triggers no acquisition or replacement. There is no + per-peer acquisition hook, `setConfiguration()`, or credential-driven ICE + restart. +- Replacement during a batch settles the old attempt and preserves stored + documents and checkpoints. The next attempt transfers missing revisions; + test interrupted AutoSync and explicit manual retry separately. +- Safe failures cover authentication, rate limits, network errors, timeouts, + and malformed responses without an automatic source or route-policy change. +- Real relay-only connections verify initial establishment and room + replacement after expiry, including mixed manual/managed peers and both + initiator roles. A selected relayed candidate pair is required evidence; + direct traffic alone does not validate TURN use. +- Real Obsidian checks cover HTTP behaviour, desktop/mobile lifecycle, + persistence/sharing, and a file round trip after reconnection. Validate + supported CLI/browser hosts before enabling their direct integration. + +The existing Setup connection check remains a signalling check. Credential +issuance, a disposable TURN allocation check, actual peer data transfer, and +LiveSync file synchronisation establish different facts. Tests and status +must identify which boundary they verify. + +Implement the Commonlib contract, settings, runtime expiry, and existing room +replacement integration in its own repository. Validate the packed Commonlib +artefact before updating LiveSync's exact dependency and composing the +Cloudflare source. Use deterministic provider fixtures and an open-source +Coturn test service for repeatable +coverage; verify the real provider path with operator-owned test credentials. + +Run Commonlib checks, LiveSync `npm run check`, unit tests, builds, and focused +consumer tests for the implementation. Deterministic source, lifecycle, persistence, sharing, and redaction tests +cover the implemented boundaries. Real provider allocation and host +reconnection evidence must be recorded separately before release. diff --git a/docs/p2p.md b/docs/p2p.md index 23253771..1184409c 100644 --- a/docs/p2p.md +++ b/docs/p2p.md @@ -18,7 +18,7 @@ flowchart LR The signalling relay and TURN server have different roles: - The **signalling relay** is required for peer discovery and connection negotiation. LiveSync uses Nostr-compatible WebSocket relays for this role. The relay does not store or transfer Vault contents. -- A **TURN server** is an optional fallback. WebRTC uses it to relay the encrypted peer connection only when the devices cannot establish a direct path through their networks. +- A **TURN server** is an optional fallback. WebRTC uses it to relay the encrypted peer connection when the devices cannot establish a direct path through their networks, or whenever **TURN relay only** is selected. ## The project's public signalling relay @@ -40,16 +40,50 @@ Both settings contain server addresses, but they are not interchangeable. | Setting | Required | Carries Vault contents | Purpose | | --- | --- | --- | --- | | **Signalling relay URLs** | Yes | No | Finds peers and exchanges the information needed to establish WebRTC connections. | -| **TURN server URLs** | Only when direct WebRTC connectivity fails | Encrypted WebRTC traffic | Relays traffic between peers when NAT or firewall rules prevent a direct path. | +| **TURN server URLs** | When direct WebRTC connectivity fails or **TURN relay only** is selected | Encrypted WebRTC traffic | Relays traffic between peers when NAT or firewall rules prevent a direct path. | -A TURN provider cannot read LiveSync's encrypted Vault contents, but it can observe connection metadata and traffic volume. Use a provider you trust. The project does not operate an official TURN service. +WebRTC encrypts data between the devices, including when it passes through TURN. The TURN provider cannot read the transferred data, but it can observe network addresses and traffic volume. This transport encryption also applies when LiveSync's optional database encryption is disabled. The project does not operate an official TURN service. + +## TURN credentials + +In **TURN configuration**, select **Manual** to enter your own TURN server URLs, +username, and credential, or select **Cloudflare** to enter a **TURN Key ID** and +**TURN Key API Token**. Cloudflare is optional; the project does not require a +particular TURN provider or operate a credential broker. See Cloudflare's +[credential instructions](https://developers.cloudflare.com/realtime/turn/generate-credentials/) +for creating a TURN key and its API token. + +The API token is saved with the P2P profile. Use an encrypted Setup URI to share +it with your other devices. Managed profiles use the versioned Setup URI format +and require a client which supports that format; update receiving devices +before importing it. Older clients leave the saved managed P2P connection +disabled. Select and save a manual TURN configuration in a compatible client +before downgrading if P2P must remain usable. Plain QR export redirects to +encrypted Setup URI sharing. +Markdown settings omit the connection profile group when it contains a managed +TURN source, including inactive profiles, and importing those omitted settings +preserves this device's existing profiles. Diagnostic reports redact the source +configuration. Optional configuration encryption also covers the saved token. + +Each device requests temporary TURN credentials before opening a room when no +valid credentials are cached. Cloudflare credentials have a requested lifetime +of 24 hours and remain in memory only. Expiry is checked when LiveSync next +reconciles the room connection. If necessary, it replaces the room and obtains +new credentials. There is no periodic renewal: if a long-lived room cannot +reconnect after credentials expire, disconnect and open the connection again. + +Room replacement may interrupt replication. The next synchronisation keeps +received Metadata and Chunks, resumes from its saved checkpoint, and compares +revisions to fetch missing data. An unfinished network message can be sent +again. Automatic synchronisation follows the existing peer rules; after an +interrupted manual operation, use **Replicate now** again. ## Connection compatibility profiles `P2P Configuration` includes a separate `Connection compatibility` section. Its defaults preserve the existing transport behaviour: - **P2P message size** defaults to **Standard**. **Reduced**, **Conservative**, and **Maximum compatibility** progressively limit outgoing P2P messages when a network path appears to drop larger WebRTC messages. This is not a Vault Chunk size or an IP MTU. Smaller values add framing and processing overhead. -- **Connection path** defaults to **Automatic**, which lets WebRTC select a viable direct or TURN-relayed path. **TURN relay only** forces the encrypted connection through TURN and is available only when the profile contains at least one valid `turn:` or `turns:` URL. +- **Connection path** defaults to **Automatic**, which lets WebRTC select a viable direct or TURN-relayed path. **TURN relay only** forces the encrypted connection through TURN and is available when the profile contains a valid manual TURN URL or a configured TURN credential source. The sending device controls its outgoing message size. Select the same conservative preset on every device which may send across the constrained path. Existing devices do not receive the choice retrospectively merely because another device changed it. diff --git a/docs/settings.md b/docs/settings.md index feddbb73..96fab9f8 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -485,6 +485,26 @@ Setting key: P2P_AutoBroadcast When enabled, this device notifies connected peers after a local change. The notification contains no Vault data. A receiving peer fetches the change only when it follows this device. +#### TURN configuration + +Setting key: P2P_iceServerSource + +Select **Manual** for the existing TURN server fields, or **Cloudflare** for a +TURN Key ID and TURN Key API Token. The API token is persisted with the profile +and included in encrypted Setup URI sharing. Issued temporary credentials are +kept in memory only. Reports redact the source configuration. See +[TURN credentials](p2p.md#turn-credentials) for sharing, expiry, and reconnect +behaviour. + +#### TURN Key ID and TURN Key API Token + +Setting keys: P2P_iceServerSource.configuration.turnKeyId, +P2P_iceServerSource.configuration.apiToken + +These fields appear when **Cloudflare** is selected. Enter the TURN key's ID and +its dedicated API token. The token field is masked. No account ID, custom +endpoint, or renewal interval is required. + #### TURN Server URLs (comma-separated) Setting key: P2P_turnServers @@ -515,7 +535,7 @@ The sender controls the size of its outgoing messages. Select the same conservat Setting key: P2P_connectionPath -**Automatic** lets WebRTC select a viable direct or TURN-relayed path and is the default. **TURN relay only** forces `iceTransportPolicy: 'relay'` and is available only when the profile contains at least one valid `turn:` or `turns:` URL. Removing the last valid TURN URL while relay-only mode is selected restores **Automatic** and displays a Notice. +**Automatic** lets WebRTC select a viable direct or TURN-relayed path and is the default. **TURN relay only** forces `iceTransportPolicy: 'relay'` and is available when the profile contains a valid manual TURN URL or a configured TURN credential source. Removing the manual TURN configuration while relay-only mode is selected restores **Automatic** and displays a Notice. A selected credential source which cannot supply credentials prevents the connection from opening; it does not change the connection path. This choice belongs to the P2P profile and is retained in P2P connection strings and encrypted Setup URIs. Separate profiles may use the same Group ID and credentials with different compatibility choices; only the selected P2P profile is active. diff --git a/eslint.community.config.mjs b/eslint.community.config.mjs index 4944a5f8..059b82f7 100644 --- a/eslint.community.config.mjs +++ b/eslint.community.config.mjs @@ -63,6 +63,13 @@ export default defineConfig( "@typescript-eslint/no-unnecessary-type-assertion": "warn", }, }, + { + files: ["src/integrations/**/*.ts"], + rules: { + // External-service integrations also run in Node and do not own window UI. + "obsidianmd/no-global-this": "off", + }, + }, { files: ["src/apps/**/*.{ts,js,mjs}"], rules: { diff --git a/eslint.config.mjs b/eslint.config.mjs index b436c9e3..84cbbfbd 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -99,6 +99,13 @@ export default defineConfig([ ...ImportAliasRules("."), }, }, + { + files: ["src/integrations/**/*.ts"], + rules: { + // External-service integrations also run in Node and do not own window UI. + "obsidianmd/no-global-this": "off", + }, + }, { files: ["src/apps/**/*.ts"], rules: { diff --git a/package-lock.json b/package-lock.json index 6e70af04..6261b61d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.24", + "@vrtmrz/livesync-commonlib": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.3.tgz", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4567,9 +4567,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.24", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.24.tgz", - "integrity": "sha512-gOXKo3ptEUYDkjLd5PGq2vAhOSvxOqW6xE7YWo9Y8ienglfYBz8R3eZ0I/JruvwZltH2B7Bmi41pHMjmRJQe3Q==", + "version": "0.1.25-dev.turn-credentials.3", + "resolved": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.3.tgz", + "integrity": "sha512-hngE1zlNocD8IeMgRvssX4WN7SpINOyWOp9Mzs0PGM10yzVvYtR5WcVdxWtEIbaA71DLk9SPmgmQkgu+o7Oy8g==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index abacfc2b..706922eb 100644 --- a/package.json +++ b/package.json @@ -181,7 +181,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.24", + "@vrtmrz/livesync-commonlib": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.3.tgz", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/src/apps/browser/BrowserP2PTransportSettings.svelte b/src/apps/browser/BrowserP2PTransportSettings.svelte index eed3bc52..2d13a081 100644 --- a/src/apps/browser/BrowserP2PTransportSettings.svelte +++ b/src/apps/browser/BrowserP2PTransportSettings.svelte @@ -1,105 +1,61 @@
Optional TURN server settings -

- Configure TURN only when a direct peer-to-peer connection cannot be established. -

- - - +

Configure TURN only when a direct peer-to-peer connection cannot be established.

+
- -
@@ -107,27 +63,7 @@
diff --git a/src/apps/cli/commands/runCommand.ts b/src/apps/cli/commands/runCommand.ts index 8f27b870..8e3bf50c 100644 --- a/src/apps/cli/commands/runCommand.ts +++ b/src/apps/cli/commands/runCommand.ts @@ -1,5 +1,5 @@ import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/compat/API/processSetting"; -import { configURIBase } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const"; +import { configURIBase, configURIBaseV2 } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const"; import { DEFAULT_SETTINGS, type FilePathWithPrefix, @@ -298,7 +298,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext throw new Error("setup requires one argument: "); } const setupURI = options.commandArgs[0].trim(); - if (!setupURI.startsWith(configURIBase)) { + if (!setupURI.startsWith(configURIBase) && !setupURI.startsWith(configURIBaseV2)) { throw new Error(`setup URI must start with ${configURIBase}`); } const passphrase = await standardIo.prompt("Enter setup URI passphrase: "); diff --git a/src/apps/cli/commands/runCommand.unit.spec.ts b/src/apps/cli/commands/runCommand.unit.spec.ts index 651e47d1..f34e7cfd 100644 --- a/src/apps/cli/commands/runCommand.unit.spec.ts +++ b/src/apps/cli/commands/runCommand.unit.spec.ts @@ -1,7 +1,7 @@ import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node"; import * as processSetting from "@vrtmrz/livesync-commonlib/compat/API/processSetting"; import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; -import { configURIBase } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const"; +import { configURIBase, configURIBaseV2 } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const"; import { DEFAULT_SETTINGS, REMOTE_COUCHDB, @@ -419,6 +419,31 @@ describe("runCommand abnormal cases", () => { expect(appliedSettings.useIndexedDBAdapter).toBe(false); }); + it("setup imports managed TURN through the versioned encrypted URI", async () => { + const core = createCoreMock(); + const source = { + version: 1, + id: "cloudflare", + configuration: { turnKeyId: "turn-key", apiToken: "private-token" }, + }; + const passphrase = "setup-passphrase"; + const setupURI = await processSetting.encodeSettingsToSetupURI( + { + ...DEFAULT_SETTINGS, + P2P_iceServerSource: source, + }, + passphrase + ); + expect(setupURI.startsWith(configURIBaseV2)).toBe(true); + expect(setupURI).not.toContain("private-token"); + core.services.context.standardIo.prompt.mockResolvedValue(passphrase); + await runCommand(makeOptions("setup", [setupURI]), { ...context, core }); + expect(core.services.setting.applyExternalSettings).toHaveBeenCalledWith( + expect.objectContaining({ P2P_iceServerSource: source }), + true + ); + }); + it("setup rejects encoded URI when passphrase is wrong", async () => { const core = createCoreMock(); const setupURI = await createSetupURI("correct-passphrase"); diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 1b2bff53..7f17e0a9 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -1,3 +1,4 @@ +import { useIceServerSources } from "@/serviceFeatures/useIceServerSources"; import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub"; import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage"; import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore"; @@ -524,7 +525,9 @@ export async function main( useOfflineScanner(core); } // Register P2P replicator feature. - p2pReplicator = useP2PReplicatorFeature(core); + p2pReplicator = useP2PReplicatorFeature(core, undefined, undefined, { + iceServerSources: useIceServerSources(core.services.API.webCompatFetch.bind(core.services.API)), + }); // Add target filter to prevent internal files are handled core.services.vault.isTargetFile.addHandler(async (target) => { const targetPath = stripAllPrefixes(getPathFromUXFileInfo(target)); diff --git a/src/apps/webapp/WebAppRuntime.ts b/src/apps/webapp/WebAppRuntime.ts index 63940081..9fa841cb 100644 --- a/src/apps/webapp/WebAppRuntime.ts +++ b/src/apps/webapp/WebAppRuntime.ts @@ -1,3 +1,4 @@ +import { useIceServerSources } from "@/serviceFeatures/useIceServerSources"; /** Browser runtime for Self-hosted LiveSync over the File System Access API. */ import { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; @@ -217,7 +218,9 @@ export class WebAppRuntime { useRedFlagFeatures(core); useCheckRemoteSize(core); useRemoteConfiguration(core); - this.p2p = useP2PReplicatorFeature(core); + this.p2p = useP2PReplicatorFeature(core, undefined, undefined, { + iceServerSources: useIceServerSources(core.services.API.webCompatFetch.bind(core.services.API)), + }); this.paneHost = { services: core.services, p2p: this.p2p, diff --git a/src/apps/webpeer/src/WebPeerRuntime.ts b/src/apps/webpeer/src/WebPeerRuntime.ts index c6e7d433..da0d87e8 100644 --- a/src/apps/webpeer/src/WebPeerRuntime.ts +++ b/src/apps/webpeer/src/WebPeerRuntime.ts @@ -1,3 +1,4 @@ +import { useIceServerSources } from "@/serviceFeatures/useIceServerSources"; import { type P2PSyncSetting, SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { EVENT_LAYOUT_READY } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; @@ -70,9 +71,8 @@ export class WebPeerRuntime { isScheduled: () => this.restartScheduled, }, }); - this.p2p = useP2PReplicatorFeature({ - services: this.services, - serviceModules: {}, + this.p2p = useP2PReplicatorFeature({ services: this.services, serviceModules: {} }, undefined, undefined, { + iceServerSources: useIceServerSources(this.services.API.webCompatFetch.bind(this.services.API)), }); this.p2pLogCollector = new P2PLogCollector(this.events); this.paneHost = { diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 37f0a7b3..5acfcb64 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,33 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "Configure TURN when a direct connection cannot be established or when you select TURN relay only.": + "Configure TURN when a direct connection cannot be established or when you select TURN relay only.", + "TURN configuration could not be decrypted.": "TURN configuration could not be decrypted.", + "TURN configuration": "TURN configuration", + Manual: "Manual", + Cloudflare: "Cloudflare", + "TURN Key ID": "TURN Key ID", + "TURN Key API Token": "TURN Key API Token", + "Unsupported TURN configuration": "Unsupported TURN configuration", + "The API token is saved with this profile and included in encrypted Setup URI sharing. Temporary TURN credentials are kept in memory only.": + "The API token is saved with this profile and included in encrypted Setup URI sharing. Temporary TURN credentials are kept in memory only.", + "TURN relay only requires a TURN server or a configured credential source under Advanced Settings.": + "TURN relay only requires a TURN server or a configured credential source under Advanced Settings.", + "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.": + "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.", + "Cloudflare TURN configuration is invalid.": "Cloudflare TURN configuration is invalid.", + "Cloudflare TURN configuration contains an unsupported field.": + "Cloudflare TURN configuration contains an unsupported field.", + "Enter a TURN Key ID.": "Enter a TURN Key ID.", + "TURN Key ID contains unsupported characters.": "TURN Key ID contains unsupported characters.", + "Enter a TURN Key API Token.": "Enter a TURN Key API Token.", + "TURN Key API Token must use Bearer token syntax.": "TURN Key API Token must use Bearer token syntax.", + "TURN configuration source version is not supported.": "TURN configuration source version is not supported.", + "TURN configuration source is invalid.": "TURN configuration source is invalid.", + "The selected TURN configuration source is not supported.": + "The selected TURN configuration source is not supported.", + "Setup Complete: Preparing to Fetch from Another Device": "Setup Complete: Preparing to Fetch from Another Device", "The P2P connection has been configured successfully. The initial synchronisation data must now be fetched from an online source device.": "The P2P connection has been configured successfully. The initial synchronisation data must now be fetched from an online source device.", @@ -28,8 +55,8 @@ export const liveSyncProvisionalEnglishMessages = { "The project's public signalling relay is a best-effort convenience operated by the project author. It does not store Vault contents, but signalling metadata may be visible to the relay. Availability and log retention are not guaranteed. You can replace it with your own Nostr-compatible relay.", "Learn more about P2P connections": "Learn more about P2P connections", "Learn more about signalling and TURN": "Learn more about signalling and TURN", - "TURN relays the encrypted WebRTC connection only when a direct path cannot be established. A TURN provider cannot read encrypted Vault contents, but it can observe connection metadata and traffic volume. Use a provider you trust.": - "TURN relays the encrypted WebRTC connection only when a direct path cannot be established. A TURN provider cannot read encrypted Vault contents, but it can observe connection metadata and traffic volume. Use a provider you trust.", + "WebRTC encrypts data between your devices, including when it passes through TURN. The TURN provider cannot read the transferred data. It can see network addresses and traffic volume.": + "WebRTC encrypts data between your devices, including when it passes through TURN. The TURN provider cannot read the transferred data. It can see network addresses and traffic volume.", "Connection compatibility": "Connection compatibility", "P2P message size": "P2P message size", Standard: "Standard", diff --git a/src/common/reportTool.ts b/src/common/reportTool.ts index 5942dfa8..e8bded42 100644 --- a/src/common/reportTool.ts +++ b/src/common/reportTool.ts @@ -1,3 +1,4 @@ +import { redactTurnSourceForReport } from "./turnSettingsPrivacy"; import { REMOTE_COUCHDB, REMOTE_MINIO } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.const"; import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/settings"; import { generateCredentialObject } from "@vrtmrz/livesync-commonlib/compat/replication/httplib"; @@ -67,6 +68,7 @@ export async function generateReport(settings: ObsidianLiveSyncSettings, core: L delete pluginConfig[key as keyof ObsidianLiveSyncSettings]; } + redactTurnSourceForReport(pluginConfig); pluginConfig.couchDB_DBNAME = REDACTED; pluginConfig.couchDB_PASSWORD = REDACTED; const scheme = pluginConfig.couchDB_URI.startsWith("http:") diff --git a/src/common/reportTool.unit.spec.ts b/src/common/reportTool.unit.spec.ts new file mode 100644 index 00000000..18afd7fe --- /dev/null +++ b/src/common/reportTool.unit.spec.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/settings"; +import { REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; +import { generateReport } from "./reportTool"; + +vi.mock("./utils", () => ({ requestToCouchDBWithCredentials: vi.fn() })); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions", () => ({ + compatGlobal: { origin: "test", navigator: { userAgent: "test" } }, +})); + +describe("TURN credentials in diagnostic reports", () => { + it("redacts top-level, encrypted, and inactive encoded source copies", async () => { + const token = "private+token/with=symbols"; + const source = { version: 1, id: "cloudflare", configuration: { turnKeyId: "private-key", apiToken: token } }; + const settings = { + ...DEFAULT_SETTINGS, + remoteType: REMOTE_P2P, + P2P_iceServerSource: source, + encryptedP2PIceServerSource: "encrypted-private-copy", + remoteConfigurations: { + inactive: { + id: "inactive", + name: "Inactive TURN", + isEncrypted: false, + uri: `sls+p2p-v2://room?source=${encodeURIComponent(JSON.stringify(source))}`, + }, + }, + }; + const core = { services: { vault: { isStorageInsensitive: () => false } } } as unknown as LiveSyncBaseCore; + const report = await generateReport(settings, core); + const text = JSON.stringify(report); + expect(text).not.toContain(token); + expect(text).not.toContain(encodeURIComponent(token)); + expect(text).not.toContain("private-key"); + expect(text).not.toContain("encrypted-private-copy"); + expect(report.pluginConfig.remoteConfigurations.inactive.uri).toBe("sls+p2p-v2://"); + expect(settings.P2P_iceServerSource).toEqual(source); + expect(settings.encryptedP2PIceServerSource).toBe("encrypted-private-copy"); + }); +}); diff --git a/src/common/turnSettingsPrivacy.ts b/src/common/turnSettingsPrivacy.ts new file mode 100644 index 00000000..adc3ae32 --- /dev/null +++ b/src/common/turnSettingsPrivacy.ts @@ -0,0 +1,52 @@ +import { + hasManagedP2PIceServerSource as hasManagedTurnSettings, + type ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; + +import { iceServerSourceDefinitions } from "@/integrations/iceServerSources"; + +export { hasManagedTurnSettings }; + +/** Reports retain the selected source label, but no opaque source configuration. */ +export function redactTurnSourceForReport(settings: Partial): void { + if (settings.encryptedP2PIceServerSource) settings.encryptedP2PIceServerSource = "REDACTED"; + if (settings.P2P_iceServerSource !== undefined) { + settings.P2P_iceServerSource = { + version: 1, + id: + iceServerSourceDefinitions.find((source) => source.id === settings.P2P_iceServerSource?.id)?.id ?? + "redacted", + configuration: { redacted: true }, + }; + } +} + +/** Managed connection profiles are shared through encrypted Setup URIs. */ +export function omitManagedTurnProfilesFromMarkdown(settings: Partial): void { + if (!hasManagedTurnSettings(settings)) return; + delete settings.P2P_iceServerSource; + delete settings.encryptedP2PIceServerSource; + delete settings.remoteConfigurations; + delete settings.activeConfigurationId; + delete settings.P2P_ActiveRemoteConfigurationId; +} + +/** An omitted profile group leaves this device's existing connection selection intact. */ +export function preserveManagedTurnProfilesOnMarkdownImport( + incoming: Partial, + current: ObsidianLiveSyncSettings, + merged: ObsidianLiveSyncSettings +): void { + if ( + !hasManagedTurnSettings(current) || + incoming.remoteConfigurations !== undefined || + incoming.P2P_iceServerSource !== undefined + ) { + return; + } + merged.remoteConfigurations = structuredClone(current.remoteConfigurations); + merged.activeConfigurationId = current.activeConfigurationId; + merged.P2P_ActiveRemoteConfigurationId = current.P2P_ActiveRemoteConfigurationId; + merged.P2P_iceServerSource = structuredClone(current.P2P_iceServerSource); + merged.encryptedP2PIceServerSource = current.encryptedP2PIceServerSource; +} diff --git a/src/common/turnSettingsPrivacy.unit.spec.ts b/src/common/turnSettingsPrivacy.unit.spec.ts new file mode 100644 index 00000000..5a56b967 --- /dev/null +++ b/src/common/turnSettingsPrivacy.unit.spec.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + hasManagedTurnSettings, + omitManagedTurnProfilesFromMarkdown, + preserveManagedTurnProfilesOnMarkdownImport, + redactTurnSourceForReport, +} from "./turnSettingsPrivacy"; + +function configuredSettings() { + return { + ...DEFAULT_SETTINGS, + P2P_iceServerSource: { + version: 1, + id: "cloudflare", + configuration: { turnKeyId: "private-key-id", apiToken: "private-token" }, + }, + remoteConfigurations: { + managed: { + id: "managed", + name: "Managed TURN", + isEncrypted: false, + uri: "sls+p2p-v2://room?source=private-token", + }, + }, + activeConfigurationId: "central", + P2P_ActiveRemoteConfigurationId: "managed", + }; +} + +describe("managed TURN settings privacy", () => { + it("redacts all opaque source fields, including unknown integrations", () => { + const settings = configuredSettings(); + settings.P2P_iceServerSource.id = "private-token"; + redactTurnSourceForReport(settings); + expect(JSON.stringify(settings.P2P_iceServerSource)).not.toMatch(/private-token|private-key-id/); + expect(settings.P2P_iceServerSource.configuration).toEqual({ redacted: true }); + }); + + it("omits the whole managed profile group from Markdown, including inactive sources", () => { + const settings = configuredSettings(); + settings.P2P_iceServerSource.id = "manual"; + expect(hasManagedTurnSettings(settings)).toBe(true); + omitManagedTurnProfilesFromMarkdown(settings); + expect(JSON.stringify(settings)).not.toMatch(/private-token|private-key-id|sls\+p2p-v2/); + expect(settings).not.toHaveProperty("remoteConfigurations"); + expect(settings).not.toHaveProperty("activeConfigurationId"); + expect(settings).not.toHaveProperty("P2P_ActiveRemoteConfigurationId"); + }); + + it("preserves existing profiles and both selections when Markdown omits the group", () => { + const current = configuredSettings(); + const incoming = { ...DEFAULT_SETTINGS }; + delete (incoming as Partial).remoteConfigurations; + delete (incoming as Partial).P2P_iceServerSource; + const merged = { ...DEFAULT_SETTINGS, ...incoming }; + preserveManagedTurnProfilesOnMarkdownImport(incoming, current, merged); + expect(merged.remoteConfigurations).toEqual(current.remoteConfigurations); + expect(merged.remoteConfigurations).not.toBe(current.remoteConfigurations); + expect(merged.P2P_iceServerSource).toEqual(current.P2P_iceServerSource); + expect(merged.activeConfigurationId).toBe("central"); + expect(merged.P2P_ActiveRemoteConfigurationId).toBe("managed"); + }); + + it("retains the manual-only Markdown contract", () => { + const settings = { ...DEFAULT_SETTINGS }; + const before = structuredClone(settings); + omitManagedTurnProfilesFromMarkdown(settings); + expect(settings).toEqual(before); + }); +}); diff --git a/src/common/types.ts b/src/common/types.ts index 2be50128..374e73d6 100644 --- a/src/common/types.ts +++ b/src/common/types.ts @@ -51,7 +51,7 @@ export type queueItem = { export const FileWatchEventQueueMax = 10; -export { configURIBase, configURIBaseQR } from "@vrtmrz/livesync-commonlib/compat/common/types"; +export { configURIBase, configURIBaseV2, configURIBaseQR } from "@vrtmrz/livesync-commonlib/compat/common/types"; export { CHeader, diff --git a/src/features/P2PSync/TurnConfiguration.svelte b/src/features/P2PSync/TurnConfiguration.svelte new file mode 100644 index 00000000..780b4d68 --- /dev/null +++ b/src/features/P2PSync/TurnConfiguration.svelte @@ -0,0 +1,83 @@ + + +
+ + {#if sourceId === "manual"} + + + + {:else if definition} + {#each definition.fields as field (field.key)} + + {/each} +

{translate("The API token is saved with this profile and included in encrypted Setup URI sharing. Temporary TURN credentials are kept in memory only.")}

+ {/if} + {#if error} +

{translateIfAvailable(error)}

+ {/if} +
+ + diff --git a/src/integrations/cloudflare/iceServerSource.ts b/src/integrations/cloudflare/iceServerSource.ts new file mode 100644 index 00000000..07e81150 --- /dev/null +++ b/src/integrations/cloudflare/iceServerSource.ts @@ -0,0 +1,384 @@ +import { IceServerSourceError } from "@vrtmrz/livesync-commonlib/p2p"; +import type { IceServerConfiguration, IceServerSource } from "@vrtmrz/livesync-commonlib/p2p"; +import { + CLOUDFLARE_TURN_CREDENTIAL_ENDPOINT, + CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS, + parseCloudflareIceServerSourceConfiguration, + type CloudflareIceServerSourceConfiguration, + validateCloudflareIceServerSourceConfiguration, +} from "./settings"; + +/** Fetch-compatible function supplied by the host composition. */ +export type CloudflareIceServerSourceFetch = (input: string | Request, init?: RequestInit) => Promise; + +export interface CloudflareIceServerSourceDependencies { + readonly fetch: CloudflareIceServerSourceFetch; + readonly now?: () => number; + readonly requestDeadlineMs?: number; +} + +export const CLOUDFLARE_TURN_REQUEST_DEADLINE_MS = 15_000 as const; +export const CLOUDFLARE_TURN_MAX_RESPONSE_BYTES = 32 * 1024; +export const CLOUDFLARE_TURN_MAX_ICE_SERVER_ENTRIES = 16 as const; +export const CLOUDFLARE_TURN_MAX_ICE_SERVER_URLS = 32 as const; +export const CLOUDFLARE_TURN_MIN_REMAINING_LIFETIME_MS = 1_000 as const; + +type IceServerSourceFailureCode = "configuration" | "authentication" | "unavailable" | "invalid-response"; + +const SOURCE_FAILURE_MESSAGES: Record = { + configuration: "The Cloudflare TURN source configuration is invalid.", + authentication: "The Cloudflare TURN credential request was not authorised.", + unavailable: "The Cloudflare TURN service is unavailable.", + "invalid-response": "The Cloudflare TURN service returned an invalid response.", +}; + +function sourceFailure(code: IceServerSourceFailureCode, retryable: boolean): IceServerSourceError { + return new IceServerSourceError(code, SOURCE_FAILURE_MESSAGES[code], retryable); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function abortError(): Error { + try { + return new DOMException("The operation was aborted.", "AbortError"); + } catch { + const error = new Error("The operation was aborted."); + error.name = "AbortError"; + return error; + } +} + +function throwIfAborted(signal: AbortSignal): void { + if (signal.aborted) { + throw abortError(); + } +} + +function isControlCharacter(value: string): boolean { + return Array.from(value).some((character) => { + const code = character.charCodeAt(0); + return code <= 0x1f || code === 0x7f; + }); +} + +function isPort(value: string): boolean { + if (!/^\d{1,5}$/.test(value)) return false; + const port = Number(value); + return port >= 1 && port <= 65_535; +} + +function isHost(value: string): boolean { + return value.length > 0 && /^[A-Za-z0-9._-]+$/.test(value); +} + +/** + * Validates the URL forms accepted by WebRTC's ICE server configuration. + * TURN URLs may carry only the standard transport query parameter; userinfo, + * paths, fragments, and arbitrary query values are not accepted. + */ +export function isSupportedIceServerUrl(value: string): boolean { + if (value.length === 0 || value.length > 2_048 || isControlCharacter(value)) return false; + const schemeMatch = /^(stun|stuns|turn|turns):(.+)$/i.exec(value); + if (!schemeMatch) return false; + + const remainder = schemeMatch[2]; + const queryIndex = remainder.indexOf("?"); + const authority = queryIndex >= 0 ? remainder.slice(0, queryIndex) : remainder; + const query = queryIndex >= 0 ? remainder.slice(queryIndex + 1) : ""; + if (authority.length === 0 || authority.includes("/") || authority.includes("#") || authority.includes("@")) { + return false; + } + if (authority.includes("%")) return false; + + if (authority.startsWith("[")) { + const closingBracket = authority.indexOf("]"); + if (closingBracket < 0) return false; + const host = authority.slice(1, closingBracket); + if (!/^[0-9A-Fa-f:.]+$/.test(host) || !host.includes(":")) return false; + const suffix = authority.slice(closingBracket + 1); + if (suffix !== "" && (!suffix.startsWith(":") || !isPort(suffix.slice(1)))) return false; + } else { + const colonIndex = authority.lastIndexOf(":"); + const host = colonIndex >= 0 ? authority.slice(0, colonIndex) : authority; + if (!isHost(host) || (colonIndex >= 0 && !isPort(authority.slice(colonIndex + 1)))) return false; + // IPv6 literals must use brackets so a colon cannot be interpreted as + // an ambiguous port separator. + if (colonIndex >= 0 && host.includes(":")) return false; + } + + if (query.length === 0) return true; + const queryParts = query.split("&"); + return queryParts.length === 1 && /^transport=(udp|tcp)$/i.test(queryParts[0]); +} + +function isTurnUrl(value: string): boolean { + return /^(turn|turns):/i.test(value); +} + +function isCredential(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 4_096 && !isControlCharacter(value); +} + +function normaliseIceServers(value: unknown): readonly RTCIceServer[] { + if (!isRecord(value) || !Array.isArray(value.iceServers)) { + throw sourceFailure("invalid-response", false); + } + if (value.iceServers.length === 0 || value.iceServers.length > CLOUDFLARE_TURN_MAX_ICE_SERVER_ENTRIES) { + throw sourceFailure("invalid-response", false); + } + + const servers: RTCIceServer[] = []; + let urlCount = 0; + let hasTurnServer = false; + + for (const candidate of value.iceServers) { + if (!isRecord(candidate)) throw sourceFailure("invalid-response", false); + const rawUrls = candidate.urls; + const urls = + typeof rawUrls === "string" + ? [rawUrls] + : Array.isArray(rawUrls) && rawUrls.every((url): url is string => typeof url === "string") + ? [...rawUrls] + : undefined; + if (!urls || urls.length === 0) throw sourceFailure("invalid-response", false); + + urlCount += urls.length; + if (urlCount > CLOUDFLARE_TURN_MAX_ICE_SERVER_URLS || urls.some((url) => !isSupportedIceServerUrl(url))) { + throw sourceFailure("invalid-response", false); + } + + const turnEntry = urls.some(isTurnUrl); + hasTurnServer ||= turnEntry; + const normalised: RTCIceServer = { urls }; + if (turnEntry) { + if (!isCredential(candidate.username) || !isCredential(candidate.credential)) { + throw sourceFailure("invalid-response", false); + } + normalised.username = candidate.username; + normalised.credential = candidate.credential; + } + servers.push(normalised); + } + + if (!hasTurnServer) throw sourceFailure("invalid-response", false); + return Object.freeze(servers); +} + +class BoundedResponseError extends Error { + constructor(readonly kind: "too-large" | "invalid-length" | "read-failed") { + super(kind); + } +} + +async function readResponseBody(response: Response): Promise { + const contentLength = response.headers.get("content-length"); + if (contentLength !== null) { + const declaredLength = Number(contentLength); + if (!Number.isFinite(declaredLength) || declaredLength < 0) { + throw new BoundedResponseError("invalid-length"); + } + if (declaredLength > CLOUDFLARE_TURN_MAX_RESPONSE_BYTES) { + throw new BoundedResponseError("too-large"); + } + } + + if (!response.body) { + try { + const text = await response.text(); + if (new TextEncoder().encode(text).byteLength > CLOUDFLARE_TURN_MAX_RESPONSE_BYTES) { + throw new BoundedResponseError("too-large"); + } + return text; + } catch (error) { + if (error instanceof BoundedResponseError) throw error; + throw new BoundedResponseError("read-failed"); + } + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + try { + while (true) { + const result = await reader.read(); + if (result.done) break; + totalBytes += result.value.byteLength; + if (totalBytes > CLOUDFLARE_TURN_MAX_RESPONSE_BYTES) { + try { + await reader.cancel(); + } catch { + // The response is already invalid because it exceeded the + // bound; cancellation failure must not change the safe + // classification or expose a host-specific error. + } + throw new BoundedResponseError("too-large"); + } + chunks.push(result.value); + } + } catch (error) { + if (error instanceof BoundedResponseError) throw error; + throw new BoundedResponseError("read-failed"); + } finally { + reader.releaseLock(); + } + + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +} + +function classifyHttpFailure(status: number): IceServerSourceError { + if (status === 401 || status === 403) { + return sourceFailure("authentication", false); + } + if (status === 408 || status === 429 || status >= 500) { + return sourceFailure("unavailable", true); + } + return sourceFailure("unavailable", false); +} + +function parseResponseBody(body: string): readonly RTCIceServer[] { + let value: unknown; + try { + value = JSON.parse(body) as unknown; + } catch { + throw sourceFailure("invalid-response", false); + } + return normaliseIceServers(value); +} + +function createSource( + configuration: CloudflareIceServerSourceConfiguration, + dependencies: CloudflareIceServerSourceDependencies +): IceServerSource { + const now = dependencies.now ?? Date.now; + const requestDeadlineMs = dependencies.requestDeadlineMs ?? CLOUDFLARE_TURN_REQUEST_DEADLINE_MS; + + return { + async acquire(signal: AbortSignal): Promise { + throwIfAborted(signal); + const requestStartedAt = now(); + if (!Number.isFinite(requestStartedAt)) { + throw sourceFailure("unavailable", true); + } + + const requestController = new AbortController(); + let cancelledByCaller = false; + let rejectCaller: ((reason?: unknown) => void) | undefined; + const callerAbort = new Promise((_resolve, reject) => { + rejectCaller = reject; + }); + let timedOut = false; + const onAbort = () => { + cancelledByCaller = true; + requestController.abort(); + rejectCaller?.(abortError()); + }; + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) { + signal.removeEventListener("abort", onAbort); + requestController.abort(); + throw abortError(); + } + let timeoutId: ReturnType | undefined; + const deadline = new Promise((_resolve, reject) => { + timeoutId = globalThis.setTimeout(() => { + timedOut = true; + requestController.abort(); + reject(sourceFailure("unavailable", true)); + }, requestDeadlineMs); + }); + + const cleanup = () => { + if (timeoutId !== undefined) globalThis.clearTimeout(timeoutId); + signal.removeEventListener("abort", onAbort); + }; + + const endpoint = `${CLOUDFLARE_TURN_CREDENTIAL_ENDPOINT}/${configuration.turnKeyId}/credentials/generate-ice-servers`; + let response: Response; + try { + response = await Promise.race([ + dependencies.fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${configuration.apiToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ ttl: CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS }), + signal: requestController.signal, + redirect: "error", + credentials: "omit", + cache: "no-store", + }), + callerAbort, + deadline, + ]); + } catch { + cleanup(); + if (cancelledByCaller || signal.aborted) throw abortError(); + if (timedOut) throw sourceFailure("unavailable", true); + throw sourceFailure("unavailable", true); + } + + if (cancelledByCaller || signal.aborted) { + cleanup(); + throw abortError(); + } + if (timedOut || requestController.signal.aborted) { + cleanup(); + throw sourceFailure("unavailable", true); + } + if (response.status !== 201) { + cleanup(); + throw classifyHttpFailure(response.status); + } + + let body: string; + try { + body = await Promise.race([readResponseBody(response), callerAbort, deadline]); + } catch (error) { + cleanup(); + if (cancelledByCaller || signal.aborted) throw abortError(); + if (timedOut) throw sourceFailure("unavailable", true); + if (error instanceof BoundedResponseError && error.kind === "read-failed") { + throw sourceFailure("unavailable", true); + } + throw sourceFailure("invalid-response", false); + } + + try { + throwIfAborted(signal); + const iceServers = parseResponseBody(body); + const expiresAt = requestStartedAt + CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS * 1_000; + if (!Number.isFinite(expiresAt) || expiresAt <= now() + CLOUDFLARE_TURN_MIN_REMAINING_LIFETIME_MS) { + throw sourceFailure("invalid-response", false); + } + return { iceServers, expiresAt }; + } finally { + cleanup(); + } + }, + }; +} + +/** + * Creates a Cloudflare source after validating its persisted configuration. + * Validation is synchronous and performs no network request. + */ +export function createCloudflareIceServerSource( + configuration: Readonly>, + dependencies: CloudflareIceServerSourceDependencies +): IceServerSource { + const parsed = parseCloudflareIceServerSourceConfiguration(configuration); + if (!parsed) throw sourceFailure("configuration", false); + return createSource(parsed, dependencies); +} + +/** Exposes the provider validation for the integration catalogue and UI. */ +export { validateCloudflareIceServerSourceConfiguration }; diff --git a/src/integrations/cloudflare/iceServerSource.unit.spec.ts b/src/integrations/cloudflare/iceServerSource.unit.spec.ts new file mode 100644 index 00000000..8b61d356 --- /dev/null +++ b/src/integrations/cloudflare/iceServerSource.unit.spec.ts @@ -0,0 +1,164 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CLOUDFLARE_TURN_MAX_RESPONSE_BYTES, + CLOUDFLARE_TURN_REQUEST_DEADLINE_MS, + createCloudflareIceServerSource, +} from "./iceServerSource"; +import { + CLOUDFLARE_TURN_CREDENTIAL_ENDPOINT, + CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS, + validateCloudflareIceServerSourceConfiguration, +} from "./settings"; + +const configuration = { + turnKeyId: "key-123", + apiToken: "token_abc-123", +} as const; + +function response(body: unknown, status = 201): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function validBody() { + return { + iceServers: [ + { + urls: ["turn:relay.example.test:3478?transport=udp", "turns:relay.example.test:5349"], + username: "turn-user", + credential: "turn-password", + }, + { urls: "stun:stun.example.test:3478" }, + ], + }; +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("Cloudflare ICE server source", () => { + it("requests the fixed endpoint with the bearer token and TTL", async () => { + const now = 1_000_000; + let requestUrl: string | Request | undefined; + let requestInit: RequestInit | undefined; + const fetch = vi.fn(async (input: string | Request, init?: RequestInit) => { + requestUrl = input; + requestInit = init; + return response(validBody()); + }); + const source = createCloudflareIceServerSource(configuration, { fetch, now: () => now }); + + const result = await source.acquire(new AbortController().signal); + + expect(requestUrl).toBe(`${CLOUDFLARE_TURN_CREDENTIAL_ENDPOINT}/key-123/credentials/generate-ice-servers`); + expect(requestInit).toMatchObject({ + method: "POST", + redirect: "error", + credentials: "omit", + cache: "no-store", + body: JSON.stringify({ ttl: CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS }), + }); + expect(new Headers(requestInit?.headers).get("authorization")).toBe("Bearer token_abc-123"); + expect(new Headers(requestInit?.headers).get("content-type")).toBe("application/json"); + expect(requestInit?.signal).toBeInstanceOf(AbortSignal); + expect(result.iceServers).toHaveLength(2); + expect(result.expiresAt).toBe(now + CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS * 1_000); + }); + + it("rejects malformed, oversized, and STUN-only responses without exposing secrets", async () => { + const cases: Array<{ body: unknown; expectedCode: string }> = [ + { body: { iceServers: [] }, expectedCode: "invalid-response" }, + { body: { iceServers: [{ urls: "turn:relay.example.test:3478" }] }, expectedCode: "invalid-response" }, + { body: { iceServers: [{ urls: "stun:stun.example.test:3478" }] }, expectedCode: "invalid-response" }, + ]; + for (const testCase of cases) { + const source = createCloudflareIceServerSource(configuration, { + fetch: vi.fn(async () => response(testCase.body)), + now: () => 1_000_000, + }); + const error = await source.acquire(new AbortController().signal).catch((reason: unknown) => reason); + expect(error).toMatchObject({ code: testCase.expectedCode }); + expect(String(error)).not.toContain(configuration.apiToken); + expect(String(error)).not.toContain(configuration.turnKeyId); + } + + const oversized = "x".repeat(CLOUDFLARE_TURN_MAX_RESPONSE_BYTES + 1); + const source = createCloudflareIceServerSource(configuration, { + fetch: vi.fn(async () => new Response(oversized, { status: 201 })), + now: () => 1_000_000, + }); + const error = await source.acquire(new AbortController().signal).catch((reason: unknown) => reason); + expect(error).toMatchObject({ code: "invalid-response" }); + }); + + it("classifies authentication and transient provider failures", async () => { + const authSource = createCloudflareIceServerSource(configuration, { + fetch: vi.fn(async () => response({}, 401)), + }); + await expect(authSource.acquire(new AbortController().signal)).rejects.toMatchObject({ + code: "authentication", + retryable: false, + }); + + const transientSource = createCloudflareIceServerSource(configuration, { + fetch: vi.fn(async () => response({}, 503)), + }); + await expect(transientSource.acquire(new AbortController().signal)).rejects.toMatchObject({ + code: "unavailable", + retryable: true, + }); + }); + + it("propagates caller cancellation and turns a deadline into an unavailable failure", async () => { + const controller = new AbortController(); + const fetch = vi.fn((_input: string | Request, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")), { + once: true, + }); + }); + }); + const source = createCloudflareIceServerSource(configuration, { fetch }); + const cancelled = source.acquire(controller.signal); + controller.abort(); + await expect(cancelled).rejects.toMatchObject({ name: "AbortError" }); + + vi.useFakeTimers(); + const timedSource = createCloudflareIceServerSource(configuration, { fetch }); + const timed = timedSource.acquire(new AbortController().signal); + const assertion = expect(timed).rejects.toMatchObject({ code: "unavailable", retryable: true }); + await vi.advanceTimersByTimeAsync(CLOUDFLARE_TURN_REQUEST_DEADLINE_MS); + await assertion; + }); + + it("rejects an issuance which has no usable remaining lifetime", async () => { + let now = 1_000_000; + const source = createCloudflareIceServerSource(configuration, { + fetch: vi.fn(async () => { + now += CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS * 1_000; + return response(validBody()); + }), + now: () => now, + }); + await expect(source.acquire(new AbortController().signal)).rejects.toMatchObject({ + code: "invalid-response", + }); + }); +}); + +describe("Cloudflare ICE source validation", () => { + it("rejects unknown fields and malformed bearer credentials", () => { + expect(validateCloudflareIceServerSourceConfiguration({ ...configuration, unexpected: "value" })).toContain( + "unsupported field" + ); + expect( + validateCloudflareIceServerSourceConfiguration({ turnKeyId: "key/id", apiToken: configuration.apiToken }) + ).toContain("unsupported characters"); + expect( + validateCloudflareIceServerSourceConfiguration({ ...configuration, apiToken: "token with spaces" }) + ).toContain("Bearer token syntax"); + }); +}); diff --git a/src/integrations/cloudflare/settings.ts b/src/integrations/cloudflare/settings.ts new file mode 100644 index 00000000..3cd88f96 --- /dev/null +++ b/src/integrations/cloudflare/settings.ts @@ -0,0 +1,87 @@ +/** The source identifier persisted in a P2P profile for Cloudflare TURN. */ +export const CLOUDFLARE_ICE_SERVER_SOURCE_ID = "cloudflare" as const; + +/** The lifetime requested from Cloudflare for each issued credential set. */ +export const CLOUDFLARE_TURN_CREDENTIAL_TTL_SECONDS = 86_400 as const; + +/** The Cloudflare TURN credential-generation endpoint. */ +export const CLOUDFLARE_TURN_CREDENTIAL_ENDPOINT = "https://rtc.live.cloudflare.com/v1/turn/keys" as const; + +/** A validated Cloudflare TURN source configuration. */ +export interface CloudflareIceServerSourceConfiguration { + readonly turnKeyId: string; + readonly apiToken: string; +} + +const CLOUDFLARE_CONFIGURATION_KEYS = ["turnKeyId", "apiToken"] as const; + +// TURN Key IDs are inserted into one fixed URL path. Keep the accepted set +// deliberately narrower than URI escaping so a configuration cannot alter +// the request path or add a query string. +const TURN_KEY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._~-]{0,255}$/; + +// RFC 6750's b64token grammar, including optional trailing padding. This +// also excludes whitespace and control characters from the Authorization +// header without exposing the token in a validation message. +const BEARER_TOKEN_PATTERN = /^[A-Za-z0-9._~+/-]+={0,2}$/; +const MAX_BEARER_TOKEN_LENGTH = 4_096; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasOnlyCloudflareConfigurationKeys(value: Record): boolean { + const keys = Object.keys(value); + return ( + keys.length === CLOUDFLARE_CONFIGURATION_KEYS.length && + CLOUDFLARE_CONFIGURATION_KEYS.every((key) => Object.prototype.hasOwnProperty.call(value, key)) + ); +} + +/** + * Returns a safe validation message for a Cloudflare source configuration. + * The result never includes the supplied Key ID or API token. + */ +export function validateCloudflareIceServerSourceConfiguration(value: unknown): string | undefined { + if (!isRecord(value)) { + return "Cloudflare TURN configuration is invalid."; + } + if (!hasOnlyCloudflareConfigurationKeys(value)) { + return "Cloudflare TURN configuration contains an unsupported field."; + } + + const turnKeyId = value.turnKeyId; + if (typeof turnKeyId !== "string" || turnKeyId.length === 0) { + return "Enter a TURN Key ID."; + } + if (!TURN_KEY_ID_PATTERN.test(turnKeyId)) { + return "TURN Key ID contains unsupported characters."; + } + + const apiToken = value.apiToken; + if (typeof apiToken !== "string" || apiToken.length === 0) { + return "Enter a TURN Key API Token."; + } + if (apiToken.length > MAX_BEARER_TOKEN_LENGTH || !BEARER_TOKEN_PATTERN.test(apiToken)) { + return "TURN Key API Token must use Bearer token syntax."; + } + + return undefined; +} + +/** + * Converts an untrusted profile value into a validated source configuration. + * The returned object is a fresh copy so later settings mutations cannot + * change a source which is already being used by the P2P owner. + */ +export function parseCloudflareIceServerSourceConfiguration( + value: unknown +): CloudflareIceServerSourceConfiguration | undefined { + if (validateCloudflareIceServerSourceConfiguration(value) !== undefined || !isRecord(value)) { + return undefined; + } + return { + turnKeyId: value.turnKeyId as string, + apiToken: value.apiToken as string, + }; +} diff --git a/src/integrations/iceServerSources.ts b/src/integrations/iceServerSources.ts new file mode 100644 index 00000000..aaba78f6 --- /dev/null +++ b/src/integrations/iceServerSources.ts @@ -0,0 +1,85 @@ +import { CLOUDFLARE_ICE_SERVER_SOURCE_ID, validateCloudflareIceServerSourceConfiguration } from "./cloudflare/settings"; + +export const MANUAL_ICE_SERVER_SOURCE_ID = "manual" as const; + +export type IceServerSourceSelectionId = typeof MANUAL_ICE_SERVER_SOURCE_ID | typeof CLOUDFLARE_ICE_SERVER_SOURCE_ID; + +export interface IceServerSourceFieldDefinition { + readonly key: string; + readonly label: string; + readonly secret: boolean; +} + +export interface IceServerSourceDefinition { + readonly id: string; + readonly label: string; + readonly fields: readonly IceServerSourceFieldDefinition[]; +} + +export interface IceServerSourceDescriptorLike { + readonly version?: unknown; + readonly id?: unknown; + readonly configuration?: unknown; +} + +/** + * The service-owned field metadata used by the P2P settings dialogue. Manual + * TURN values remain the existing settings fields and therefore do not occur + * in this provider catalogue. + */ +export const iceServerSourceDefinitions = [ + { + id: CLOUDFLARE_ICE_SERVER_SOURCE_ID, + label: "Cloudflare", + fields: [ + { key: "turnKeyId", label: "TURN Key ID", secret: false }, + { key: "apiToken", label: "TURN Key API Token", secret: true }, + ], + }, +] as const satisfies readonly IceServerSourceDefinition[]; + +/** The user-facing source choice, including the existing manual mode. */ +export const turnConfigurationChoices = [ + { id: MANUAL_ICE_SERVER_SOURCE_ID, label: "Manual" }, + { id: CLOUDFLARE_ICE_SERVER_SOURCE_ID, label: "Cloudflare" }, +] as const; + +export const iceServerSourceChoices = turnConfigurationChoices; + +function isRecord(value: unknown): value is IceServerSourceDescriptorLike { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Validates a selected source descriptor without performing network access. + * An absent descriptor represents the existing manual TURN configuration. + */ +export function validateIceServerSourceConfiguration( + descriptor: IceServerSourceDescriptorLike | null | undefined +): string | undefined { + if (descriptor === undefined || descriptor === null) return undefined; + if (!isRecord(descriptor)) return "TURN configuration source is invalid."; + if (descriptor.version !== 1) return "TURN configuration source version is not supported."; + if (descriptor.id === MANUAL_ICE_SERVER_SOURCE_ID) { + return undefined; + } + if (descriptor.id !== CLOUDFLARE_ICE_SERVER_SOURCE_ID) { + return "The selected TURN configuration source is not supported."; + } + return validateCloudflareIceServerSourceConfiguration(descriptor.configuration); +} + +export function getIceServerSourceDefinition(id: string): IceServerSourceDefinition | undefined { + return iceServerSourceDefinitions.find((definition) => definition.id === id); +} + +/** Validate the selected settings projection, including an unavailable encrypted source. */ +export function validateTurnSettings(settings: { + readonly P2P_iceServerSource?: IceServerSourceDescriptorLike | null; + readonly encryptedP2PIceServerSource?: string; +}): string | undefined { + if (!settings.P2P_iceServerSource && settings.encryptedP2PIceServerSource) { + return "TURN configuration could not be decrypted."; + } + return validateIceServerSourceConfiguration(settings.P2P_iceServerSource); +} diff --git a/src/integrations/iceServerSources.unit.spec.ts b/src/integrations/iceServerSources.unit.spec.ts new file mode 100644 index 00000000..b3790e91 --- /dev/null +++ b/src/integrations/iceServerSources.unit.spec.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { + iceServerSourceDefinitions, + validateIceServerSourceConfiguration, + validateTurnSettings, +} from "./iceServerSources"; + +describe("ICE server source catalogue", () => { + it("blocks an unavailable encrypted source instead of presenting manual settings as valid", () => { + expect(validateTurnSettings({ encryptedP2PIceServerSource: "private-ciphertext" })).toBe( + "TURN configuration could not be decrypted." + ); + expect(validateTurnSettings({})).toBeUndefined(); + }); + + it("describes the Cloudflare fields without owning manual TURN fields", () => { + expect(iceServerSourceDefinitions).toEqual([ + { + id: "cloudflare", + label: "Cloudflare", + fields: [ + { key: "turnKeyId", label: "TURN Key ID", secret: false }, + { key: "apiToken", label: "TURN Key API Token", secret: true }, + ], + }, + ]); + }); + + it("accepts absent or explicit manual selection and rejects unsupported versions", () => { + expect(validateIceServerSourceConfiguration(undefined)).toBeUndefined(); + expect(validateIceServerSourceConfiguration({ version: 1, id: "manual" })).toBeUndefined(); + expect(validateIceServerSourceConfiguration({ version: 2, id: "cloudflare", configuration: {} })).toContain( + "version" + ); + expect(validateIceServerSourceConfiguration({ version: 1, id: "unknown", configuration: {} })).toContain( + "not supported" + ); + }); +}); diff --git a/src/main.ts b/src/main.ts index 549c7941..d83451da 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,3 +1,4 @@ +import { useIceServerSources } from "@/serviceFeatures/useIceServerSources"; import { getLanguage, Notice, Plugin, type App, type PluginManifest } from "./deps"; import { setGetLanguage } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; setGetLanguage(getLanguage); @@ -182,7 +183,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin { const replicator = useP2PReplicatorFeature( core, (_compatibilityReplicator, p2p) => createInteractiveP2PReplication(p2p), - createOpenRebuildUI(this.app) + createOpenRebuildUI(this.app), + { iceServerSources: useIceServerSources(core.services.API.webCompatFetch.bind(core.services.API)) } ); setupManager.registerP2PSetupConnectionProbe(replicator.connectionProbe); useP2PReplicatorCommands(core, replicator); diff --git a/src/modules/features/ModuleObsidianSettingAsMarkdown.ts b/src/modules/features/ModuleObsidianSettingAsMarkdown.ts index cbbdab3e..b44e140c 100644 --- a/src/modules/features/ModuleObsidianSettingAsMarkdown.ts +++ b/src/modules/features/ModuleObsidianSettingAsMarkdown.ts @@ -1,3 +1,8 @@ +import { + hasManagedTurnSettings, + omitManagedTurnProfilesFromMarkdown, + preserveManagedTurnProfilesOnMarkdownImport, +} from "@/common/turnSettingsPrivacy"; // import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser"; import { isObjectDifferent } from "octagonal-wheels/object"; import { EVENT_SETTING_SAVED, eventHub } from "@/common/events"; @@ -129,6 +134,7 @@ export class ModuleObsidianSettingsAsMarkdown extends AbstractModule { let settingToApply = { ...DEFAULT_SETTINGS } as ObsidianLiveSyncSettings; settingToApply = { ...settingToApply, ...newSetting }; + preserveManagedTurnProfilesOnMarkdownImport(newSetting, this.settings, settingToApply); if (!settingToApply?.writeCredentialsForSettingSync) { //New setting does not contains credentials. settingToApply.couchDB_USER = this.settings.couchDB_USER; @@ -208,11 +214,18 @@ export class ModuleObsidianSettingsAsMarkdown extends AbstractModule { delete saveData.couchDB_CustomHeaders; delete saveData.bucketCustomHeaders; } + omitManagedTurnProfilesFromMarkdown(saveData); return saveData; } async saveSettingToMarkdown(filename: string) { const saveData = this.generateSettingForMarkdown(); + if (hasManagedTurnSettings(this.settings)) { + this._log( + "Share TURN provider credentials through an encrypted Setup URI. Connection profiles are omitted from Markdown settings.", + LOG_LEVEL_INFO + ); + } const file = await this.core.storageAccess.isExists(filename); if (!file) { diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts index 260ec3c0..209f3eaa 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts @@ -1,3 +1,5 @@ +import { copySetupURI } from "@/serviceFeatures/setupObsidian/setupUri"; +import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; import { REMOTE_COUCHDB, REMOTE_MINIO, @@ -416,6 +418,15 @@ export function paneRemoteConfig( }) .addItem((item) => { item.setTitle("📤 Export").onClick(async () => { + if (config.uri.startsWith("sls+p2p-v2://")) { + await copySetupURI( + this.core, + createInstanceLogFunction("TURN setup sharing", this.services.API), + true, + getSettingsFromEditingSettings(this.editingSettings) + ); + return; + } await this.services.UI.promptCopyToClipboard( `Remote configuration: ${config.name}`, config.uri diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte index 8a79a891..9a064586 100644 --- a/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte +++ b/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte @@ -1,4 +1,6 @@ @@ -339,24 +345,24 @@ {translateMessage( - "TURN relay only is available when at least one valid TURN server URL is configured under Advanced Settings." + "TURN relay only requires a TURN server or a configured credential source under Advanced Settings." )} {translateMessage( - "TURN relay only requires at least one valid TURN server URL. Connection path has been restored to Automatic." + "TURN relay only requires TURN configuration. Connection path has been restored to Automatic." )} {translateMessage( - "TURN server settings are only necessary if you are behind a strict NAT or firewall that prevents direct P2P connections. In most cases, you can leave these fields blank." + "Configure TURN when a direct connection cannot be established or when you select TURN relay only." )} - + {translateMessage( - "TURN relays the encrypted WebRTC connection only when a direct path cannot be established. A TURN provider cannot read encrypted Vault contents, but it can observe connection metadata and traffic volume. Use a provider you trust." + "WebRTC encrypts data between your devices, including when it passes through TURN. The TURN provider cannot read the transferred data. It can see network addresses and traffic volume." )} {translateMessage("Learn more about signalling and TURN")}. - - - - - - - - - + {error} diff --git a/src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte b/src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte index 5300558b..08208e1d 100644 --- a/src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte +++ b/src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte @@ -1,6 +1,5 @@ From 11a07b26af247d590f04a24ed0221b4dcef80f7d Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 16 Sep 2026 03:43:09 +0000 Subject: [PATCH 09/15] Use host preparation for TURN connection settings --- devs.md | 2 +- .../2026_08_p2p_transport_compatibility.md | 8 +- .../design_docs/renewable_turn_credentials.md | 586 ++++-------------- docs/p2p.md | 16 +- docs/settings.md | 7 +- package-lock.json | 8 +- package.json | 2 +- .../BrowserP2PTransportSettings.svelte | 19 +- src/apps/cli/commands/runCommand.unit.spec.ts | 11 +- src/apps/cli/main.ts | 4 +- src/apps/webapp/WebAppRuntime.ts | 4 +- src/apps/webpeer/src/WebPeerRuntime.ts | 4 +- .../messages/LiveSyncProvisionalMessages.ts | 8 +- src/common/reportTool.ts | 4 +- src/common/reportTool.unit.spec.ts | 13 +- src/common/turnSettingsPrivacy.ts | 48 +- src/common/turnSettingsPrivacy.unit.spec.ts | 100 ++- src/features/P2PSync/TurnConfiguration.svelte | 64 +- src/integrations/cloudflare/settings.ts | 50 +- ...{iceServerSource.ts => turnCredentials.ts} | 277 ++++----- ...t.spec.ts => turnCredentials.unit.spec.ts} | 83 ++- src/integrations/iceServerSources.ts | 74 --- .../iceServerSources.unit.spec.ts | 31 - src/integrations/turnSettings.ts | 14 + src/main.ts | 4 +- .../SetupWizard/dialogs/SetupRemoteP2P.svelte | 12 +- .../dialogs/p2pSetupConnectionProbe.ts | 22 +- .../p2pSetupConnectionProbe.unit.spec.ts | 35 ++ .../setupObsidian/qrCode.unit.spec.ts | 8 +- src/serviceFeatures/useIceServerSources.ts | 17 - .../useP2PSettingsPreparation.ts | 18 + .../useP2PSettingsPreparation.unit.spec.ts | 48 ++ .../webpeer/browser-smoke.test.ts | 14 +- 33 files changed, 654 insertions(+), 961 deletions(-) rename src/integrations/cloudflare/{iceServerSource.ts => turnCredentials.ts} (53%) rename src/integrations/cloudflare/{iceServerSource.unit.spec.ts => turnCredentials.unit.spec.ts} (69%) delete mode 100644 src/integrations/iceServerSources.ts delete mode 100644 src/integrations/iceServerSources.unit.spec.ts create mode 100644 src/integrations/turnSettings.ts delete mode 100644 src/serviceFeatures/useIceServerSources.ts create mode 100644 src/serviceFeatures/useP2PSettingsPreparation.ts create mode 100644 src/serviceFeatures/useP2PSettingsPreparation.unit.spec.ts diff --git a/devs.md b/devs.md index d95d0a68..855fbc38 100644 --- a/devs.md +++ b/devs.md @@ -189,7 +189,7 @@ steps required to add a built-in provider. Commonlib owns one stable `LiveSyncP2PService`, its `P2PRoomSessionOwner`, and the replaceable Trystero room session. Host commands, event handlers, and views consume the focused transport, connection-probe admission, directory, peer-admission, transfer, change-relay, configuration, and diagnostic views returned by the service feature. They must not retain the deprecated compatibility Replicator as an ordinary service locator, close Trystero-owned raw peers, or install another Trystero transport generation at the application root. The exact implemented ownership and shutdown boundaries are recorded in Commonlib's [P2P transport lifecycle](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/p2p-transport-lifecycle.md) design document. -The proposed [TURN credential sources design](docs/design_docs/renewable_turn_credentials.md) covers credential expiry in the existing room reuse decision, replication continuation after room replacement, persisted and shared provider tokens, report redaction, and optional integrations on the device. It records the Commonlib work and compatibility boundaries before implementation. +The [TURN connection settings design](docs/design_docs/renewable_turn_credentials.md) describes how the host prepares temporary ICE credentials in a connection-only settings copy. It covers room reuse and expiry, replication continuation, profile persistence and sharing, and report redaction. ### Conflict Merge Policy diff --git a/docs/adr/2026_08_p2p_transport_compatibility.md b/docs/adr/2026_08_p2p_transport_compatibility.md index 004c6e5d..5eb38be0 100644 --- a/docs/adr/2026_08_p2p_transport_compatibility.md +++ b/docs/adr/2026_08_p2p_transport_compatibility.md @@ -60,15 +60,15 @@ The first settings revision retains the existing storage and dialogue contract o A future interface may present the existing comma-separated value as ordered `turn:` and `turns:` URL rows without changing its serialised representation. A structured list of multiple credential profiles is deferred until a provider or self-hosted use case requires different credentials in the same P2P profile. -Static long-term credentials are the supported first stage. Managed credentials use an optional source implementation on the device, behind a service-independent acquisition contract. Service-specific requests and settings belong under `src/integrations/`; Commonlib owns acquisition coordination and the P2P lifecycle. A separately operated HTTPS credential endpoint is an optional future source, not a prerequisite. +Static long-term credentials remain supported. For managed credentials, a host preparation hook requests ICE settings and places them on a connection-only copy of `P2PSyncSetting`. Service-specific requests and validation belong under `src/integrations/`; Commonlib consumes that copy and owns room reuse, expiry checks, and replacement. It has no provider catalogue or source factory. -A user-supplied provider API token is persisted as a sensitive P2P profile setting and included in encrypted Setup URI sharing, so that participating devices can use the same configuration without repeated token entry. Optional configuration encryption must cover every saved copy. Reports and logs redact the complete provider configuration and issued credentials, including inactive profiles and settings projections. Coturn's server-side shared authentication secret remains outside client settings. +A user-supplied provider API token is persisted as a sensitive P2P profile setting and included in encrypted Setup URI sharing, so that participating devices can use the same configuration without repeated token entry. Existing profile-URI encryption covers the saved token; its flat runtime projection is omitted from persistence. Reports and logs redact the complete provider configuration and issued credentials, including inactive profiles and settings projections. Coturn's server-side shared authentication secret remains outside client settings. Issued short-lived TURN credentials and their expiry remain in memory. The existing room reuse decision checks both the effective connection settings and credential validity. When reconciliation finds expired credentials, it uses the normal room retirement and replacement path with newly acquired credentials. Replacement may cancel an in-progress transfer; the next replication attempt uses stored checkpoints and revision comparison to retain received progress. Whether that next attempt starts automatically follows the existing synchronisation policy. -Time passing alone does not trigger acquisition or disconnection. This design adds no renewal timer, per-peer acquisition hook, configuration update on raw peers, or credential-driven ICE restart. Internal peer reconnection within an unchanged room does not guarantee fresh issuance. Acquisition failure is reported without changing the selected source or route policy. See [TURN credential sources](../design_docs/renewable_turn_credentials.md) for the proposed contract, persistence and sharing formats, room replacement, and verified replication continuation behaviour. +Time passing alone does not trigger acquisition or disconnection. This design adds no renewal timer, per-peer acquisition hook, configuration update on raw peers, or credential-driven ICE restart. Internal peer reconnection within an unchanged room does not guarantee fresh issuance. Acquisition failure is reported without changing the selected provider or route policy. See [TURN connection settings](../design_docs/renewable_turn_credentials.md) for the preparation hook, persistence and sharing formats, room replacement, and verified replication continuation behaviour. -When managed sources are introduced, relay-only validation accepts a valid managed TURN source configuration as well as the existing manual URL list. Failure to acquire usable TURN entries keeps relay-only mode selected and reports the connection failure; it does not restore `Automatic` silently. +Relay-only validation accepts a valid managed TURN configuration as well as the existing manual URL list. Failure to acquire usable TURN entries keeps relay-only mode selected and reports the connection failure; it does not restore `Automatic` silently. ### TURN allocation check and route diagnostics diff --git a/docs/design_docs/renewable_turn_credentials.md b/docs/design_docs/renewable_turn_credentials.md index cdef0ae8..4398e9d4 100644 --- a/docs/design_docs/renewable_turn_credentials.md +++ b/docs/design_docs/renewable_turn_credentials.md @@ -1,503 +1,171 @@ --- -date: 2026-09-15 -commonlib-version: "0.1.25-dev.turn-credentials.5" +date: 2026-09-16 +commonlib-version: "0.1.25-dev.turn-credentials.6" self-hosted-livesync-version: "1.0.28" status: unreleased --- -# TURN credential sources +# TURN credentials in P2P connection settings -## Purpose and decisions +## Purpose -This developer design addresses [Issue #1182](https://github.com/vrtmrz/obsidian-livesync/issues/1182) -through a service-independent interface for acquiring TURN credentials. +This design addresses [Issue #1182](https://github.com/vrtmrz/obsidian-livesync/issues/1182) +by acquiring temporary TURN credentials on the device before opening a P2P room. The [P2P transport compatibility ADR](../adr/2026_08_p2p_transport_compatibility.md) -records the accepted policy. The contract, lifecycle, settings, and host -integration are implemented locally. Real provider issuance, relay-only -Obsidian synchronisation, and synchronisation after explicit reconnection have -been verified. Expiry-driven TURN reconnection remains release validation work. +records the connection and persistence policy. -The design uses these decisions: +LiveSync prepares a connection copy of `P2PSyncSetting`. Commonlib owns the room +lifecycle and consumes the resulting ICE settings. Service-specific HTTP and +validation remain under `src/integrations/`; Commonlib has no provider catalogue +or versioned acquisition descriptor. Cloudflare is the first optional integration. +Manual TURN configuration remains available without a provider account. -- Acquire credentials on the device through an optional service integration. -- Persist the user-supplied provider API token with the P2P profile and include - it in encrypted Setup URI sharing for additional devices. -- Redact provider configuration and issued credentials from reports and logs. -- Keep issued short-lived credentials in memory only. -- Keep a local expiry alongside issued credentials and check it in the - existing room reuse decision. -- When that decision finds expired credentials, acquire a new configuration - and use the existing room replacement lifecycle. Replacement may cancel - an in-progress transfer; the next replication attempt reuses stored progress. -- Check expiry when the room lifecycle is reconciled. Add no renewal timer, - per-peer acquisition hook, `setConfiguration()`, or credential-driven ICE - restart. +## Settings and ownership -Manual TURN configuration remains supported without a provider account. -Cloudflare is the first optional integration. A separate credential endpoint, -a general authentication framework, runtime extension loading, and migration -of existing service integrations are outside the first delivery. - -## Ownership and composition - -An **ICE server source**, represented by `IceServerSource`, supplies ICE server -URLs, access credentials, and their expiry. This is developer vocabulary for -the acquisition contract; it is separate from a Replicator provider. - -| Component | Responsibility | Owner | +| Setting | Meaning | Lifetime | | --- | --- | --- | -| Source contract | Acquisition result, validation, and safe failure categories | Commonlib | -| Credential state and room reuse | Memory cache, expiry check, acquisition, cancellation, and room replacement | Commonlib `P2PRoomSessionOwner` | -| Physical peer creation | Use the configuration supplied when joining the room | Existing Trystero implementation | -| Source catalogue and settings | Explicit source selection and host dependencies | LiveSync | -| Cloudflare source | Provider request, response conversion, and configuration validation | LiveSync `src/integrations/cloudflare/` | +| `P2P_managedType` | Provider identifier; `CF` selects Cloudflare | P2P profile | +| `P2P_managedId` | Provider key identifier; Cloudflare TURN Key ID | P2P profile | +| `P2P_managedToken` | Provider API token used to request credentials | P2P profile | +| `P2P_iceServers` | Prepared `RTCIceServer[]` | One room connection | +| `P2P_iceServersExpiresAt` | Absolute expiry in Unix milliseconds | One room connection | -Implementation placement: +The first three values use ordinary ConnStr query parameters `managedType`, +`managedId`, and `token`. The existing `appId` parameter continues to identify the +P2P application. Commonlib reads and writes the three scalar values so profile +editing and activation preserve them. The host interprets the provider identifier. +An absent identifier selects the existing manual fields; an unsupported identifier +produces an explicit error when a connection is requested. -```text -Commonlib - P2P source contract and private credential cache - Expiry check in the existing room owner and session construction +Keep `P2P_turnServers`, `P2P_turnUsername`, and `P2P_turnCredential` for manual +configuration. Issuance does not overwrite them. Retain the complete ICE array: +individual entries can contain different credentials or STUN-only URLs. -LiveSync - src/integrations/iceServerSources.ts - src/integrations/cloudflare/iceServerSource.ts - src/integrations/cloudflare/settings.ts - src/serviceFeatures/useIceServerSources.ts -``` +## Host preparation -`integrations/` groups code which connects external services to the common -contract. It does not imply a hosted project service or a public extension -marketplace. The service feature composes a closed catalogue of source -factories with explicit dependencies, following -[Service feature and legacy Module boundaries](service_feature_and_legacy_module_boundaries.md). -An integration receives neither `LiveSyncBaseCore` nor ownership of replication. +The optional `prepareP2PSettings(settings, signal)` composition hook receives a +snapshot of requested P2P settings. LiveSync supplies the same preparation function +to Obsidian, CLI, WebApp, and WebPeer using each host's HTTP adapter. -Supply the catalogue through an optional composition argument to -`useP2PReplicatorFeature`, preserving its manual-only default for existing -Commonlib consumers. Factories validate settings without network access; -acquisition runs only when requested by the P2P owner. Unsupported sources -produce an explicit configuration error. - -```mermaid -flowchart LR - R["Existing room lifecycle reconciliation"] --> D{"Same binding and valid credentials?"} - D -->|"Yes"| K["Keep current room"] - D -->|"No"| C["Retire current room, if present"] - C --> A["Reuse valid cached credentials or acquire"] - A --> O["Open room with resolved ICE configuration"] -``` - -## Settings and dependencies - -Present a `TURN configuration` choice with `Manual` and `Cloudflare`. -The catalogue supplies each integration's label and fields; the common P2P -engine does not branch on a service name. - -| Input | Manual | Cloudflare | -| --- | --- | --- | -| TURN server URLs | Existing field | Supplied by the API | -| TURN username and credential | Existing fields | Issued in memory | -| TURN Key ID | Unused | Required and persisted | -| TURN Key API Token | Unused | Required, masked in the dialogue, and persisted | - -The first Cloudflare implementation requests a 24-hour lifetime internally. -It needs no account ID, email address, custom endpoint URL, or renewal interval -setting. This lifetime is a design default, not a provider default. - -Dependencies are an injected HTTP operation, a clock, cancellation/deadline -handling, and the existing settings and P2P lifecycle services. No Cloudflare -SDK, credential broker, or new operating-system secret-store dependency is -required. - -Retain `P2P_turnServers`, `P2P_turnUsername`, and `P2P_turnCredential` for manual -configuration. An absent source selection means manual. Add a versioned P2P -profile descriptor, `P2P_iceServerSource`: - -```json -{ - "version": 1, - "id": "cloudflare", - "configuration": { - "turnKeyId": "user-supplied-key-id", - "apiToken": "user-supplied-turn-key-api-token" - } -} -``` - -Commonlib owns the JSON envelope; each source owns validation of its -configuration. Unsupported identifiers and versions remain preserved in -storage and produce an explicit unsupported result when selected. Loading an -inactive profile performs no acquisition. - -The selected source configuration, including token changes, participates in -the effective P2P configuration identity. Issued credentials and their expiry -are separate runtime state. Room reuse requires both a matching identity and -usable credentials. Under managed selection, unused manual credentials do -not affect that identity; manual selection preserves the existing projection. -Keep the identity opaque and absent from diagnostics. Apply source changes -and expired runtime credentials through the existing room replacement policy. - -## Persistence, sharing, and redaction - -The API token is an ordinary sensitive connection setting. Persist it with -the profile so that restarting a device and configuring another device do -not require re-entry. This does not claim operating-system keychain storage. -Persist the source only in the ordinary P2P profile URI, covered by the -existing optional configuration encryption. The top-level source is an -in-memory and sharing projection restored when the selected profile is -activated. A source draft without a Group ID is not persisted. Failure to -encrypt a managed profile must leave the prior saved settings intact and -report a safe error; it must not silently save a plaintext replacement. - -| Destination | Provider API token | Issued TURN username and credential | -| --- | --- | --- | -| Saved P2P profile | Included | Omitted | -| Setup URI and QR code | Included with the source and Key ID | Omitted | -| Runtime room configuration | Available only to the source | Cached in memory and passed to WebRTC | -| General report or diagnostic log | Redacted | Redacted | - -Setup URI, QR code, and profile sharing include the source configuration as -ordinary connection settings. Preserve the independent main-remote and P2P -selections and the receiving device's own peer name. Setup URIs keep their -existing passphrase encryption; QR codes keep their existing unencrypted -format and 'FOR YOUR EYES ONLY' display. Include managed sources in inactive -profiles as well. Issued temporary credentials remain runtime state. - -Markdown settings export must not leak tokens through either the top-level -source or a profile URI. For this first delivery, omit the profile collection, -its selections, and the source projection together when managed profiles are -present, and explain that connection sharing uses Setup URIs or QR codes. -Importing Markdown without that group preserves the local profiles and -selections rather than replacing them with a filtered collection. - -Reports expose only safe source labels and acquisition state. Redact the -entire opaque source configuration, including unknown source configurations, -and every stored or projected copy. Preserve the existing scheme-only -redaction of profile URIs in `src/common/reportTool.ts`. Do not log request -headers, raw API bodies, source identity values, or HTTP errors which embed -credentials. Use one redaction policy across report and diagnostic paths; -cover inactive profiles and encoded values in tests. - -## Acquisition contract - -Commonlib exports the acquisition contract from `/p2p`: +For a managed selection, the function validates the provider inputs, requests +credentials, and returns a connection copy: ```typescript -type IceServerConfiguration = { - iceServers: readonly RTCIceServer[]; - expiresAt: number | null; +return { + ...settings, + P2P_iceServers: iceServers, + P2P_iceServersExpiresAt: expiresAt, }; - -declare class IceServerSourceError extends Error { - constructor( - code: "configuration" | "authentication" | "unavailable" | "invalid-response", - message: string, - retryable: boolean - ); -} - -interface IceServerSource { - acquire(signal: AbortSignal): Promise; -} ``` -`expiresAt` is a local Unix timestamp in milliseconds. `null` represents -non-expiring manual configuration; managed results require a finite expiry. -Sources throw a typed, safe failure or propagate cancellation. The room owner -calls the same operation when it needs an initial or replacement credential -set. A source does not save settings, schedule renewal, mutate peers, or -start replication. +Commonlib takes the prepared ICE fields into its session snapshot and passes that +snapshot through `ReplicatorHostEnv.settings`. The hook does not change the +requested room identity, persist settings, own replication, or schedule renewal. +Its HTTP request must settle on cancellation and has a bounded deadline. The room +owner also stops waiting for preparation when the connection request is retired. +An explicitly managed configuration requires a preparation hook and usable ICE +credentials; acquisition failure does not select a fallback provider or route. -Validate supported `stun:`, `stuns:`, `turn:`, and `turns:` URLs, complete TURN -credentials, bounded response size and entry count, and enough remaining -lifetime for connection establishment. A managed TURN source must return at -least one usable TURN entry. Copy the validated result before handing it to -WebRTC; unknown fields never become arbitrary `RTCConfiguration` options. -Preserve ordinary STUN behaviour and the selected connection-path policy. +## Room reuse and expiry -### Cloudflare request +The active connection settings hold the issued credentials. They are the only +credential cache. The existing room reuse decision checks: -The source calls the fixed provider API: +1. whether the requested database and connection settings still match; and +2. whether the active connection's credentials have enough remaining lifetime. -```http -POST https://rtc.live.cloudflare.com/v1/turn/keys/{TURN_KEY_ID}/credentials/generate-ice-servers -Authorization: Bearer {TURN_KEY_API_TOKEN} -Content-Type: application/json +The static connection signature includes the provider type, key ID, and token. +It excludes the generated ICE array and expiry. Comparing the prepared and stored +settings directly would incorrectly trigger issuance on every reconciliation. -{"ttl":86400} -``` +When reuse is unavailable, the owner retires the existing room, obtains a fresh +connection copy, and opens its replacement. It checks settings, room demand, +cancellation, and expiry again before publishing the replacement. A late result +cannot reopen a closed room or apply credentials requested for different settings. +Explicit reconnection acquires fresh credentials. Closing the room releases its +credential references. Preserve a 30-second connection-establishment margin. -Cloudflare returns an `iceServers` array. Its documented maximum lifetime is -48 hours, and the returned ICE server structure has no TTL. Derive the local -expiry from the requested TTL and the time before the request started, -allowing for request duration and a connection-establishment margin. Reject a -response which has already become too old. See -[credential generation](https://developers.cloudflare.com/realtime/turn/generate-credentials/) -and [the TURN FAQ](https://developers.cloudflare.com/realtime/turn/faq/). +Reconciliation runs at existing connection, settings, and lifecycle boundaries. +Time passing alone does not trigger acquisition or disconnection. There is no +renewal timer, per-peer acquisition, raw WebRTC configuration update, ICE restart, +or general retry mechanism. Trystero's internal peer reconnection within an +unchanged room uses that room's existing configuration. -Only the Key ID, API token, and requested lifetime go to the provider. The -source has no need for a Vault passphrase, Group ID, peer name, or file data. -Use a TURN Key API Token, not an account-wide API key. Cloudflare documents a -server-side secret model; this design explicitly permits users to place and -share their own issuance token on their participating devices. Whoever -receives that token can issue credentials under its authority. +Normal retirement may cancel an in-progress transfer. A later replication attempt +uses stored checkpoints and revision comparison to retain received progress. +An unfinished network message may be sent again. Whether another attempt starts +automatically continues to follow the existing synchronisation policy. -All maintained hosts inject `API.webCompatFetch`, using standard fetch -cancellation and redirect controls. The source refuses redirects, omits cookies, -requests `no-store`, and applies a 15-second deadline. It bounds the response to -32 KiB, 16 ICE entries, and 32 URLs. Commonlib independently validates the -result and requires at least 30 seconds of remaining lifetime before use. +## Persistence, sharing, and privacy -A read-only CORS preflight on 15 September 2026 returned HTTP 204 and allowed -POST, `Authorization`, and `Content-Type` from the requested origin. This -establishes preflight support, not successful authenticated issuance. Obsidian's -`nativeFetch` adapter is not used here because its `requestUrl` path does not -forward all required fetch controls. Provider HTTP behaviour is covered by -fixtures; operator-owned credentials are still required for real issuance and -TURN allocation validation. +Persist provider values only inside the selected P2P profile URI. Flat values in +runtime settings are a projection restored by profile activation. Profile edits +update that URI explicitly. General settings saves do not rebuild a P2P profile +from unrelated flat settings. Flat-settings migration creates and selects its +P2P profile once, independently of the selected main remote. -## Room reuse and credential expiry +Existing whole-profile encryption covers the saved API token. The default mode +uses the existing built-in key; a user-supplied configuration passphrase has its +existing protection semantics. Failure to encrypt a managed profile leaves the +previous saved data intact. No separate encrypted-token field is added. A draft +containing provider credentials but no Group ID remains unsaved. -### Runtime state and decision +Setup URIs and ordinary settings QR codes already contain `remoteConfigurations`. +The provider values travel inside that profile URI, including inactive profiles. +Omit their duplicate flat projections from sharing. No new URI scheme, encoded QR +slot, or encryption envelope is needed. Setup URIs retain passphrase encryption; +QR codes retain their unencrypted format and 'FOR YOUR EYES ONLY' display. -Keep one private cached result for the effective source configuration in the -P2P room owner. It contains the validated ICE servers, `expiresAt`, and the -source identity which produced them. Reuse it while that source still matches -and its remaining lifetime is sufficient. Clear it on source change, explicit -disconnect, suspension, or owner disposal. Neither the credentials nor the -expiry becomes a persisted setting. +Issued ICE credentials and expiry appear only in connection copies. Remove both +runtime fields at save, import, and sharing boundaries, including +`TrysteroReplicator.getAllConfig`, which starts from the session settings. +Incoming settings cannot install an issued credential override. Reports omit +runtime ICE fields, redact provider values, and retain scheme-only profile URIs. +Logs use safe errors and omit request headers, raw responses, and connection +signatures. Ordinary plaintext in process memory is permitted. -`expiresAt` is derived from issuance time and the requested TTL. A fixed TTL -value alone cannot identify whether an earlier issuance has expired. Keep the -expiry check separate from the stable settings signature rather than making -wall-clock time an ordinary configuration field. +Markdown settings omit managed provider values and the profile collection with +its selections. If that group is omitted during import, preserve the corresponding +local P2P connection values as well as the profiles. This prevents combining an +imported room with the local provider token or overwriting the saved profile. -The existing `reconcileTransport()` reuse decision becomes conceptually: +## Cloudflare integration -```typescript -const reusable = - current?.host.isServing && - bindingsMatch(activeBinding, desiredBinding) && - credentialsRemainUsable(activeCredentials, now); -``` +The UI presents `Manual` and `Cloudflare`, with `TURN Key ID` and a masked +`TURN Key API Token` input for Cloudflare. It requires no account ID, custom +endpoint, SDK, credential broker, or renewal interval setting. -Manual configuration has no managed expiry and preserves the existing -behaviour. For a managed source, a missing or expired result makes the room -ineligible for reuse even if the saved settings have not changed. +The provider function uses Cloudflare's +[credential-generation endpoint](https://developers.cloudflare.com/realtime/turn/generate-credentials/) +and converts its response into ICE servers. The implementation requests a 24-hour +lifetime and derives local expiry from the clock before the request starts. -When reuse is unavailable, use the existing lifecycle queue: +The HTTP boundary uses the injected standard fetch adapter with cancellation, +a 15-second deadline, refused redirects, omitted cookies, and disabled caching. +It bounds the response to 32 KiB, 16 ICE entries, and 32 URLs, and validates URLs +and complete TURN credentials. These are local implementation limits. Keep this +validation at the provider boundary instead of repeating it in Commonlib. -1. Retire the current session, if present. Its cancellation and settlement - path also handles any in-progress transfers. -2. Resolve valid cached credentials for the desired source, or await a new - `acquire()` result. Serialised reconciliation shares this work rather than - issuing a request for each physical peer. -3. Construct the replacement session with a temporary, resolved ICE - configuration. Keep that configuration separate from persisted manual - fields and the settings projection used for policy changes. -4. Before publishing the session, recheck the source identity, expiry, - enabled state, and room demand. Discard obsolete results and candidates. +The token is supplied and shared by the user on their devices. The provider +function sends the key ID, API token, and requested lifetime; it has no need for +Vault data, the Group ID, or the Vault passphrase. -Acquisition and room opening have bounded deadlines. A result which expires -before publication is unusable. Each reconciliation makes one acquisition -attempt; a later explicit retry or existing reconciliation can try again. A credential test uses its own result and does not replace -the active room's cache. +## Setup and verification -### When the check runs +The Setup connection test remains a signalling check. A separately owned trial +uses signalling-only settings and performs no managed TURN issuance. The existing +active-relay admission rule still applies. Success does not verify the API token, +TURN allocation, or document transfer. Actual room connections use the preparation +hook and preserve the selected route policy on failure. -Use existing reconciliation opportunities, including explicit connection, -changes to room demand, and applicable settings/lifecycle events. Time passing -alone does not run reconciliation or close a room. If reconciliation runs -after expiry, ordinary replacement may interrupt a transfer; no additional -idle wait or transfer-preservation mechanism is required. +Focused tests cover provider validation and cancellation, room reuse and expiry, +late results after configuration changes or closure, migration without duplicate +profiles, Markdown import through save/reload, safe acquisition failures, and +exclusion of runtime credentials from storage and sharing. -Not every operation passes this decision. A transfer admitted directly by an -existing session, a signalling WebSocket reconnect, and Trystero's internal -physical-peer reconnection can proceed without owner reconciliation. This -scope checks credential validity during room reconciliation and acquires a -new set when needed. Individual physical connection attempts use the room's -existing configuration. -A room which remains open beyond expiry may require an explicit reconnect -before new TURN-dependent peers can connect. - -### Existing transport boundary - -The inspected baseline is Commonlib `0.1.24` and Trystero `0.25.3`, as pinned -in the LiveSync lockfile: - -| Package boundary | Relevant behaviour | -| --- | --- | -| Commonlib `P2PRoomSessionOwner.reconcileTransport()` | Reuses an equivalent serving room; otherwise retires it and constructs another session. | -| Commonlib `P2PRoomSession.retire()` | Rejects new work, cancels current finite operations, waits for settlement, and disposes the room. | -| Commonlib `TrysteroReplicatorP2PServer.start()` | Supplies resolved options to Trystero when joining the room. | -| Trystero `dist/strategy.mjs` and `dist/offer-pool.mjs` | The final room leave destroys the outgoing offer pool; a later join can use new options. | -| Trystero `dist/shared-peer.mjs` | Live physical peers may survive logical room leave/rejoin under Trystero ownership. | - -Use the normal retire-before-open path. LiveSync does not close raw peers or -create another transport generation. The design requires no Trystero peer -factory extension, eager-pool change, or existing-peer configuration update. -Verify fresh TURN allocation after normal room replacement in the maintained -host topology; a still-connected shared peer can remain usable and is not -proof that a fresh allocation used the new credentials. This assumes one -active P2P room per host; pool replacement while another room remains open -needs separate validation. - -### Replication after interruption - -Commonlib `0.1.24` uses `replicateShim()` for P2P transfer. Its checkpoint is -stored in database-local documents, using the source and destination database -names and a source-side marker. The Trystero peer ID is not the checkpoint -identity. Rejoining the same databases with a new peer ID therefore retains -replication progress. - -For each batch, the shim reads changes, compares destination revisions with -`revsDiff`, fetches missing revisions, writes them with `new_edits: false`, -and invokes the processing callback before advancing the checkpoint. Room -retirement does not delete the database documents or replication checkpoints. - -Consequently, the next replication attempt starts at the last committed -checkpoint. If interruption or a lost response leaves writes beyond that -checkpoint, it may scan that batch again; revision comparison avoids fetching -already stored revisions again. Missing or incomplete document revisions are -retried. This preserves received Metadata and Chunks, but does not resume a -partially received network message at its last byte. Normal P2P calls use -`rewind: false`; database replacement, removed checkpoint state, or an explicit -rewind can require an earlier scan. - -Starting that next attempt follows existing synchronisation policy. An -unfinished AutoSync baseline remains eligible when an accepted matching peer -is advertised again: `P2PAutomationCoordinator` only records completed -baselines. A cancelled manual transfer does not automatically restart merely -because the room reconnects; the next requested synchronisation uses the -same stored progress. This feature adds no universal transfer retry loop and -does not report a cancelled attempt as successful. - -A focused check executed the pinned `ReplicatorShim.js` with in-memory -database boundaries and confirmed both cancellation after a committed batch -and loss of completion after writes but before the checkpoint. Both subsequent -attempts fetched only missing revisions. The pinned automation coordinator -also allowed another attempt after a cancelled baseline. These checks verify -the algorithms; they do not establish real WebRTC reconnection or file -reflection behaviour, which remains part of implementation validation. - -### Failure and cancellation - -Acquisition failure leaves the attempted room opening unavailable and reports -a safe, actionable state. Do not fall back to saved manual credentials, -choose another provider, or relax relay-only mode. Authentication and -configuration errors wait for correction or an explicit retry. Transient -failures are marked retryable for the existing lifecycle or an explicit retry; -this source adds no automatic acquisition or reconnect loop. - -Explicit disconnect, source changes, and application suspension invalidate -pending acquisition. A late HTTP result cannot publish a room or restore an -obsolete source. Cancellation must take effect while room opening awaits -acquisition rather than waiting behind it in the lifecycle queue. The owner -rechecks current demand and configuration before exposing a replacement. - -## Compatibility and verification - -### Stored settings and sharing formats - -Update Commonlib's P2P setting type, `pickP2PSyncSettings`, connection-string -parser, Setup URI processing, and settings encryption together. Update the -LiveSync Setup dialogue, import handler, profile export, Markdown settings, -and report paths. Existing fixed-field serialisers would otherwise discard -the source. Generated credentials never populate the manual fields. - -P2P profiles keep `sls+p2p://` and carry the optional source descriptor in an -additional `source` query parameter. Full Setup URIs keep -`obsidian://setuplivesync?settings=` and encrypt the existing settings object -directly. QR codes carry the same source through an appended setting index. -Existing setting indices and the established formats remain unchanged. - -Missing source settings use the ordinary manual defaults. Older clients follow -their existing handling of additional fields; this feature adds no URI-version -gate or stored-settings restriction to prevent them from loading settings. -Clients which support source descriptors preserve unknown identifiers and -versions and validate them before activation, rather than silently selecting -manual TURN when an explicitly configured source is unsupported. - -Persist the ordinary P2P settings projection alongside its profile. Keep the -Group ID, enabled state, and autostart preference consistent with the current -settings. The source configuration is persisted only inside the profile URI -and restored by the existing profile activation. Issued credentials never -populate the persisted manual TURN fields. - -The P2P data protocol and Group ID remain unchanged. A peer using manually -configured TURN can communicate with one using issued credentials; validate -that interoperability without requiring both peers to use the same issuer. - -### Real-provider verification - -On 15 September 2026, the local LiveSync build with Commonlib -`0.1.25-dev.turn-credentials.3` passed a real Cloudflare TURN check in two -isolated Obsidian 1.12.7 instances on one Linux host. Both instances used the -Cloudflare source and `P2P_connectionPath: "relay"`, with a local Nostr relay -used only for signalling. - -- The source received HTTP 201 responses and acquired credentials with a - requested 24-hour lifetime. The Obsidian instances also received successful - issuance responses through their own HTTP integration. -- Both endpoints reported selected local and remote candidates of type - `relay`, using UDP, before transferring a note. The receiving Vault contained - the expected note content after replication completed. -- Explicitly disconnecting one instance removed its peer advertisement from - the other. Reconnecting issued credentials again and established a new - relay-only connection. A second note then travelled in the reverse direction - and appeared with the expected content in the receiving Vault. - -This check covers initial provider issuance, real relayed replication, and -credential reacquisition after an explicit disconnect. It does not establish -natural TTL expiry, interruption within a replication batch, mobile operating -system behaviour, mixed manual/managed peers, or connectivity between different -networks. Those cases retain their separate validation requirements. The -results contain no provider token, TURN username, or TURN credential. - -### Acceptance criteria for implementation - -- Manual configuration, default STUN, and existing Setup URIs retain their - behaviour. Unsupported managed sources fail explicitly. -- Provider tokens survive restart, profile selection, optional configuration - encryption, and ordinary Setup URI and QR code sharing. Reports and logs reveal no - tokens or issued credentials, including inactive and encoded copies. -- Issued credentials never enter persisted settings, exports, or reports. -- Equivalent settings and valid credentials reuse the room. Expired - credentials cause the next owner reconciliation to acquire and replace - through the existing lifecycle; manual settings retain their behaviour. -- Concurrent reconciliation does not duplicate acquisition. Late responses - after disconnect, source change, or suspension cannot publish a room. - Expiry tests cover delayed responses and clock changes. -- Time passing alone triggers no acquisition or replacement. There is no - per-peer acquisition hook, `setConfiguration()`, or credential-driven ICE - restart. -- Replacement during a batch settles the old attempt and preserves stored - documents and checkpoints. The next attempt transfers missing revisions; - test interrupted AutoSync and explicit manual retry separately. -- Safe failures cover authentication, rate limits, network errors, timeouts, - and malformed responses without an automatic source or route-policy change. -- Real relay-only connections verify initial establishment and room - replacement after expiry, including mixed manual/managed peers and both - initiator roles. A selected relayed candidate pair is required evidence; - direct traffic alone does not validate TURN use. -- Real Obsidian checks cover HTTP behaviour, desktop/mobile lifecycle, - persistence/sharing, and a file round trip after reconnection. Validate - supported CLI/browser hosts before enabling their direct integration. - -The existing Setup connection check remains a signalling check. Credential -issuance, a disposable TURN allocation check, actual peer data transfer, and -LiveSync file synchronisation establish different facts. Tests and status -must identify which boundary they verify. - -Implement the Commonlib contract, settings, runtime expiry, and existing room -replacement integration in its own repository. Validate the packed Commonlib -artefact before updating LiveSync's exact dependency and composing the -Cloudflare source. Use deterministic provider fixtures and an open-source -Coturn test service for repeatable -coverage; verify the real provider path with operator-owned test credentials. - -Run Commonlib checks, LiveSync `npm run check`, unit tests, builds, and focused -consumer tests for the implementation. Deterministic source, lifecycle, persistence, sharing, and redaction tests -cover the implemented boundaries. Real provider allocation and host -reconnection evidence must be recorded separately before release. +Validate Commonlib as an exact packed artefact before testing its LiveSync +consumer. Verify the changed settings and restart boundary in real Obsidian. +Previously observed provider issuance and relay synchronisation do not establish +expiry-driven reconnection for a revised build. Fresh TURN allocation after +expiry, mobile runtimes, and cross-network behaviour require their own runtime +verification; a surviving Trystero shared peer is not evidence of new allocation. diff --git a/docs/p2p.md b/docs/p2p.md index c2af90f8..0a0c1d09 100644 --- a/docs/p2p.md +++ b/docs/p2p.md @@ -56,16 +56,16 @@ for creating a TURN key and its API token. The API token is saved with the P2P profile and included when sharing settings through an existing Setup URI or QR code. Setup URIs retain their existing passphrase encryption. QR codes retain their existing unencrypted format and -'FOR YOUR EYES ONLY' display. Missing source settings use the ordinary manual -configuration defaults. Receiving clients need support for the selected source +'FOR YOUR EYES ONLY' display. Missing provider settings use the ordinary manual +configuration defaults. Receiving clients need support for the selected provider to acquire its temporary TURN credentials. Markdown settings omit the connection profile group when it contains a managed -TURN source, including inactive profiles, and importing those omitted settings -preserves this device's existing profiles. Diagnostic reports redact the source -configuration. Optional configuration encryption also covers the saved token. +TURN provider, including inactive profiles, and importing those omitted settings +preserves this device's existing profiles. Diagnostic reports redact provider settings. The existing profile-URI +encryption also covers the saved token. -Each device requests temporary TURN credentials before opening a room when no -valid credentials are cached. Cloudflare credentials have a requested lifetime +Each device requests temporary TURN credentials when opening a new room. +An existing room reuses its credentials while they remain valid. Cloudflare credentials have a requested lifetime of 24 hours and remain in memory only. Expiry is checked when LiveSync next reconciles the room connection. If necessary, it replaces the room and obtains new credentials. There is no periodic renewal: if a long-lived room cannot @@ -82,7 +82,7 @@ interrupted manual operation, use **Replicate now** again. `P2P Configuration` includes a separate `Connection compatibility` section. Its defaults preserve the existing transport behaviour: - **P2P message size** defaults to **Standard**. **Reduced**, **Conservative**, and **Maximum compatibility** progressively limit outgoing P2P messages when a network path appears to drop larger WebRTC messages. This is not a Vault Chunk size or an IP MTU. Smaller values add framing and processing overhead. -- **Connection path** defaults to **Automatic**, which lets WebRTC select a viable direct or TURN-relayed path. **TURN relay only** forces the encrypted connection through TURN and is available when the profile contains a valid manual TURN URL or a configured TURN credential source. +- **Connection path** defaults to **Automatic**, which lets WebRTC select a viable direct or TURN-relayed path. **TURN relay only** forces the encrypted connection through TURN and is available when the profile contains a valid manual TURN URL or a configured TURN provider. The sending device controls its outgoing message size. Select the same conservative preset on every device which may send across the constrained path. Existing devices do not receive the choice retrospectively merely because another device changed it. diff --git a/docs/settings.md b/docs/settings.md index 84408af9..66736ac0 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -487,19 +487,18 @@ When enabled, this device notifies connected peers after a local change. The not #### TURN configuration -Setting key: P2P_iceServerSource +Setting key: P2P_managedType Select **Manual** for the existing TURN server fields, or **Cloudflare** for a TURN Key ID and TURN Key API Token. The API token is persisted with the profile and included in Setup URI and QR code sharing. Issued temporary credentials are -kept in memory only. Reports redact the source configuration. See +kept in memory only. Reports redact the provider settings. See [TURN credentials](p2p.md#turn-credentials) for sharing, expiry, and reconnect behaviour. #### TURN Key ID and TURN Key API Token -Setting keys: P2P_iceServerSource.configuration.turnKeyId, -P2P_iceServerSource.configuration.apiToken +Setting keys: P2P_managedId, P2P_managedToken These fields appear when **Cloudflare** is selected. Enter the TURN key's ID and its dedicated API token. The token field is masked. No account ID, custom diff --git a/package-lock.json b/package-lock.json index a8343088..b4c11770 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.5.tgz", + "@vrtmrz/livesync-commonlib": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.6.tgz", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4567,9 +4567,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.25-dev.turn-credentials.5", - "resolved": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.5.tgz", - "integrity": "sha512-mAKjoJoXKMl1ts8mUTvYsjJy4bluGykBCvwXDgjtY7cLrG6ct0iRrlyVrLs7GY7xnVnWeNmu/vj12D9IMzEpCQ==", + "version": "0.1.25-dev.turn-credentials.6", + "resolved": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.6.tgz", + "integrity": "sha512-16qfxsLdUhQIFD68LIC/j6p9k/QdZiocH0GtL/LFOMo5uLqK88h3NITdjlZUHjMfLJBkTfxVIjZ6dV3AzgaxXQ==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 3ff89673..7d907fee 100644 --- a/package.json +++ b/package.json @@ -181,7 +181,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.5.tgz", + "@vrtmrz/livesync-commonlib": "file:.devmemo.local/packages/vrtmrz-livesync-commonlib-0.1.25-dev.turn-credentials.6.tgz", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/src/apps/browser/BrowserP2PTransportSettings.svelte b/src/apps/browser/BrowserP2PTransportSettings.svelte index 6f332cda..56bd5831 100644 --- a/src/apps/browser/BrowserP2PTransportSettings.svelte +++ b/src/apps/browser/BrowserP2PTransportSettings.svelte @@ -5,7 +5,7 @@ import type { P2PSyncSetting } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { P2PReplicatorPaneHost } from "@/features/P2PSync/P2PReplicator/P2PReplicatorPaneHost"; import TurnConfiguration from "@/features/P2PSync/TurnConfiguration.svelte"; - import { validateIceServerSourceConfiguration } from "@/integrations/iceServerSources"; + import { validateManagedTurnSettings } from "@/integrations/turnSettings"; let { host }: { host: P2PReplicatorPaneHost } = $props(); const currentSettings = () => host.services.setting.currentSettings() as P2PSyncSetting; @@ -15,21 +15,23 @@ P2P_turnServers: settings.P2P_turnServers, P2P_turnUsername: settings.P2P_turnUsername, P2P_turnCredential: settings.P2P_turnCredential, - P2P_iceServerSource: structuredClone(settings.P2P_iceServerSource), + P2P_managedType: settings.P2P_managedType, + P2P_managedId: settings.P2P_managedId, + P2P_managedToken: settings.P2P_managedToken, }; } let draft = $state(turnSettings(currentSettings())); let saved = $state(JSON.stringify(turnSettings(currentSettings()))); const isModified = $derived(JSON.stringify(draft) !== saved); - const sourceError = $derived(validateIceServerSourceConfiguration(draft.P2P_iceServerSource)); - const sourceNeedsRoom = $derived(!!draft.P2P_iceServerSource && (draft.P2P_roomID ?? "").trim() === ""); + const sourceError = $derived(validateManagedTurnSettings(draft)); + const sourceNeedsRoom = $derived(!!draft.P2P_managedType && (draft.P2P_roomID ?? "").trim() === ""); function loadSettings(settings: P2PSyncSetting): void { const next = turnSettings(settings); draft = next; saved = JSON.stringify(next); } - onMount(() => host.services.context.events.onEvent("setting-saved", (settings) => loadSettings(settings as P2PSyncSetting))); + onMount(() => host.services.context.events.onEvent("setting-saved", () => loadSettings(currentSettings()))); async function save(): Promise { if (sourceError || sourceNeedsRoom) return; @@ -38,8 +40,11 @@ const next = { ...settings, ...values, remoteConfigurations: { ...settings.remoteConfigurations } }; const profileId = settings.P2P_ActiveRemoteConfigurationId || (settings.remoteType === REMOTE_P2P ? settings.activeConfigurationId : ""); - if (profileId && next.remoteConfigurations[profileId]) { - upsertRemoteConfigurationInPlace(next, "p2p", { id: profileId }); + const selected = next.remoteConfigurations[profileId]; + if (selected?.uri.startsWith("sls+p2p://")) { + upsertRemoteConfigurationInPlace(next, "p2p", { id: profileId, activateForP2P: true }); + } else if (values.P2P_managedType) { + upsertRemoteConfigurationInPlace(next, "p2p", { activateForP2P: true }); } return next; }, true); diff --git a/src/apps/cli/commands/runCommand.unit.spec.ts b/src/apps/cli/commands/runCommand.unit.spec.ts index 3844cdf3..f4d3a45a 100644 --- a/src/apps/cli/commands/runCommand.unit.spec.ts +++ b/src/apps/cli/commands/runCommand.unit.spec.ts @@ -421,16 +421,15 @@ describe("runCommand abnormal cases", () => { it("setup imports managed TURN through the existing encrypted URI", async () => { const core = createCoreMock(); - const source = { - version: 1, - id: "cloudflare", - configuration: { turnKeyId: "turn-key", apiToken: "private-token" }, + const profiles = { + turn: { id: "turn", name: "TURN", isEncrypted: false, + uri: "sls+p2p://room?managedType=CF&managedId=turn-key&token=private-token" }, }; const passphrase = "setup-passphrase"; const setupURI = await processSetting.encodeSettingsToSetupURI( { ...DEFAULT_SETTINGS, - P2P_iceServerSource: source, + remoteConfigurations: profiles, }, passphrase ); @@ -439,7 +438,7 @@ describe("runCommand abnormal cases", () => { core.services.context.standardIo.prompt.mockResolvedValue(passphrase); await runCommand(makeOptions("setup", [setupURI]), { ...context, core }); expect(core.services.setting.applyExternalSettings).toHaveBeenCalledWith( - expect.objectContaining({ P2P_iceServerSource: source }), + expect.objectContaining({ remoteConfigurations: profiles }), true ); }); diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 7f17e0a9..3fd1ccb9 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -1,4 +1,4 @@ -import { useIceServerSources } from "@/serviceFeatures/useIceServerSources"; +import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation"; import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub"; import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage"; import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore"; @@ -526,7 +526,7 @@ export async function main( } // Register P2P replicator feature. p2pReplicator = useP2PReplicatorFeature(core, undefined, undefined, { - iceServerSources: useIceServerSources(core.services.API.webCompatFetch.bind(core.services.API)), + prepareP2PSettings: useP2PSettingsPreparation(core.services.API.webCompatFetch.bind(core.services.API)), }); // Add target filter to prevent internal files are handled core.services.vault.isTargetFile.addHandler(async (target) => { diff --git a/src/apps/webapp/WebAppRuntime.ts b/src/apps/webapp/WebAppRuntime.ts index 9fa841cb..48405a4c 100644 --- a/src/apps/webapp/WebAppRuntime.ts +++ b/src/apps/webapp/WebAppRuntime.ts @@ -1,4 +1,4 @@ -import { useIceServerSources } from "@/serviceFeatures/useIceServerSources"; +import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation"; /** Browser runtime for Self-hosted LiveSync over the File System Access API. */ import { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; @@ -219,7 +219,7 @@ export class WebAppRuntime { useCheckRemoteSize(core); useRemoteConfiguration(core); this.p2p = useP2PReplicatorFeature(core, undefined, undefined, { - iceServerSources: useIceServerSources(core.services.API.webCompatFetch.bind(core.services.API)), + prepareP2PSettings: useP2PSettingsPreparation(core.services.API.webCompatFetch.bind(core.services.API)), }); this.paneHost = { services: core.services, diff --git a/src/apps/webpeer/src/WebPeerRuntime.ts b/src/apps/webpeer/src/WebPeerRuntime.ts index da0d87e8..b4f69758 100644 --- a/src/apps/webpeer/src/WebPeerRuntime.ts +++ b/src/apps/webpeer/src/WebPeerRuntime.ts @@ -1,4 +1,4 @@ -import { useIceServerSources } from "@/serviceFeatures/useIceServerSources"; +import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation"; import { type P2PSyncSetting, SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { EVENT_LAYOUT_READY } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; @@ -72,7 +72,7 @@ export class WebPeerRuntime { }, }); this.p2p = useP2PReplicatorFeature({ services: this.services, serviceModules: {} }, undefined, undefined, { - iceServerSources: useIceServerSources(this.services.API.webCompatFetch.bind(this.services.API)), + prepareP2PSettings: useP2PSettingsPreparation(this.services.API.webCompatFetch.bind(this.services.API)), }); this.p2pLogCollector = new P2PLogCollector(this.events); this.paneHost = { diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 7274a81a..80f96040 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -21,17 +21,11 @@ export const liveSyncProvisionalEnglishMessages = { "TURN relay only requires a TURN server or a configured credential source under Advanced Settings.", "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.": "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.", - "Cloudflare TURN configuration is invalid.": "Cloudflare TURN configuration is invalid.", - "Cloudflare TURN configuration contains an unsupported field.": - "Cloudflare TURN configuration contains an unsupported field.", "Enter a TURN Key ID.": "Enter a TURN Key ID.", "TURN Key ID contains unsupported characters.": "TURN Key ID contains unsupported characters.", "Enter a TURN Key API Token.": "Enter a TURN Key API Token.", "TURN Key API Token must use Bearer token syntax.": "TURN Key API Token must use Bearer token syntax.", - "TURN configuration source version is not supported.": "TURN configuration source version is not supported.", - "TURN configuration source is invalid.": "TURN configuration source is invalid.", - "The selected TURN configuration source is not supported.": - "The selected TURN configuration source is not supported.", + "The selected TURN configuration is not supported.": "The selected TURN configuration is not supported.", "Setup Complete: Preparing to Fetch from Another Device": "Setup Complete: Preparing to Fetch from Another Device", "The P2P connection has been configured successfully. The initial synchronisation data must now be fetched from an online source device.": diff --git a/src/common/reportTool.ts b/src/common/reportTool.ts index e8bded42..ab174bb8 100644 --- a/src/common/reportTool.ts +++ b/src/common/reportTool.ts @@ -1,4 +1,4 @@ -import { redactTurnSourceForReport } from "./turnSettingsPrivacy"; +import { redactTurnSettingsForReport } from "./turnSettingsPrivacy"; import { REMOTE_COUCHDB, REMOTE_MINIO } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.const"; import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/settings"; import { generateCredentialObject } from "@vrtmrz/livesync-commonlib/compat/replication/httplib"; @@ -68,7 +68,7 @@ export async function generateReport(settings: ObsidianLiveSyncSettings, core: L delete pluginConfig[key as keyof ObsidianLiveSyncSettings]; } - redactTurnSourceForReport(pluginConfig); + redactTurnSettingsForReport(pluginConfig); pluginConfig.couchDB_DBNAME = REDACTED; pluginConfig.couchDB_PASSWORD = REDACTED; const scheme = pluginConfig.couchDB_URI.startsWith("http:") diff --git a/src/common/reportTool.unit.spec.ts b/src/common/reportTool.unit.spec.ts index 295e6ab0..0fe7aa23 100644 --- a/src/common/reportTool.unit.spec.ts +++ b/src/common/reportTool.unit.spec.ts @@ -10,19 +10,21 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions", () => ({ })); describe("TURN credentials in diagnostic reports", () => { - it("redacts top-level and inactive encoded source copies", async () => { + it("redacts provider tokens in all profiles and runtime credentials", async () => { const token = "private+token/with=symbols"; - const source = { version: 1, id: "cloudflare", configuration: { turnKeyId: "private-key", apiToken: token } }; + const provider = { P2P_managedType: "CF", P2P_managedId: "private-key", P2P_managedToken: token }; const settings = { ...DEFAULT_SETTINGS, remoteType: REMOTE_P2P, - P2P_iceServerSource: source, + ...provider, + P2P_iceServers: [{ urls: "turn:example.test", username: "issued-user", credential: "issued-password" }], + P2P_iceServersExpiresAt: 123456789, remoteConfigurations: { inactive: { id: "inactive", name: "Inactive TURN", isEncrypted: false, - uri: `sls+p2p://room?source=${encodeURIComponent(JSON.stringify(source))}`, + uri: `sls+p2p://room?managedType=CF&managedId=private-key&token=${encodeURIComponent(token)}`, }, }, }; @@ -33,6 +35,7 @@ describe("TURN credentials in diagnostic reports", () => { expect(text).not.toContain(encodeURIComponent(token)); expect(text).not.toContain("private-key"); expect(report.pluginConfig.remoteConfigurations.inactive.uri).toBe("sls+p2p://"); - expect(settings.P2P_iceServerSource).toEqual(source); + expect(settings.P2P_managedToken).toBe(token); + expect(text).not.toMatch(/issued-user|issued-password|P2P_iceServers/); }); }); diff --git a/src/common/turnSettingsPrivacy.ts b/src/common/turnSettingsPrivacy.ts index 61b67f66..6a67ddcb 100644 --- a/src/common/turnSettingsPrivacy.ts +++ b/src/common/turnSettingsPrivacy.ts @@ -1,45 +1,50 @@ import { - hasManagedP2PIceServerSource, + hasManagedP2PTurnConfiguration, type ObsidianLiveSyncSettings, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { pickP2PSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { CLOUDFLARE_TURN_TYPE } from "@/integrations/cloudflare/settings"; -import { iceServerSourceDefinitions } from "@/integrations/iceServerSources"; - -/** Include inactive profiles when deciding whether Markdown would disclose source settings. */ +/** Include inactive profiles when deciding whether Markdown would disclose provider settings. */ export function hasManagedTurnSettings(settings: Partial): boolean { return ( - hasManagedP2PIceServerSource(settings) || + hasManagedP2PTurnConfiguration(settings) || Object.values(settings.remoteConfigurations ?? {}).some(({ uri }) => { if (!uri.startsWith("sls+p2p://")) return false; const queryStart = uri.indexOf("?"); - return queryStart >= 0 && new URLSearchParams(uri.slice(queryStart + 1).split("#", 1)[0]).has("source"); + return ( + queryStart >= 0 && new URLSearchParams(uri.slice(queryStart + 1).split("#", 1)[0]).has("managedType") + ); }) ); } -/** Reports retain the selected source label, but no opaque source configuration. */ -export function redactTurnSourceForReport(settings: Partial): void { - if (settings.P2P_iceServerSource !== undefined) { - settings.P2P_iceServerSource = { - version: 1, - id: - iceServerSourceDefinitions.find((source) => source.id === settings.P2P_iceServerSource?.id)?.id ?? - "redacted", - configuration: { redacted: true }, - }; +/** Reports retain a recognised provider label and omit issued credentials. */ +export function redactTurnSettingsForReport(settings: Partial): void { + if (settings.P2P_managedType) { + settings.P2P_managedType = + settings.P2P_managedType === CLOUDFLARE_TURN_TYPE ? CLOUDFLARE_TURN_TYPE : "redacted"; } + if (settings.P2P_managedId !== undefined) settings.P2P_managedId = "redacted"; + if (settings.P2P_managedToken !== undefined) settings.P2P_managedToken = "redacted"; + delete settings.P2P_iceServers; + delete settings.P2P_iceServersExpiresAt; } /** Managed connection profiles are shared through Setup URIs and QR codes. */ export function omitManagedTurnProfilesFromMarkdown(settings: Partial): void { + delete settings.P2P_iceServers; + delete settings.P2P_iceServersExpiresAt; if (!hasManagedTurnSettings(settings)) return; - delete settings.P2P_iceServerSource; + delete settings.P2P_managedType; + delete settings.P2P_managedId; + delete settings.P2P_managedToken; delete settings.remoteConfigurations; delete settings.activeConfigurationId; delete settings.P2P_ActiveRemoteConfigurationId; } -/** An omitted profile group leaves this device's existing connection selection intact. */ +/** Preserve the complete connection when Markdown omits its profile group. */ export function preserveManagedTurnProfilesOnMarkdownImport( incoming: Partial, current: ObsidianLiveSyncSettings, @@ -48,12 +53,11 @@ export function preserveManagedTurnProfilesOnMarkdownImport( if ( !hasManagedTurnSettings(current) || incoming.remoteConfigurations !== undefined || - incoming.P2P_iceServerSource !== undefined - ) { + incoming.P2P_managedType !== undefined + ) return; - } merged.remoteConfigurations = structuredClone(current.remoteConfigurations); merged.activeConfigurationId = current.activeConfigurationId; merged.P2P_ActiveRemoteConfigurationId = current.P2P_ActiveRemoteConfigurationId; - merged.P2P_iceServerSource = structuredClone(current.P2P_iceServerSource); + Object.assign(merged, pickP2PSyncSettings(current)); } diff --git a/src/common/turnSettingsPrivacy.unit.spec.ts b/src/common/turnSettingsPrivacy.unit.spec.ts index 0124028b..c0243c79 100644 --- a/src/common/turnSettingsPrivacy.unit.spec.ts +++ b/src/common/turnSettingsPrivacy.unit.spec.ts @@ -1,26 +1,55 @@ import { describe, expect, it } from "vitest"; -import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + DEFAULT_SETTINGS, + REMOTE_P2P, + type ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + SettingService, + type SettingServiceDependencies, +} from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService"; +import { ServiceContext } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase"; +import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; import { hasManagedTurnSettings, omitManagedTurnProfilesFromMarkdown, preserveManagedTurnProfilesOnMarkdownImport, - redactTurnSourceForReport, + redactTurnSettingsForReport, } from "./turnSettingsPrivacy"; +class MemorySettingService extends SettingService { + readonly items = new Map(); + saved?: ObsidianLiveSyncSettings; + protected setItem(key: string, value: string) { + this.items.set(key, value); + } + protected getItem(key: string) { + return this.items.get(key) ?? ""; + } + protected deleteItem(key: string) { + this.items.delete(key); + } + protected saveData(settings: ObsidianLiveSyncSettings) { + this.saved = structuredClone(settings); + return Promise.resolve(); + } + protected loadData() { + return Promise.resolve(this.saved); + } +} + function configuredSettings() { return { ...DEFAULT_SETTINGS, - P2P_iceServerSource: { - version: 1, - id: "cloudflare", - configuration: { turnKeyId: "private-key-id", apiToken: "private-token" }, - }, + P2P_managedType: "CF", + P2P_managedId: "private-key-id", + P2P_managedToken: "private-token", remoteConfigurations: { managed: { id: "managed", name: "Managed TURN", isEncrypted: false, - uri: "sls+p2p://room?source=private-token", + uri: "sls+p2p://room?managedType=CF&managedId=private-key-id&token=private-token", }, }, activeConfigurationId: "central", @@ -29,17 +58,56 @@ function configuredSettings() { } describe("managed TURN settings privacy", () => { - it("redacts all opaque source fields, including unknown integrations", () => { + it("preserves the active managed room through Markdown import, save, and reload", async () => { + const current = { + ...configuredSettings(), + remoteType: REMOTE_P2P, + activeConfigurationId: "managed", + P2P_roomID: "local-room", + P2P_relays: "wss://local-relay.example.test", + P2P_passphrase: "local-passphrase", + }; + const originalURI = ConnectionStringParser.serialize({ type: "p2p", settings: current }); + current.remoteConfigurations.managed.uri = originalURI; + const service = new MemorySettingService(new ServiceContext(), { + APIService: { + getSystemVaultName: () => "test-vault", + getAppID: () => "test-app", + addLog: () => undefined, + confirm: { askString: async () => "" }, + } as unknown as SettingServiceDependencies["APIService"], + }); + service.settings = structuredClone(current); + const incoming: Partial = { + P2P_roomID: "imported-room", + P2P_relays: "wss://imported-relay.example.test", + P2P_passphrase: "imported-passphrase", + }; + const merged = { ...structuredClone(DEFAULT_SETTINGS), ...incoming }; + preserveManagedTurnProfilesOnMarkdownImport(incoming, current, merged); + await service.applyExternalSettings(merged, true); + const saved = service.saved!.remoteConfigurations.managed; + const uri = saved.isEncrypted ? await service.decryptConfigurationItem(saved.uri, "*") : saved.uri; + expect(uri).toBe(originalURI); + expect(service.settings.P2P_roomID).toBe("local-room"); + await service.loadSettings(); + expect(service.settings.P2P_roomID).toBe("local-room"); + }); + + it("redacts provider fields and issued credentials, including unknown integrations", () => { const settings = configuredSettings(); - settings.P2P_iceServerSource.id = "private-token"; - redactTurnSourceForReport(settings); - expect(JSON.stringify(settings.P2P_iceServerSource)).not.toMatch(/private-token|private-key-id/); - expect(settings.P2P_iceServerSource.configuration).toEqual({ redacted: true }); + settings.P2P_managedType = "private-token"; + redactTurnSettingsForReport(settings); + expect([settings.P2P_managedType, settings.P2P_managedId, settings.P2P_managedToken]).toEqual([ + "redacted", + "redacted", + "redacted", + ]); }); it("omits the whole managed profile group from Markdown, including inactive sources", () => { const settings = configuredSettings(); - settings.P2P_iceServerSource.id = "manual"; + settings.P2P_managedType = ""; expect(hasManagedTurnSettings(settings)).toBe(true); omitManagedTurnProfilesFromMarkdown(settings); expect(JSON.stringify(settings)).not.toMatch(/private-token|private-key-id|sls\+p2p/); @@ -52,12 +120,12 @@ describe("managed TURN settings privacy", () => { const current = configuredSettings(); const incoming = { ...DEFAULT_SETTINGS }; delete (incoming as Partial).remoteConfigurations; - delete (incoming as Partial).P2P_iceServerSource; + delete (incoming as Partial).P2P_managedType; const merged = { ...DEFAULT_SETTINGS, ...incoming }; preserveManagedTurnProfilesOnMarkdownImport(incoming, current, merged); expect(merged.remoteConfigurations).toEqual(current.remoteConfigurations); expect(merged.remoteConfigurations).not.toBe(current.remoteConfigurations); - expect(merged.P2P_iceServerSource).toEqual(current.P2P_iceServerSource); + expect(merged.P2P_managedToken).toEqual(current.P2P_managedToken); expect(merged.activeConfigurationId).toBe("central"); expect(merged.P2P_ActiveRemoteConfigurationId).toBe("managed"); }); diff --git a/src/features/P2PSync/TurnConfiguration.svelte b/src/features/P2PSync/TurnConfiguration.svelte index c79afcab..80c525eb 100644 --- a/src/features/P2PSync/TurnConfiguration.svelte +++ b/src/features/P2PSync/TurnConfiguration.svelte @@ -1,47 +1,33 @@
- {#if sourceId === "manual"} + {#if managedType === ""}