diff --git a/src/LiveSyncBaseCore.ts b/src/LiveSyncBaseCore.ts index 743214fd..9870a448 100644 --- a/src/LiveSyncBaseCore.ts +++ b/src/LiveSyncBaseCore.ts @@ -38,6 +38,11 @@ export interface LiveSyncCoreFeatureViews { readonly replicationScheduling: ReplicationSchedulingControl; } +export interface StartupDatabaseOptions { + readonly ignoreSuspending?: boolean; + readonly continueOnFileFailure?: boolean; +} + type CompatibilityReplicatorView = ReplicatorInstance & Partial; export class LiveSyncBaseCore< @@ -78,7 +83,8 @@ export class LiveSyncBaseCore< ) => ServiceModules, extraModuleInitialiser: (core: LiveSyncBaseCore) => AbstractModule[], addOnsInitialiser: (core: LiveSyncBaseCore) => TCommands[], - featuresInitialiser: (core: LiveSyncBaseCore, coreFeatureViews: LiveSyncCoreFeatureViews) => void + featuresInitialiser: (core: LiveSyncBaseCore, coreFeatureViews: LiveSyncCoreFeatureViews) => void, + readonly startupDatabaseOptions: StartupDatabaseOptions = {} ) { this._services = serviceHub; this.registerReplicatorProviders(); diff --git a/src/apps/cli/commands/daemonCommand.unit.spec.ts b/src/apps/cli/commands/daemonCommand.unit.spec.ts index b99d2176..10dc251c 100644 --- a/src/apps/cli/commands/daemonCommand.unit.spec.ts +++ b/src/apps/cli/commands/daemonCommand.unit.spec.ts @@ -4,7 +4,7 @@ import { NO_INTERACTION } from "@vrtmrz/livesync-commonlib/replication"; import { runCommand } from "./runCommand"; import type { CLIOptions } from "./types"; -// Mock performFullScan so daemon tests don't require a real CouchDB connection. +// Track explicit scans: database preparation owns the daemon startup scan. vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", () => ({ performFullScan: vi.fn(async () => true), })); @@ -102,6 +102,7 @@ function createDaemonContext(core: ReturnType) { describe("daemon command", () => { beforeEach(() => { vi.restoreAllMocks(); + vi.mocked(offlineScanner.performFullScan).mockClear(); vi.useFakeTimers(); }); @@ -109,27 +110,16 @@ describe("daemon command", () => { vi.useRealTimers(); }); - it("calls performFullScan during startup", async () => { + it("does not repeat the startup scan after initial replication", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); - await runCommand(makeDaemonOptions(), createDaemonContext(core)); + expect(await runCommand(makeDaemonOptions(), createDaemonContext(core))).toBe(true); - expect(offlineScanner.performFullScan).toHaveBeenCalledTimes(1); - }); - - it("returns false when performFullScan fails", async () => { - const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(false); - - const result = await runCommand(makeDaemonOptions(), createDaemonContext(core)); - - expect(result).toBe(false); + expect(offlineScanner.performFullScan).not.toHaveBeenCalled(); }); it("polling mode: calls setTimeout when interval option is set", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); const context = createDaemonContext(core); @@ -143,7 +133,6 @@ describe("daemon command", () => { it("polling mode: applies settings with suspendFileWatching=false before setting interval", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(10), createDaemonContext(core)); @@ -156,7 +145,6 @@ describe("daemon command", () => { it("liveSync mode: calls applyPartial and applySettings", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(), createDaemonContext(core)); @@ -176,7 +164,6 @@ describe("daemon command", () => { liveSync: false, syncOnStart: false, })); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); const result = await runCommand(makeDaemonOptions(), createDaemonContext(core)); @@ -194,7 +181,6 @@ describe("daemon command", () => { liveSync: true, syncOnStart: false, })); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(), createDaemonContext(core)); @@ -205,22 +191,21 @@ describe("daemon command", () => { expect(warningCalls.length).toBe(0); }); - it("calls replicate before performFullScan", async () => { + it("completes initial replication before restoring automatic synchronisation", async () => { const core = createCoreMock(); const callOrder: string[] = []; core.services.replication.replicateUnattended = vi.fn(async () => { callOrder.push("replicate"); return { status: "completed" as const }; }); - vi.mocked(offlineScanner.performFullScan).mockImplementation(async () => { - callOrder.push("performFullScan"); - return true; + core.services.control.applySettings.mockImplementation(async () => { + callOrder.push("restoreSettings"); }); const context = createDaemonContext(core); await runCommand(makeDaemonOptions(), context); - expect(callOrder).toEqual(["replicate", "performFullScan"]); + expect(callOrder).toEqual(["replicate", "restoreSettings"]); expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({ trigger: "daemon", interaction: NO_INTERACTION, @@ -234,12 +219,11 @@ describe("daemon command", () => { status: "failed" as const, error: new Error("initial replication failed"), })); - vi.mocked(offlineScanner.performFullScan).mockClear(); const result = await runCommand(makeDaemonOptions(), createDaemonContext(core)); expect(result).toBe(false); - // performFullScan should NOT have been called + expect(core.services.control.applySettings).not.toHaveBeenCalled(); expect(offlineScanner.performFullScan).not.toHaveBeenCalled(); expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({ trigger: "daemon", @@ -249,7 +233,6 @@ describe("daemon command", () => { it("polling mode: registers onUnload handler that clears timeout", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); await runCommand(makeDaemonOptions(10), createDaemonContext(core)); @@ -265,7 +248,6 @@ describe("daemon command", () => { it("polling backoff: interval escalates on failure, caps at 300000ms, then halves on recovery", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); // startup replicate (call 1) succeeds; poll calls 2–7 fail; call 8 succeeds. let callCount = 0; @@ -320,7 +302,6 @@ describe("daemon command", () => { it("polling error handling: replicate rejection is caught and written to standard error", async () => { const core = createCoreMock(); - vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true); // Make replicate succeed on the initial call (startup), then fail on the poll. let callCount = 0; diff --git a/src/apps/cli/commands/runCommand.ts b/src/apps/cli/commands/runCommand.ts index c2328c7e..8f27b870 100644 --- a/src/apps/cli/commands/runCommand.ts +++ b/src/apps/cli/commands/runCommand.ts @@ -15,11 +15,6 @@ import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_b import type { CLICommandContext, CLIOptions } from "./types"; import { toArrayBuffer, toDatabaseRelativePath } from "./utils"; import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p"; -import { - performFullScan, - VaultScanResults, -} from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; -import { UnresolvedErrorManager } from "@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node"; import { writeStderrLine, writeStdoutLine } from "@/apps/cli/cliOutput"; @@ -59,9 +54,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext // accept whatever configuration the remote has. await core.services.setting.applyPartial({ disableCheckingConfigMismatch: true }, true); - // 1. Replicate the configured remote into the local database so the - // mirror scan has content to work with. - log("Replicating from remote..."); + // Database preparation has already reconciled the local database and Vault. + // Replicate before restoring automatic synchronisation. + log("Replicating with remote..."); const replResult = await core.services.replication.replicateUnattended({ trigger: "daemon", interaction: NO_INTERACTION, @@ -73,17 +68,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext replicationScheduling.markInitialOneShotSatisfied(); log("Initial replication complete"); - // 2. Mirror scan to reconcile PouchDB ↔ local filesystem. - const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events); - log("Running mirror scan..."); - const scanOk = await performFullScan(core, log, errorManager, false, true); - if (!scanOk) { - writeStderrLine(standardIo, "[Daemon] Mirror scan failed, cannot continue"); - return false; - } - log("Mirror scan complete"); - - // 3. Re-enable sync. + // Re-enable sync. const restoreSyncSettings = async () => { await core.services.setting.applyPartial( { @@ -530,9 +515,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext if (options.command === "mirror") { writeStderrLine(standardIo, "[Command] mirror"); - const log = (msg: unknown) => writeStderrLine(standardIo, `[Mirror] ${String(msg)}`); - const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events); - return (await performFullScan(core, log, errorManager, false, true)) === VaultScanResults.COMPLETED; + // Database preparation has already completed the mirror scan. + return true; } if (options.command === "remote-add") { diff --git a/src/apps/cli/main.bootstrap.unit.spec.ts b/src/apps/cli/main.bootstrap.unit.spec.ts new file mode 100644 index 00000000..91679968 --- /dev/null +++ b/src/apps/cli/main.bootstrap.unit.spec.ts @@ -0,0 +1,195 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as chokidar from "chokidar"; +import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; +import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { ServiceFileHandler } from "@/serviceModules/FileHandler"; +import { ServiceFileAccessCLI } from "./serviceModules/ServiceFileAccessImpl"; +import { runCommand } from "./commands/runCommand"; +import { createDefaultCliSettings } from "./cliSettingsDefaults"; +import { main, type CliCommandRunner } from "./main"; + +vi.mock("chokidar", { spy: true }); + +function createStandardIoMock() { + return { + readStdin: vi.fn(async () => ""), + prompt: vi.fn(async () => ""), + writeStdout: vi.fn(), + writeStderr: vi.fn(), + }; +} + +describe("CLI database preparation", () => { + const originalArgv = process.argv.slice(); + const originalExitCode = process.exitCode; + let directory: string; + let vaultPath: string; + let settingsPath: string; + let signalHandlers: Map<"SIGINT" | "SIGTERM", Set>; + let standardIo: ReturnType; + + beforeEach(async () => { + vi.mocked(chokidar.watch).mockClear(); + directory = await mkdtemp(join(tmpdir(), "livesync-cli-bootstrap-")); + vaultPath = join(directory, "vault"); + settingsPath = join(directory, "settings.json"); + await mkdir(join(vaultPath, "notes"), { recursive: true }); + await writeFile(join(vaultPath, "notes/local.md"), "local content"); + await writeFile(settingsPath, JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true })); + standardIo = createStandardIoMock(); + signalHandlers = new Map( + (["SIGINT", "SIGTERM"] as const).map((signal) => [signal, new Set(process.listeners(signal))]) + ); + process.exitCode = undefined; + vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`__EXIT__:${code ?? 0}`); + }); + }); + + afterEach(async () => { + for (const [signal, originalHandlers] of signalHandlers) { + for (const handler of process.listeners(signal)) { + if (!originalHandlers.has(handler)) process.removeListener(signal, handler); + } + } + process.argv = originalArgv.slice(); + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + await rm(directory, { recursive: true, force: true }); + }); + + async function start(command: "daemon" | "mirror" | "ls", runner: CliCommandRunner, exitCode = 1) { + process.argv = ["node", "livesync-cli", directory, "--vault", vaultPath, "--settings", settingsPath, command]; + // Daemon probes return false so the real core unloads without keeping a daemon alive. + await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`); + } + + it.each([ + { command: "daemon" as const, suspendFileWatching: false }, + { command: "mirror" as const, suspendFileWatching: false }, + { command: "mirror" as const, suspendFileWatching: true }, + ])( + "prepares the Vault before $command (watching suspended: $suspendFileWatching)", + async ({ command, suspendFileWatching }) => { + await writeFile( + settingsPath, + JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching }) + ); + await mkdir(join(vaultPath, ".livesync")); + await writeFile(join(vaultPath, ".livesync/ignore"), "*.tmp\n"); + await writeFile(join(vaultPath, "notes/ignored.tmp"), "ignored"); + const storedPaths: string[] = []; + let content: string | undefined; + const runner = vi.fn(async (_options, { core }) => { + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + const file = await core.serviceModules.databaseFileAccess.fetch("notes/local.md" as FilePathWithPrefix); + content = file ? await file.body.text() : undefined; + return false; + }); + + await start(command, runner); + + expect(runner).toHaveBeenCalledOnce(); + expect(storedPaths).toEqual(["notes/local.md"]); + expect(content).toBe("local content"); + expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content"); + } + ); + + it("runs the mirror scan once and exits without starting file watching", async () => { + const enumerate = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles"); + const watch = vi.mocked(chokidar.watch); + const runner = vi.fn(runCommand); + + await start("mirror", runner, 0); + + expect(runner).toHaveBeenCalledOnce(); + expect(enumerate).toHaveBeenCalledOnce(); + expect(watch).not.toHaveBeenCalled(); + }); + + it.each([ + { command: "daemon" as const, commandRuns: true }, + { command: "mirror" as const, commandRuns: false }, + ])("handles an individual file failure during $command preparation", async ({ command, commandRuns }) => { + const store = vi + .spyOn(ServiceFileHandler.prototype, "storeFileToDB") + .mockRejectedValue(new Error("file failed")); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + + await start(command, runner); + + expect(store).toHaveBeenCalledOnce(); + expect(runner).toHaveBeenCalledTimes(commandRuns ? 1 : 0); + expect(unload).toHaveBeenCalledOnce(); + expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content"); + }); + + it("does not import vault files for standalone database commands", async () => { + const storedPaths: string[] = []; + const runner = vi.fn(async (_options, { core }) => { + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + return false; + }); + + await start("ls", runner); + + expect(runner).toHaveBeenCalledOnce(); + expect(storedPaths).toEqual([]); + }); + + it("unloads without starting the command when database preparation fails", async () => { + vi.spyOn(ControlService.prototype, "onReady").mockResolvedValue(false); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + const settingsBefore = await readFile(settingsPath, "utf-8"); + + await start("daemon", runner); + + expect(runner).not.toHaveBeenCalled(); + expect(unload).toHaveBeenCalledOnce(); + expect(unload.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(process.exit).mock.invocationCallOrder[0]); + expect(process.exit).toHaveBeenCalledWith(1); + expect(await readFile(settingsPath, "utf-8")).toBe(settingsBefore); + }); + + it("stops the daemon when the startup scanner refuses a suspended Vault scan", async () => { + await writeFile( + settingsPath, + JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching: true }) + ); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + + await start("daemon", runner); + + expect(runner).not.toHaveBeenCalled(); + expect(unload).toHaveBeenCalledOnce(); + expect(process.exit).toHaveBeenCalledWith(1); + }); + + it("unloads when database preparation throws", async () => { + const ready = vi.spyOn(ControlService.prototype, "onReady").mockRejectedValue(new Error("scan failed")); + const unload = vi.spyOn(ControlService.prototype, "onUnload"); + const runner = vi.fn(async () => false); + + try { + await start("daemon", runner); + + expect(runner).not.toHaveBeenCalled(); + expect(unload).toHaveBeenCalledOnce(); + expect(standardIo.writeStderr.mock.calls.flat().join("")).toContain("scan failed"); + } finally { + const control = ready.mock.contexts[0]; + if (unload.mock.calls.length === 0 && control instanceof ControlService) await control.onUnload(); + } + }); +}); diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 895b5a16..1b2bff53 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -1,6 +1,6 @@ import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub"; import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage"; -import { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; +import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore"; import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules"; import { LOG_LEVEL_VERBOSE, @@ -24,6 +24,7 @@ import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/ import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { IgnoreRules } from "./serviceModules/IgnoreRules"; import { useP2PReplicatorFeature, type UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/p2p"; +import { useOfflineScanner } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; import type { ReplicationSchedulingControl } from "@/serviceFeatures/replicationScheduling"; import { createNodeStandardIo, fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node"; import type { StandardIo } from "@vrtmrz/livesync-commonlib/context"; @@ -41,6 +42,27 @@ import { } from "./settingsPersistence"; const SETTINGS_FILE = ".livesync/settings.json"; + +interface CLIVaultSyncMode { + readonly watchFiles: boolean; + readonly reflectReplicationResults: boolean; + readonly startupDatabaseOptions: StartupDatabaseOptions; +} + +// Commands which synchronise a physical Vault with the local database. +const VAULT_SYNC_MODES: Readonly>> = { + daemon: { + watchFiles: true, + reflectReplicationResults: true, + startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: true }, + }, + mirror: { + watchFiles: false, + reflectReplicationResults: false, + startupDatabaseOptions: { ignoreSuspending: true, continueOnFileFailure: false }, + }, +}; + ensureGlobalNodeLocalStorage(); defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG; @@ -296,6 +318,7 @@ export async function main( commandRunner: CliCommandRunner = runCommand ) { const options = parseArgs(standardIo); + const vaultSyncMode = VAULT_SYNC_MODES[options.command]; if (options.interval && options.command !== "daemon") { writeStderrLine( standardIo, @@ -357,9 +380,6 @@ export async function main( // Resolve vault path: mirror positional argument takes priority, // then --vault flag, otherwise fall back to databasePath. - // For daemon mode, enable chokidar file watching so the _changes feed picks up events. - // mirror runs a single full scan and doesn't need continuous watching. - const watchEnabled = options.command === "daemon"; const vaultPath = options.command === "mirror" && options.commandArgs[0] ? path.resolve(options.commandArgs[0]) @@ -385,7 +405,7 @@ export async function main( infoLog(`Settings: ${settingsPath}`); infoLog(""); let ignoreRules: IgnoreRules | undefined; - if (options.command === "daemon" || options.command === "mirror") { + if (vaultSyncMode) { ignoreRules = new IgnoreRules(vaultPath, (message, detail) => { if (detail === undefined) { writeStderrLine(standardIo, message); @@ -426,9 +446,8 @@ export async function main( } writeStderrLine(standardIo, prefix, message); }, true); - // Prevent replication result from being processed automatically in non-daemon commands. - // In daemon mode the default handler must run so changes are applied to the filesystem. - if (options.command !== "daemon") { + // Only modes which reflect replication results use the default filesystem handler. + if (!vaultSyncMode?.reflectReplicationResults) { serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => { writeStderrLine( standardIo, @@ -489,12 +508,21 @@ export async function main( const core = new LiveSyncBaseCore( serviceHubInstance, (core: LiveSyncBaseCore, serviceHub: InjectableServiceHub) => { - return initialiseServiceModulesCLI(vaultPath, core, serviceHub, ignoreRules, watchEnabled); + return initialiseServiceModulesCLI( + vaultPath, + core, + serviceHub, + ignoreRules, + vaultSyncMode?.watchFiles ?? false + ); }, (core) => [], () => [], // No add-ons (core, coreFeatureViews) => { replicationScheduling = coreFeatureViews.replicationScheduling; + if (vaultSyncMode) { + useOfflineScanner(core); + } // Register P2P replicator feature. p2pReplicator = useP2PReplicatorFeature(core); // Add target filter to prevent internal files are handled @@ -512,7 +540,7 @@ export async function main( return await Promise.resolve(true); }, -1 /* highest priority */); - // Apply user-defined ignore rules for daemon mode (lower priority, runs after dotfile check). + // Apply user-defined ignore rules after the dotfile check. if (ignoreRules) { const rules = ignoreRules; core.services.vault.isTargetFile.addHandler(async (target) => { @@ -524,7 +552,8 @@ export async function main( return true; }, 0); } - } + }, + vaultSyncMode?.startupDatabaseOptions ); if (!replicationScheduling) { throw new Error("Replication scheduling was not provided during core feature composition."); @@ -577,7 +606,7 @@ export async function main( : originalSettingsText; // Capture sync settings before suspendAllSync() clobbers them. - // Used by daemon mode to restore the correct sync behaviour after the mirror scan. + // Used by daemon mode to restore sync behaviour after initial replication. const settingsBeforeSuspend = cloneSettings(core.services.setting.currentSettings()); const durableSettingsBeforeSuspend = cloneSettings(settingsBeforeSuspend); applyStoredSetting(durableSettingsBeforeSuspend, settingsAfterLoadText, "useIndexedDBAdapter"); @@ -592,7 +621,15 @@ export async function main( }; await core.services.setting.suspendAllSync(); const settingsAfterSuspend = cloneSettings(core.services.setting.currentSettings()); - await core.services.control.onReady(); + let readyResult = false; + try { + readyResult = await core.services.control.onReady(); + } finally { + if (!readyResult) await core.services.control.onUnload(); + } + if (!readyResult) { + throw new Error("Failed to initialise LiveSync."); + } const settingsBeforeCommand = cloneSettings(core.services.setting.currentSettings()); const transientSettingKeys = changedSettingKeys(settingsBeforeSuspend, settingsAfterSuspend); for (const key of CLI_RUNTIME_ONLY_SETTING_KEYS) { diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 557e591a..e82ae069 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -12,7 +12,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json index b68a120a..30262fa6 100644 --- a/src/apps/cli/testdeno/deno.json +++ b/src/apps/cli/testdeno/deno.json @@ -5,6 +5,7 @@ "test:p2p:compose": "deno run -A --no-check run-compose-p2p.ts", "test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts", "test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts", + "test:daemon-startup": "deno test --env-file=.test.env -A --no-check test-daemon-startup.ts", "test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts", "test:remote-commands": "deno test --env-file=.test.env -A --no-check test-remote-commands.ts", "test:settings-writeback": "deno test -A --no-check test-settings-writeback.ts", diff --git a/src/apps/cli/testdeno/run-ci-suite.ts b/src/apps/cli/testdeno/run-ci-suite.ts index 9b5b0c60..7c391ce9 100644 --- a/src/apps/cli/testdeno/run-ci-suite.ts +++ b/src/apps/cli/testdeno/run-ci-suite.ts @@ -4,6 +4,7 @@ const TASKS = [ "test:setup-put-cat", "test:mirror", "test:daemon", + "test:daemon-startup", "test:push-pull", "test:decoupled-vault", "test:sync-two-local", diff --git a/src/apps/cli/testdeno/test-daemon-startup.ts b/src/apps/cli/testdeno/test-daemon-startup.ts new file mode 100644 index 00000000..8810faa1 --- /dev/null +++ b/src/apps/cli/testdeno/test-daemon-startup.ts @@ -0,0 +1,152 @@ +import { assertEquals } from "@std/assert"; +import { join } from "@std/path"; +import { TempDir } from "./helpers/temp.ts"; +import { runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts"; +import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; +import { startCliInBackground, type BackgroundCliProcess } from "./helpers/backgroundCli.ts"; +import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; + +function envOrDefault(keys: string[], fallback: string): string { + for (const key of keys) { + const value = Deno.env.get(key)?.trim(); + if (value) return value; + } + return fallback; +} + +function waitForTick(): Promise { + return new Promise((resolve) => setTimeout(resolve, 100)); +} + +async function waitForText(filePath: string, expected: string, timeoutMs = 45_000): Promise { + const deadline = Date.now() + timeoutMs; + let actual = ""; + while (Date.now() < deadline) { + try { + actual = await Deno.readTextFile(filePath); + if (actual === expected) return; + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error; + } + await waitForTick(); + } + throw new Error( + `Timed out waiting for ${filePath} to contain ${JSON.stringify(expected)}; actual=${JSON.stringify(actual)}` + ); +} + +async function waitForMissing(filePath: string, timeoutMs = 45_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + await Deno.stat(filePath); + } catch (error) { + if (error instanceof Deno.errors.NotFound) return; + throw error; + } + await waitForTick(); + } + throw new Error(`Timed out waiting for ${filePath} to be removed`); +} + +async function stopDaemon(daemon: BackgroundCliProcess | undefined): Promise { + if (!daemon) return; + await daemon.stop().catch(() => {}); +} + +Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files", async () => { + await using workDir = await TempDir.create("livesync-cli-daemon-startup"); + + const couchdbUri = envOrDefault(["COUCHDB_URI", "hostname"], "http://127.0.0.1:5989").replace(/\/$/, ""); + const couchdbUser = envOrDefault(["COUCHDB_USER", "username"], "admin"); + const couchdbPassword = envOrDefault(["COUCHDB_PASSWORD", "password"], "testpassword"); + const dbPrefix = envOrDefault(["COUCHDB_DBNAME", "dbname"], "livesync-test-db-ci"); + const dbname = `${dbPrefix}-daemon-startup-${Date.now()}-${Math.floor(Math.random() * 1_000_000)}`.toLowerCase(); + + const databaseA = workDir.join("database-a"); + const databaseB = workDir.join("database-b"); + const databaseC = workDir.join("database-c"); + const vaultA = workDir.join("vault-a"); + const vaultB = workDir.join("vault-b"); + const vaultC = workDir.join("vault-c"); + const settingsA = workDir.join("settings-a.json"); + const settingsB = workDir.join("settings-b.json"); + const settingsC = workDir.join("settings-c.json"); + + await Promise.all([ + Deno.mkdir(databaseA, { recursive: true }), + Deno.mkdir(databaseB, { recursive: true }), + Deno.mkdir(databaseC, { recursive: true }), + Deno.mkdir(vaultA, { recursive: true }), + Deno.mkdir(vaultB, { recursive: true }), + Deno.mkdir(vaultC, { recursive: true }), + ]); + + const startupPath = "notes/present-before-start.md"; + const deletePath = "notes/deleted-while-stopped.md"; + const remoteOnlyPath = "notes/remote-only.md"; + const startupFileA = join(vaultA, startupPath); + const deleteFileA = join(vaultA, deletePath); + const startupFileB = join(vaultB, startupPath); + const deleteFileB = join(vaultB, deletePath); + const remoteOnlyFileB = join(vaultB, remoteOnlyPath); + + await Deno.mkdir(join(vaultA, "notes"), { recursive: true }); + await Deno.writeTextFile(startupFileA, "created before daemon startup\n"); + const initialTime = new Date(Date.now() - 10_000); + await Deno.utime(startupFileA, initialTime, initialTime); + await Deno.writeTextFile(deleteFileA, "delete this after the first run\n"); + + let daemonA: BackgroundCliProcess | undefined; + let daemonB: BackgroundCliProcess | undefined; + try { + await startCouchdb(couchdbUri, couchdbUser, couchdbPassword, dbname); + for (const settings of [settingsA, settingsB, settingsC]) { + await initSettingsFile(settings); + await applyCouchdbSettings(settings, couchdbUri, couchdbUser, couchdbPassword, dbname, true); + } + + // A pre-existing local file must be uploaded by the daemon's startup scan. + daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon"); + await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000); + + // A separate daemon proves that the first startup replication reached CouchDB + // and that remote files are reflected into its filesystem. + daemonB = startCliInBackground(databaseB, "--vault", vaultB, "--settings", settingsB, "daemon"); + await daemonB.waitUntilContains("[Daemon] Initial replication complete", 45_000); + await waitForText(startupFileB, "created before daemon startup\n"); + await waitForText(deleteFileB, "delete this after the first run\n"); + + // Changes made while A is stopped must be found by its next startup scan. + assertEquals(await daemonA.stop(), 0, daemonA.combined); + daemonA = undefined; + await Deno.writeTextFile(startupFileA, "edited while daemon was stopped\n"); + await Deno.remove(deleteFileA); + + daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon"); + await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000); + await waitForText(startupFileB, "edited while daemon was stopped\n"); + await waitForMissing(deleteFileB); + + // Seed a file into a third local database without creating it in vault C. + // After C's finite sync, it exists only remotely from B's point of view. + await runCliWithInputOrFail( + "created in a different local database\n", + databaseC, + "--vault", + vaultC, + "--settings", + settingsC, + "put", + remoteOnlyPath + ); + await runCliOrFail(databaseC, "--vault", vaultC, "--settings", settingsC, "sync"); + await waitForText(remoteOnlyFileB, "created in a different local database\n"); + assertEquals(await Deno.readTextFile(startupFileB), "edited while daemon was stopped\n"); + assertEquals((await Deno.stat(remoteOnlyFileB)).isFile, true); + } finally { + await stopDaemon(daemonB); + await stopDaemon(daemonA); + await stopCouchdb().catch(() => {}); + } +}); diff --git a/src/modules/main/ModuleLiveSyncMain.ts b/src/modules/main/ModuleLiveSyncMain.ts index 613dd119..7cdc656a 100644 --- a/src/modules/main/ModuleLiveSyncMain.ts +++ b/src/modules/main/ModuleLiveSyncMain.ts @@ -45,7 +45,13 @@ export class ModuleLiveSyncMain extends AbstractModule { } // Ordinary start-up may continue when individual files could not be // processed. Explicit Fetch and Rebuild flows retain the strict default. - const initialisationResult = await this.services.databaseEvents.initialiseDatabase(false, false, false, true); + const { ignoreSuspending = false, continueOnFileFailure = true } = this.core.startupDatabaseOptions; + const initialisationResult = await this.services.databaseEvents.initialiseDatabase( + false, + false, + ignoreSuspending, + continueOnFileFailure + ); if (initialisationResult === VaultScanResults.FAILED) { this._log($msg("Ui.Common.LocalDatabaseInitialisationFailed"), LOG_LEVEL_NOTICE); //TODO:stop all sync. diff --git a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts index d21b8e8d..1fc7a139 100644 --- a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts +++ b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts @@ -25,6 +25,7 @@ describe("ModuleLiveSyncMain", () => { const log = vi.fn(); const host = { core: { + startupDatabaseOptions: {}, services: { appLifecycle: { onLayoutReady: vi.fn(async () => true), @@ -58,6 +59,7 @@ describe("ModuleLiveSyncMain", () => { }; const host = { core: { + startupDatabaseOptions: {}, services: { appLifecycle }, }, services: { @@ -77,4 +79,33 @@ describe("ModuleLiveSyncMain", () => { expect(result).toBe(true); expect(log).toHaveBeenCalledWith("Ui.Common.SomeFilesCouldNotBeSynchronised", LOG_LEVEL_NOTICE); }); + + it("passes strict startup database options to initialisation", async () => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: { + ignoreSuspending: true, + continueOnFileFailure: false, + }, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + databaseEvents: { initialiseDatabase }, + }, + settings: { + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false); + }); }); diff --git a/updates.md b/updates.md index a59de8a8..0cd2e254 100644 --- a/updates.md +++ b/updates.md @@ -14,6 +14,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Fixed +- CLI: daemon and mirror now scan the Vault during database initialisation, following the Obsidian startup sequence. The daemon completes this scan before replication; mirror runs the scan once and still exits with an error if any file cannot be processed. - CLI: file enumeration now includes current files even after individual path lookups or earlier scans. ## 1.0.28