mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-09-16 07:37:06 +00:00
fix(cli): prepare daemon and mirror Vaults during startup
This commit is contained in:
@@ -38,6 +38,11 @@ export interface LiveSyncCoreFeatureViews {
|
||||
readonly replicationScheduling: ReplicationSchedulingControl;
|
||||
}
|
||||
|
||||
export interface StartupDatabaseOptions {
|
||||
readonly ignoreSuspending?: boolean;
|
||||
readonly continueOnFileFailure?: boolean;
|
||||
}
|
||||
|
||||
type CompatibilityReplicatorView = ReplicatorInstance & Partial<LiveSyncAbstractReplicator>;
|
||||
|
||||
export class LiveSyncBaseCore<
|
||||
@@ -78,7 +83,8 @@ export class LiveSyncBaseCore<
|
||||
) => ServiceModules,
|
||||
extraModuleInitialiser: (core: LiveSyncBaseCore<T, TCommands>) => AbstractModule[],
|
||||
addOnsInitialiser: (core: LiveSyncBaseCore<T, TCommands>) => TCommands[],
|
||||
featuresInitialiser: (core: LiveSyncBaseCore<T, TCommands>, coreFeatureViews: LiveSyncCoreFeatureViews) => void
|
||||
featuresInitialiser: (core: LiveSyncBaseCore<T, TCommands>, coreFeatureViews: LiveSyncCoreFeatureViews) => void,
|
||||
readonly startupDatabaseOptions: StartupDatabaseOptions = {}
|
||||
) {
|
||||
this._services = serviceHub;
|
||||
this.registerReplicatorProviders();
|
||||
|
||||
@@ -4,7 +4,7 @@ import { NO_INTERACTION } from "@vrtmrz/livesync-commonlib/replication";
|
||||
import { runCommand } from "./runCommand";
|
||||
import type { CLIOptions } from "./types";
|
||||
|
||||
// Mock performFullScan so daemon tests don't require a real CouchDB connection.
|
||||
// Track explicit scans: database preparation owns the daemon startup scan.
|
||||
vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", () => ({
|
||||
performFullScan: vi.fn(async () => true),
|
||||
}));
|
||||
@@ -102,6 +102,7 @@ function createDaemonContext(core: ReturnType<typeof createCoreMock>) {
|
||||
describe("daemon command", () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.mocked(offlineScanner.performFullScan).mockClear();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
@@ -109,27 +110,16 @@ describe("daemon command", () => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("calls performFullScan during startup", async () => {
|
||||
it("does not repeat the startup scan after initial replication", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
await runCommand(makeDaemonOptions(), createDaemonContext(core));
|
||||
expect(await runCommand(makeDaemonOptions(), createDaemonContext(core))).toBe(true);
|
||||
|
||||
expect(offlineScanner.performFullScan).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("returns false when performFullScan fails", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(false);
|
||||
|
||||
const result = await runCommand(makeDaemonOptions(), createDaemonContext(core));
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(offlineScanner.performFullScan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("polling mode: calls setTimeout when interval option is set", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
|
||||
const context = createDaemonContext(core);
|
||||
@@ -143,7 +133,6 @@ describe("daemon command", () => {
|
||||
|
||||
it("polling mode: applies settings with suspendFileWatching=false before setting interval", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
await runCommand(makeDaemonOptions(10), createDaemonContext(core));
|
||||
|
||||
@@ -156,7 +145,6 @@ describe("daemon command", () => {
|
||||
|
||||
it("liveSync mode: calls applyPartial and applySettings", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
await runCommand(makeDaemonOptions(), createDaemonContext(core));
|
||||
|
||||
@@ -176,7 +164,6 @@ describe("daemon command", () => {
|
||||
liveSync: false,
|
||||
syncOnStart: false,
|
||||
}));
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
const result = await runCommand(makeDaemonOptions(), createDaemonContext(core));
|
||||
|
||||
@@ -194,7 +181,6 @@ describe("daemon command", () => {
|
||||
liveSync: true,
|
||||
syncOnStart: false,
|
||||
}));
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
await runCommand(makeDaemonOptions(), createDaemonContext(core));
|
||||
|
||||
@@ -205,22 +191,21 @@ describe("daemon command", () => {
|
||||
expect(warningCalls.length).toBe(0);
|
||||
});
|
||||
|
||||
it("calls replicate before performFullScan", async () => {
|
||||
it("completes initial replication before restoring automatic synchronisation", async () => {
|
||||
const core = createCoreMock();
|
||||
const callOrder: string[] = [];
|
||||
core.services.replication.replicateUnattended = vi.fn(async () => {
|
||||
callOrder.push("replicate");
|
||||
return { status: "completed" as const };
|
||||
});
|
||||
vi.mocked(offlineScanner.performFullScan).mockImplementation(async () => {
|
||||
callOrder.push("performFullScan");
|
||||
return true;
|
||||
core.services.control.applySettings.mockImplementation(async () => {
|
||||
callOrder.push("restoreSettings");
|
||||
});
|
||||
|
||||
const context = createDaemonContext(core);
|
||||
await runCommand(makeDaemonOptions(), context);
|
||||
|
||||
expect(callOrder).toEqual(["replicate", "performFullScan"]);
|
||||
expect(callOrder).toEqual(["replicate", "restoreSettings"]);
|
||||
expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({
|
||||
trigger: "daemon",
|
||||
interaction: NO_INTERACTION,
|
||||
@@ -234,12 +219,11 @@ describe("daemon command", () => {
|
||||
status: "failed" as const,
|
||||
error: new Error("initial replication failed"),
|
||||
}));
|
||||
vi.mocked(offlineScanner.performFullScan).mockClear();
|
||||
|
||||
const result = await runCommand(makeDaemonOptions(), createDaemonContext(core));
|
||||
|
||||
expect(result).toBe(false);
|
||||
// performFullScan should NOT have been called
|
||||
expect(core.services.control.applySettings).not.toHaveBeenCalled();
|
||||
expect(offlineScanner.performFullScan).not.toHaveBeenCalled();
|
||||
expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({
|
||||
trigger: "daemon",
|
||||
@@ -249,7 +233,6 @@ describe("daemon command", () => {
|
||||
|
||||
it("polling mode: registers onUnload handler that clears timeout", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
await runCommand(makeDaemonOptions(10), createDaemonContext(core));
|
||||
|
||||
@@ -265,7 +248,6 @@ describe("daemon command", () => {
|
||||
|
||||
it("polling backoff: interval escalates on failure, caps at 300000ms, then halves on recovery", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
// startup replicate (call 1) succeeds; poll calls 2–7 fail; call 8 succeeds.
|
||||
let callCount = 0;
|
||||
@@ -320,7 +302,6 @@ describe("daemon command", () => {
|
||||
|
||||
it("polling error handling: replicate rejection is caught and written to standard error", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
|
||||
// Make replicate succeed on the initial call (startup), then fail on the poll.
|
||||
let callCount = 0;
|
||||
|
||||
@@ -15,11 +15,6 @@ import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_b
|
||||
import type { CLICommandContext, CLIOptions } from "./types";
|
||||
import { toArrayBuffer, toDatabaseRelativePath } from "./utils";
|
||||
import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p";
|
||||
import {
|
||||
performFullScan,
|
||||
VaultScanResults,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
|
||||
import { UnresolvedErrorManager } from "@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager";
|
||||
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { writeStderrLine, writeStdoutLine } from "@/apps/cli/cliOutput";
|
||||
@@ -59,9 +54,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
// accept whatever configuration the remote has.
|
||||
await core.services.setting.applyPartial({ disableCheckingConfigMismatch: true }, true);
|
||||
|
||||
// 1. Replicate the configured remote into the local database so the
|
||||
// mirror scan has content to work with.
|
||||
log("Replicating from remote...");
|
||||
// Database preparation has already reconciled the local database and Vault.
|
||||
// Replicate before restoring automatic synchronisation.
|
||||
log("Replicating with remote...");
|
||||
const replResult = await core.services.replication.replicateUnattended({
|
||||
trigger: "daemon",
|
||||
interaction: NO_INTERACTION,
|
||||
@@ -73,17 +68,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
replicationScheduling.markInitialOneShotSatisfied();
|
||||
log("Initial replication complete");
|
||||
|
||||
// 2. Mirror scan to reconcile PouchDB ↔ local filesystem.
|
||||
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
|
||||
log("Running mirror scan...");
|
||||
const scanOk = await performFullScan(core, log, errorManager, false, true);
|
||||
if (!scanOk) {
|
||||
writeStderrLine(standardIo, "[Daemon] Mirror scan failed, cannot continue");
|
||||
return false;
|
||||
}
|
||||
log("Mirror scan complete");
|
||||
|
||||
// 3. Re-enable sync.
|
||||
// Re-enable sync.
|
||||
const restoreSyncSettings = async () => {
|
||||
await core.services.setting.applyPartial(
|
||||
{
|
||||
@@ -530,9 +515,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
|
||||
if (options.command === "mirror") {
|
||||
writeStderrLine(standardIo, "[Command] mirror");
|
||||
const log = (msg: unknown) => writeStderrLine(standardIo, `[Mirror] ${String(msg)}`);
|
||||
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
|
||||
return (await performFullScan(core, log, errorManager, false, true)) === VaultScanResults.COMPLETED;
|
||||
// Database preparation has already completed the mirror scan.
|
||||
return true;
|
||||
}
|
||||
|
||||
if (options.command === "remote-add") {
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import * as chokidar from "chokidar";
|
||||
import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
|
||||
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { ServiceFileHandler } from "@/serviceModules/FileHandler";
|
||||
import { ServiceFileAccessCLI } from "./serviceModules/ServiceFileAccessImpl";
|
||||
import { runCommand } from "./commands/runCommand";
|
||||
import { createDefaultCliSettings } from "./cliSettingsDefaults";
|
||||
import { main, type CliCommandRunner } from "./main";
|
||||
|
||||
vi.mock("chokidar", { spy: true });
|
||||
|
||||
function createStandardIoMock() {
|
||||
return {
|
||||
readStdin: vi.fn(async () => ""),
|
||||
prompt: vi.fn(async () => ""),
|
||||
writeStdout: vi.fn(),
|
||||
writeStderr: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("CLI database preparation", () => {
|
||||
const originalArgv = process.argv.slice();
|
||||
const originalExitCode = process.exitCode;
|
||||
let directory: string;
|
||||
let vaultPath: string;
|
||||
let settingsPath: string;
|
||||
let signalHandlers: Map<"SIGINT" | "SIGTERM", Set<NodeJS.SignalsListener>>;
|
||||
let standardIo: ReturnType<typeof createStandardIoMock>;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.mocked(chokidar.watch).mockClear();
|
||||
directory = await mkdtemp(join(tmpdir(), "livesync-cli-bootstrap-"));
|
||||
vaultPath = join(directory, "vault");
|
||||
settingsPath = join(directory, "settings.json");
|
||||
await mkdir(join(vaultPath, "notes"), { recursive: true });
|
||||
await writeFile(join(vaultPath, "notes/local.md"), "local content");
|
||||
await writeFile(settingsPath, JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true }));
|
||||
standardIo = createStandardIoMock();
|
||||
signalHandlers = new Map(
|
||||
(["SIGINT", "SIGTERM"] as const).map((signal) => [signal, new Set(process.listeners(signal))])
|
||||
);
|
||||
process.exitCode = undefined;
|
||||
vi.spyOn(process, "exit").mockImplementation((code) => {
|
||||
throw new Error(`__EXIT__:${code ?? 0}`);
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const [signal, originalHandlers] of signalHandlers) {
|
||||
for (const handler of process.listeners(signal)) {
|
||||
if (!originalHandlers.has(handler)) process.removeListener(signal, handler);
|
||||
}
|
||||
}
|
||||
process.argv = originalArgv.slice();
|
||||
process.exitCode = originalExitCode;
|
||||
vi.restoreAllMocks();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function start(command: "daemon" | "mirror" | "ls", runner: CliCommandRunner, exitCode = 1) {
|
||||
process.argv = ["node", "livesync-cli", directory, "--vault", vaultPath, "--settings", settingsPath, command];
|
||||
// Daemon probes return false so the real core unloads without keeping a daemon alive.
|
||||
await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`);
|
||||
}
|
||||
|
||||
it.each([
|
||||
{ command: "daemon" as const, suspendFileWatching: false },
|
||||
{ command: "mirror" as const, suspendFileWatching: false },
|
||||
{ command: "mirror" as const, suspendFileWatching: true },
|
||||
])(
|
||||
"prepares the Vault before $command (watching suspended: $suspendFileWatching)",
|
||||
async ({ command, suspendFileWatching }) => {
|
||||
await writeFile(
|
||||
settingsPath,
|
||||
JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching })
|
||||
);
|
||||
await mkdir(join(vaultPath, ".livesync"));
|
||||
await writeFile(join(vaultPath, ".livesync/ignore"), "*.tmp\n");
|
||||
await writeFile(join(vaultPath, "notes/ignored.tmp"), "ignored");
|
||||
const storedPaths: string[] = [];
|
||||
let content: string | undefined;
|
||||
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
|
||||
for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) {
|
||||
storedPaths.push(doc.path);
|
||||
}
|
||||
const file = await core.serviceModules.databaseFileAccess.fetch("notes/local.md" as FilePathWithPrefix);
|
||||
content = file ? await file.body.text() : undefined;
|
||||
return false;
|
||||
});
|
||||
|
||||
await start(command, runner);
|
||||
|
||||
expect(runner).toHaveBeenCalledOnce();
|
||||
expect(storedPaths).toEqual(["notes/local.md"]);
|
||||
expect(content).toBe("local content");
|
||||
expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content");
|
||||
}
|
||||
);
|
||||
|
||||
it("runs the mirror scan once and exits without starting file watching", async () => {
|
||||
const enumerate = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles");
|
||||
const watch = vi.mocked(chokidar.watch);
|
||||
const runner = vi.fn<CliCommandRunner>(runCommand);
|
||||
|
||||
await start("mirror", runner, 0);
|
||||
|
||||
expect(runner).toHaveBeenCalledOnce();
|
||||
expect(enumerate).toHaveBeenCalledOnce();
|
||||
expect(watch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ command: "daemon" as const, commandRuns: true },
|
||||
{ command: "mirror" as const, commandRuns: false },
|
||||
])("handles an individual file failure during $command preparation", async ({ command, commandRuns }) => {
|
||||
const store = vi
|
||||
.spyOn(ServiceFileHandler.prototype, "storeFileToDB")
|
||||
.mockRejectedValue(new Error("file failed"));
|
||||
const unload = vi.spyOn(ControlService.prototype, "onUnload");
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
|
||||
await start(command, runner);
|
||||
|
||||
expect(store).toHaveBeenCalledOnce();
|
||||
expect(runner).toHaveBeenCalledTimes(commandRuns ? 1 : 0);
|
||||
expect(unload).toHaveBeenCalledOnce();
|
||||
expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content");
|
||||
});
|
||||
|
||||
it("does not import vault files for standalone database commands", async () => {
|
||||
const storedPaths: string[] = [];
|
||||
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
|
||||
for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) {
|
||||
storedPaths.push(doc.path);
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
await start("ls", runner);
|
||||
|
||||
expect(runner).toHaveBeenCalledOnce();
|
||||
expect(storedPaths).toEqual([]);
|
||||
});
|
||||
|
||||
it("unloads without starting the command when database preparation fails", async () => {
|
||||
vi.spyOn(ControlService.prototype, "onReady").mockResolvedValue(false);
|
||||
const unload = vi.spyOn(ControlService.prototype, "onUnload");
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
const settingsBefore = await readFile(settingsPath, "utf-8");
|
||||
|
||||
await start("daemon", runner);
|
||||
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
expect(unload).toHaveBeenCalledOnce();
|
||||
expect(unload.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(process.exit).mock.invocationCallOrder[0]);
|
||||
expect(process.exit).toHaveBeenCalledWith(1);
|
||||
expect(await readFile(settingsPath, "utf-8")).toBe(settingsBefore);
|
||||
});
|
||||
|
||||
it("stops the daemon when the startup scanner refuses a suspended Vault scan", async () => {
|
||||
await writeFile(
|
||||
settingsPath,
|
||||
JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching: true })
|
||||
);
|
||||
const unload = vi.spyOn(ControlService.prototype, "onUnload");
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
|
||||
await start("daemon", runner);
|
||||
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
expect(unload).toHaveBeenCalledOnce();
|
||||
expect(process.exit).toHaveBeenCalledWith(1);
|
||||
});
|
||||
|
||||
it("unloads when database preparation throws", async () => {
|
||||
const ready = vi.spyOn(ControlService.prototype, "onReady").mockRejectedValue(new Error("scan failed"));
|
||||
const unload = vi.spyOn(ControlService.prototype, "onUnload");
|
||||
const runner = vi.fn<CliCommandRunner>(async () => false);
|
||||
|
||||
try {
|
||||
await start("daemon", runner);
|
||||
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
expect(unload).toHaveBeenCalledOnce();
|
||||
expect(standardIo.writeStderr.mock.calls.flat().join("")).toContain("scan failed");
|
||||
} finally {
|
||||
const control = ready.mock.contexts[0];
|
||||
if (unload.mock.calls.length === 0 && control instanceof ControlService) await control.onUnload();
|
||||
}
|
||||
});
|
||||
});
|
||||
+50
-13
@@ -1,6 +1,6 @@
|
||||
import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub";
|
||||
import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage";
|
||||
import { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore";
|
||||
import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules";
|
||||
import {
|
||||
LOG_LEVEL_VERBOSE,
|
||||
@@ -24,6 +24,7 @@ import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/
|
||||
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
|
||||
import { IgnoreRules } from "./serviceModules/IgnoreRules";
|
||||
import { useP2PReplicatorFeature, type UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/p2p";
|
||||
import { useOfflineScanner } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
|
||||
import type { ReplicationSchedulingControl } from "@/serviceFeatures/replicationScheduling";
|
||||
import { createNodeStandardIo, fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
|
||||
@@ -41,6 +42,27 @@ import {
|
||||
} from "./settingsPersistence";
|
||||
|
||||
const SETTINGS_FILE = ".livesync/settings.json";
|
||||
|
||||
interface CLIVaultSyncMode {
|
||||
readonly watchFiles: boolean;
|
||||
readonly reflectReplicationResults: boolean;
|
||||
readonly startupDatabaseOptions: StartupDatabaseOptions;
|
||||
}
|
||||
|
||||
// Commands which synchronise a physical Vault with the local database.
|
||||
const VAULT_SYNC_MODES: Readonly<Partial<Record<CLICommand, CLIVaultSyncMode>>> = {
|
||||
daemon: {
|
||||
watchFiles: true,
|
||||
reflectReplicationResults: true,
|
||||
startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: true },
|
||||
},
|
||||
mirror: {
|
||||
watchFiles: false,
|
||||
reflectReplicationResults: false,
|
||||
startupDatabaseOptions: { ignoreSuspending: true, continueOnFileFailure: false },
|
||||
},
|
||||
};
|
||||
|
||||
ensureGlobalNodeLocalStorage();
|
||||
defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG;
|
||||
|
||||
@@ -296,6 +318,7 @@ export async function main(
|
||||
commandRunner: CliCommandRunner = runCommand
|
||||
) {
|
||||
const options = parseArgs(standardIo);
|
||||
const vaultSyncMode = VAULT_SYNC_MODES[options.command];
|
||||
if (options.interval && options.command !== "daemon") {
|
||||
writeStderrLine(
|
||||
standardIo,
|
||||
@@ -357,9 +380,6 @@ export async function main(
|
||||
|
||||
// Resolve vault path: mirror positional argument takes priority,
|
||||
// then --vault flag, otherwise fall back to databasePath.
|
||||
// For daemon mode, enable chokidar file watching so the _changes feed picks up events.
|
||||
// mirror runs a single full scan and doesn't need continuous watching.
|
||||
const watchEnabled = options.command === "daemon";
|
||||
const vaultPath =
|
||||
options.command === "mirror" && options.commandArgs[0]
|
||||
? path.resolve(options.commandArgs[0])
|
||||
@@ -385,7 +405,7 @@ export async function main(
|
||||
infoLog(`Settings: ${settingsPath}`);
|
||||
infoLog("");
|
||||
let ignoreRules: IgnoreRules | undefined;
|
||||
if (options.command === "daemon" || options.command === "mirror") {
|
||||
if (vaultSyncMode) {
|
||||
ignoreRules = new IgnoreRules(vaultPath, (message, detail) => {
|
||||
if (detail === undefined) {
|
||||
writeStderrLine(standardIo, message);
|
||||
@@ -426,9 +446,8 @@ export async function main(
|
||||
}
|
||||
writeStderrLine(standardIo, prefix, message);
|
||||
}, true);
|
||||
// Prevent replication result from being processed automatically in non-daemon commands.
|
||||
// In daemon mode the default handler must run so changes are applied to the filesystem.
|
||||
if (options.command !== "daemon") {
|
||||
// Only modes which reflect replication results use the default filesystem handler.
|
||||
if (!vaultSyncMode?.reflectReplicationResults) {
|
||||
serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => {
|
||||
writeStderrLine(
|
||||
standardIo,
|
||||
@@ -489,12 +508,21 @@ export async function main(
|
||||
const core = new LiveSyncBaseCore(
|
||||
serviceHubInstance,
|
||||
(core: LiveSyncBaseCore<NodeServiceContext, never>, serviceHub: InjectableServiceHub<NodeServiceContext>) => {
|
||||
return initialiseServiceModulesCLI(vaultPath, core, serviceHub, ignoreRules, watchEnabled);
|
||||
return initialiseServiceModulesCLI(
|
||||
vaultPath,
|
||||
core,
|
||||
serviceHub,
|
||||
ignoreRules,
|
||||
vaultSyncMode?.watchFiles ?? false
|
||||
);
|
||||
},
|
||||
(core) => [],
|
||||
() => [], // No add-ons
|
||||
(core, coreFeatureViews) => {
|
||||
replicationScheduling = coreFeatureViews.replicationScheduling;
|
||||
if (vaultSyncMode) {
|
||||
useOfflineScanner(core);
|
||||
}
|
||||
// Register P2P replicator feature.
|
||||
p2pReplicator = useP2PReplicatorFeature(core);
|
||||
// Add target filter to prevent internal files are handled
|
||||
@@ -512,7 +540,7 @@ export async function main(
|
||||
return await Promise.resolve(true);
|
||||
}, -1 /* highest priority */);
|
||||
|
||||
// Apply user-defined ignore rules for daemon mode (lower priority, runs after dotfile check).
|
||||
// Apply user-defined ignore rules after the dotfile check.
|
||||
if (ignoreRules) {
|
||||
const rules = ignoreRules;
|
||||
core.services.vault.isTargetFile.addHandler(async (target) => {
|
||||
@@ -524,7 +552,8 @@ export async function main(
|
||||
return true;
|
||||
}, 0);
|
||||
}
|
||||
}
|
||||
},
|
||||
vaultSyncMode?.startupDatabaseOptions
|
||||
);
|
||||
if (!replicationScheduling) {
|
||||
throw new Error("Replication scheduling was not provided during core feature composition.");
|
||||
@@ -577,7 +606,7 @@ export async function main(
|
||||
: originalSettingsText;
|
||||
|
||||
// Capture sync settings before suspendAllSync() clobbers them.
|
||||
// Used by daemon mode to restore the correct sync behaviour after the mirror scan.
|
||||
// Used by daemon mode to restore sync behaviour after initial replication.
|
||||
const settingsBeforeSuspend = cloneSettings(core.services.setting.currentSettings());
|
||||
const durableSettingsBeforeSuspend = cloneSettings(settingsBeforeSuspend);
|
||||
applyStoredSetting(durableSettingsBeforeSuspend, settingsAfterLoadText, "useIndexedDBAdapter");
|
||||
@@ -592,7 +621,15 @@ export async function main(
|
||||
};
|
||||
await core.services.setting.suspendAllSync();
|
||||
const settingsAfterSuspend = cloneSettings(core.services.setting.currentSettings());
|
||||
await core.services.control.onReady();
|
||||
let readyResult = false;
|
||||
try {
|
||||
readyResult = await core.services.control.onReady();
|
||||
} finally {
|
||||
if (!readyResult) await core.services.control.onUnload();
|
||||
}
|
||||
if (!readyResult) {
|
||||
throw new Error("Failed to initialise LiveSync.");
|
||||
}
|
||||
const settingsBeforeCommand = cloneSettings(core.services.setting.currentSettings());
|
||||
const transientSettingKeys = changedSettingKeys(settingsBeforeSuspend, settingsAfterSuspend);
|
||||
for (const key of CLI_RUNTIME_ONLY_SETTING_KEYS) {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
"buildRun": "npm run build && npm run cli --",
|
||||
"build:docker": "docker build -f Dockerfile -t livesync-cli ../../..",
|
||||
"check": "tsc -p tsconfig.json",
|
||||
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
|
||||
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
|
||||
"test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh",
|
||||
"test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh",
|
||||
"test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh",
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
"test:p2p:compose": "deno run -A --no-check run-compose-p2p.ts",
|
||||
"test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts",
|
||||
"test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts",
|
||||
"test:daemon-startup": "deno test --env-file=.test.env -A --no-check test-daemon-startup.ts",
|
||||
"test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts",
|
||||
"test:remote-commands": "deno test --env-file=.test.env -A --no-check test-remote-commands.ts",
|
||||
"test:settings-writeback": "deno test -A --no-check test-settings-writeback.ts",
|
||||
|
||||
@@ -4,6 +4,7 @@ const TASKS = [
|
||||
"test:setup-put-cat",
|
||||
"test:mirror",
|
||||
"test:daemon",
|
||||
"test:daemon-startup",
|
||||
"test:push-pull",
|
||||
"test:decoupled-vault",
|
||||
"test:sync-two-local",
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import { assertEquals } from "@std/assert";
|
||||
import { join } from "@std/path";
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts";
|
||||
import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts";
|
||||
import { startCliInBackground, type BackgroundCliProcess } from "./helpers/backgroundCli.ts";
|
||||
import { startCouchdb, stopCouchdb } from "./helpers/docker.ts";
|
||||
|
||||
function envOrDefault(keys: string[], fallback: string): string {
|
||||
for (const key of keys) {
|
||||
const value = Deno.env.get(key)?.trim();
|
||||
if (value) return value;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function waitForTick(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
async function waitForText(filePath: string, expected: string, timeoutMs = 45_000): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let actual = "";
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
actual = await Deno.readTextFile(filePath);
|
||||
if (actual === expected) return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof Deno.errors.NotFound)) throw error;
|
||||
}
|
||||
await waitForTick();
|
||||
}
|
||||
throw new Error(
|
||||
`Timed out waiting for ${filePath} to contain ${JSON.stringify(expected)}; actual=${JSON.stringify(actual)}`
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForMissing(filePath: string, timeoutMs = 45_000): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
await Deno.stat(filePath);
|
||||
} catch (error) {
|
||||
if (error instanceof Deno.errors.NotFound) return;
|
||||
throw error;
|
||||
}
|
||||
await waitForTick();
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${filePath} to be removed`);
|
||||
}
|
||||
|
||||
async function stopDaemon(daemon: BackgroundCliProcess | undefined): Promise<void> {
|
||||
if (!daemon) return;
|
||||
await daemon.stop().catch(() => {});
|
||||
}
|
||||
|
||||
Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files", async () => {
|
||||
await using workDir = await TempDir.create("livesync-cli-daemon-startup");
|
||||
|
||||
const couchdbUri = envOrDefault(["COUCHDB_URI", "hostname"], "http://127.0.0.1:5989").replace(/\/$/, "");
|
||||
const couchdbUser = envOrDefault(["COUCHDB_USER", "username"], "admin");
|
||||
const couchdbPassword = envOrDefault(["COUCHDB_PASSWORD", "password"], "testpassword");
|
||||
const dbPrefix = envOrDefault(["COUCHDB_DBNAME", "dbname"], "livesync-test-db-ci");
|
||||
const dbname = `${dbPrefix}-daemon-startup-${Date.now()}-${Math.floor(Math.random() * 1_000_000)}`.toLowerCase();
|
||||
|
||||
const databaseA = workDir.join("database-a");
|
||||
const databaseB = workDir.join("database-b");
|
||||
const databaseC = workDir.join("database-c");
|
||||
const vaultA = workDir.join("vault-a");
|
||||
const vaultB = workDir.join("vault-b");
|
||||
const vaultC = workDir.join("vault-c");
|
||||
const settingsA = workDir.join("settings-a.json");
|
||||
const settingsB = workDir.join("settings-b.json");
|
||||
const settingsC = workDir.join("settings-c.json");
|
||||
|
||||
await Promise.all([
|
||||
Deno.mkdir(databaseA, { recursive: true }),
|
||||
Deno.mkdir(databaseB, { recursive: true }),
|
||||
Deno.mkdir(databaseC, { recursive: true }),
|
||||
Deno.mkdir(vaultA, { recursive: true }),
|
||||
Deno.mkdir(vaultB, { recursive: true }),
|
||||
Deno.mkdir(vaultC, { recursive: true }),
|
||||
]);
|
||||
|
||||
const startupPath = "notes/present-before-start.md";
|
||||
const deletePath = "notes/deleted-while-stopped.md";
|
||||
const remoteOnlyPath = "notes/remote-only.md";
|
||||
const startupFileA = join(vaultA, startupPath);
|
||||
const deleteFileA = join(vaultA, deletePath);
|
||||
const startupFileB = join(vaultB, startupPath);
|
||||
const deleteFileB = join(vaultB, deletePath);
|
||||
const remoteOnlyFileB = join(vaultB, remoteOnlyPath);
|
||||
|
||||
await Deno.mkdir(join(vaultA, "notes"), { recursive: true });
|
||||
await Deno.writeTextFile(startupFileA, "created before daemon startup\n");
|
||||
const initialTime = new Date(Date.now() - 10_000);
|
||||
await Deno.utime(startupFileA, initialTime, initialTime);
|
||||
await Deno.writeTextFile(deleteFileA, "delete this after the first run\n");
|
||||
|
||||
let daemonA: BackgroundCliProcess | undefined;
|
||||
let daemonB: BackgroundCliProcess | undefined;
|
||||
try {
|
||||
await startCouchdb(couchdbUri, couchdbUser, couchdbPassword, dbname);
|
||||
for (const settings of [settingsA, settingsB, settingsC]) {
|
||||
await initSettingsFile(settings);
|
||||
await applyCouchdbSettings(settings, couchdbUri, couchdbUser, couchdbPassword, dbname, true);
|
||||
}
|
||||
|
||||
// A pre-existing local file must be uploaded by the daemon's startup scan.
|
||||
daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon");
|
||||
await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000);
|
||||
|
||||
// A separate daemon proves that the first startup replication reached CouchDB
|
||||
// and that remote files are reflected into its filesystem.
|
||||
daemonB = startCliInBackground(databaseB, "--vault", vaultB, "--settings", settingsB, "daemon");
|
||||
await daemonB.waitUntilContains("[Daemon] Initial replication complete", 45_000);
|
||||
await waitForText(startupFileB, "created before daemon startup\n");
|
||||
await waitForText(deleteFileB, "delete this after the first run\n");
|
||||
|
||||
// Changes made while A is stopped must be found by its next startup scan.
|
||||
assertEquals(await daemonA.stop(), 0, daemonA.combined);
|
||||
daemonA = undefined;
|
||||
await Deno.writeTextFile(startupFileA, "edited while daemon was stopped\n");
|
||||
await Deno.remove(deleteFileA);
|
||||
|
||||
daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon");
|
||||
await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000);
|
||||
await waitForText(startupFileB, "edited while daemon was stopped\n");
|
||||
await waitForMissing(deleteFileB);
|
||||
|
||||
// Seed a file into a third local database without creating it in vault C.
|
||||
// After C's finite sync, it exists only remotely from B's point of view.
|
||||
await runCliWithInputOrFail(
|
||||
"created in a different local database\n",
|
||||
databaseC,
|
||||
"--vault",
|
||||
vaultC,
|
||||
"--settings",
|
||||
settingsC,
|
||||
"put",
|
||||
remoteOnlyPath
|
||||
);
|
||||
await runCliOrFail(databaseC, "--vault", vaultC, "--settings", settingsC, "sync");
|
||||
await waitForText(remoteOnlyFileB, "created in a different local database\n");
|
||||
assertEquals(await Deno.readTextFile(startupFileB), "edited while daemon was stopped\n");
|
||||
assertEquals((await Deno.stat(remoteOnlyFileB)).isFile, true);
|
||||
} finally {
|
||||
await stopDaemon(daemonB);
|
||||
await stopDaemon(daemonA);
|
||||
await stopCouchdb().catch(() => {});
|
||||
}
|
||||
});
|
||||
@@ -45,7 +45,13 @@ export class ModuleLiveSyncMain extends AbstractModule {
|
||||
}
|
||||
// Ordinary start-up may continue when individual files could not be
|
||||
// processed. Explicit Fetch and Rebuild flows retain the strict default.
|
||||
const initialisationResult = await this.services.databaseEvents.initialiseDatabase(false, false, false, true);
|
||||
const { ignoreSuspending = false, continueOnFileFailure = true } = this.core.startupDatabaseOptions;
|
||||
const initialisationResult = await this.services.databaseEvents.initialiseDatabase(
|
||||
false,
|
||||
false,
|
||||
ignoreSuspending,
|
||||
continueOnFileFailure
|
||||
);
|
||||
if (initialisationResult === VaultScanResults.FAILED) {
|
||||
this._log($msg("Ui.Common.LocalDatabaseInitialisationFailed"), LOG_LEVEL_NOTICE);
|
||||
//TODO:stop all sync.
|
||||
|
||||
@@ -25,6 +25,7 @@ describe("ModuleLiveSyncMain", () => {
|
||||
const log = vi.fn();
|
||||
const host = {
|
||||
core: {
|
||||
startupDatabaseOptions: {},
|
||||
services: {
|
||||
appLifecycle: {
|
||||
onLayoutReady: vi.fn(async () => true),
|
||||
@@ -58,6 +59,7 @@ describe("ModuleLiveSyncMain", () => {
|
||||
};
|
||||
const host = {
|
||||
core: {
|
||||
startupDatabaseOptions: {},
|
||||
services: { appLifecycle },
|
||||
},
|
||||
services: {
|
||||
@@ -77,4 +79,33 @@ describe("ModuleLiveSyncMain", () => {
|
||||
expect(result).toBe(true);
|
||||
expect(log).toHaveBeenCalledWith("Ui.Common.SomeFilesCouldNotBeSynchronised", LOG_LEVEL_NOTICE);
|
||||
});
|
||||
|
||||
it("passes strict startup database options to initialisation", async () => {
|
||||
const initialiseDatabase = vi.fn(async () => false);
|
||||
const host = {
|
||||
core: {
|
||||
startupDatabaseOptions: {
|
||||
ignoreSuspending: true,
|
||||
continueOnFileFailure: false,
|
||||
},
|
||||
services: {
|
||||
appLifecycle: {
|
||||
onLayoutReady: vi.fn(async () => true),
|
||||
},
|
||||
},
|
||||
},
|
||||
services: {
|
||||
databaseEvents: { initialiseDatabase },
|
||||
},
|
||||
settings: {
|
||||
suspendFileWatching: false,
|
||||
suspendParseReplicationResult: false,
|
||||
},
|
||||
_log: vi.fn(),
|
||||
};
|
||||
|
||||
await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never);
|
||||
|
||||
expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
|
||||
|
||||
### Fixed
|
||||
|
||||
- CLI: daemon and mirror now scan the Vault during database initialisation, following the Obsidian startup sequence. The daemon completes this scan before replication; mirror runs the scan once and still exits with an error if any file cannot be processed.
|
||||
- CLI: file enumeration now includes current files even after individual path lookups or earlier scans.
|
||||
|
||||
## 1.0.28
|
||||
|
||||
Reference in New Issue
Block a user