Compare commits

..
28 changed files with 751 additions and 207 deletions
+1 -1
View File
@@ -56,7 +56,7 @@ jobs:
case "$SELECTED_TASK" in
test:ci)
TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon","test:push-pull","test:decoupled-vault","test:sync-two-local","test:sync-locked-remote","test:remote-commands","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]'
TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon","test:daemon-startup","test:push-pull","test:decoupled-vault","test:sync-two-local","test:sync-locked-remote","test:remote-commands","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]'
;;
test:local)
TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon"]'
+1 -1
View File
@@ -199,7 +199,7 @@ jobs:
if: ${{ steps.integration_tests.outputs.present == 'true' }}
run: npm run test:docker-couchdb:start
- name: Start MinIO container
- name: Start RustFS container
if: ${{ steps.integration_tests.outputs.present == 'true' }}
run: npm run test:docker-s3:start
+2 -2
View File
@@ -87,9 +87,9 @@ Regression tests remain in the suite owned by the implementation under test. Plu
- **CLI E2E** (`src/apps/cli/testdeno/`): Host-independent consumer workflows. The canonical Compose P2P suite covers ordinary two-peer synchronisation, replacement of the current Replicator followed by transfer with the same peer, and explicit relay disconnection followed by paused and resumed reconnection. Its lifecycle entry point is included only in the Docker test build and does not add a public CLI command. Run `npm run test:e2e:cli` for the ordinary suite or `npm run test:e2e:cli:p2p` for P2P validation.
- **Self-hosted setup tools** (`utils/couchdb/`, `utils/setup/`, and `utils/flyio/`): Deno contract tests consume the exact locked Commonlib registry package, verify current CouchDB, Object Storage, and random-room P2P Setup URI defaults and remote profiles, and keep CouchDB administration separate from package-owned LiveSync database-version negotiation. `unit-ci` also provisions a real temporary CouchDB database and verifies its version document against the installed Commonlib package. Run `npm run test:setup-tools` for the local contract gate.
- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. Run focused scripts such as `npm run test:e2e:obsidian:two-vault-sync`, or use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and MinIO fixtures managed by the wrapper.
- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. Run focused scripts such as `npm run test:e2e:obsidian:two-vault-sync`, or use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and RustFS fixtures managed by the wrapper.
- **Docker Services**: Service-backed tests use CouchDB and MinIO (S3). Canonical P2P validation owns its relay through the CLI Compose runner:
- **Docker Services**: Service-backed tests use CouchDB and RustFS (S3). Canonical P2P validation owns its relay through the CLI Compose runner:
```bash
npm run test:docker-all:start # Start all test services
+7 -1
View File
@@ -38,6 +38,11 @@ export interface LiveSyncCoreFeatureViews {
readonly replicationScheduling: ReplicationSchedulingControl;
}
export interface StartupDatabaseOptions {
readonly ignoreSuspending?: boolean;
readonly continueOnFileFailure?: boolean;
}
type CompatibilityReplicatorView = ReplicatorInstance & Partial<LiveSyncAbstractReplicator>;
export class LiveSyncBaseCore<
@@ -78,7 +83,8 @@ export class LiveSyncBaseCore<
) => ServiceModules,
extraModuleInitialiser: (core: LiveSyncBaseCore<T, TCommands>) => AbstractModule[],
addOnsInitialiser: (core: LiveSyncBaseCore<T, TCommands>) => TCommands[],
featuresInitialiser: (core: LiveSyncBaseCore<T, TCommands>, coreFeatureViews: LiveSyncCoreFeatureViews) => void
featuresInitialiser: (core: LiveSyncBaseCore<T, TCommands>, coreFeatureViews: LiveSyncCoreFeatureViews) => void,
readonly startupDatabaseOptions: StartupDatabaseOptions = {}
) {
this._services = serviceHub;
this.registerReplicatorProviders();
@@ -92,10 +92,9 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
}
async getFiles(): Promise<NodeFile[]> {
if (this.fileCache.size === 0) {
await this.scanDirectory();
}
return Array.from(this.fileCache.values());
const files = new Map<string, NodeFile>();
await this.scanDirectoryInto("", files);
return Array.from(files.values());
}
async renameFile(file: NodeFile, newPath: string): Promise<NodeFile> {
@@ -147,6 +146,10 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
* Helper method to recursively scan directory and populate file cache
*/
async scanDirectory(relativePath: string = ""): Promise<void> {
await this.scanDirectoryInto(relativePath, this.fileCache);
}
private async scanDirectoryInto(relativePath: string, files: Map<string, NodeFile>): Promise<void> {
const fullPath = this.resolvePath(relativePath);
try {
const directoryStat = await this.storage.stat(relativePath);
@@ -160,10 +163,10 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
path: entryPath as FilePath,
stat,
};
this.fileCache.set(entryPath, file);
files.set(entryPath, file);
}
for (const entryPath of entries.folders) {
await this.scanDirectory(entryPath);
await this.scanDirectoryInto(entryPath, files);
}
} catch (error) {
// Directory doesn't exist or is not readable
@@ -0,0 +1,118 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node";
import { NodeFileSystemAdapter } from "./NodeFileSystemAdapter";
describe("NodeFileSystemAdapter file enumeration", () => {
const tempDirs: string[] = [];
const paths = ["a.md", "folder/b.md", "folder/sub/c.md"];
async function createVault() {
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-enumeration-"));
tempDirs.push(directory);
for (const file of paths) {
await fs.mkdir(path.dirname(path.join(directory, file)), { recursive: true });
await fs.writeFile(path.join(directory, file), `content of ${file}`);
}
return { directory, adapter: new NodeFileSystemAdapter(directory) };
}
afterEach(async () => {
await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })));
});
it("lists every file when one file was refreshed before the first enumeration", async () => {
const { adapter } = await createVault();
expect(await adapter.refreshFile("folder/b.md")).not.toBeNull();
expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
});
it("lists every file after a path lookup without any replication", async () => {
const { adapter } = await createVault();
expect((await adapter.getAbstractFileByPath("folder/b.md"))?.path).toBe("folder/b.md");
expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
});
it("lists every file on the first enumeration without a prior path lookup", async () => {
const { adapter } = await createVault();
expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
});
it("excludes a deleted file after its cache entry is refreshed", async () => {
const { directory, adapter } = await createVault();
await adapter.getFiles();
await fs.rm(path.join(directory, "folder/b.md"));
expect(await adapter.refreshFile("folder/b.md")).toBeNull();
expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(["a.md", "folder/sub/c.md"]);
});
it("reflects files added and deleted between enumerations", async () => {
const { directory, adapter } = await createVault();
expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
await fs.rm(path.join(directory, "folder/b.md"));
const updatedContent = "updated content of a.md";
await fs.writeFile(path.join(directory, "a.md"), updatedContent);
await fs.writeFile(path.join(directory, "later.md"), "content of later.md");
const files = await adapter.getFiles();
expect(files.map((file) => file.path).sort()).toEqual(["a.md", "folder/sub/c.md", "later.md"]);
expect(files.find((file) => file.path === "a.md")?.stat.size).toBe(updatedContent.length);
});
it("returns complete listings from simultaneous calls", async () => {
const { adapter } = await createVault();
const originalStat = adapter.storage.stat.bind(adapter.storage);
let releaseFolderStat!: () => void;
const folderStatReleased = new Promise<void>((resolve) => {
releaseFolderStat = resolve;
});
let folderStatStarted!: () => void;
const folderStatStartedPromise = new Promise<void>((resolve) => {
folderStatStarted = resolve;
});
let pauseFolderStat = true;
const statSpy = vi.spyOn(adapter.storage, "stat").mockImplementation(async (relativePath) => {
const stat = await originalStat(relativePath);
if (pauseFolderStat && relativePath === "folder") {
pauseFolderStat = false;
folderStatStarted();
await folderStatReleased;
}
return stat;
});
const firstListing = adapter.getFiles();
let listings: Awaited<ReturnType<typeof adapter.getFiles>>[] | undefined;
try {
await folderStatStartedPromise;
const secondListing = adapter.getFiles();
const secondFiles = await secondListing;
releaseFolderStat();
const firstFiles = await firstListing;
listings = [secondFiles, firstFiles];
} finally {
releaseFolderStat();
statSpy.mockRestore();
}
if (!listings) throw new Error("Expected both concurrent listings to complete");
expect(listings.map((files) => files.map((file) => file.path).sort())).toEqual([paths, paths]);
});
it("returns an empty listing for an empty vault", async () => {
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-enumeration-empty-"));
tempDirs.push(directory);
const adapter = new NodeFileSystemAdapter(directory);
await expect(adapter.getFiles()).resolves.toEqual([]);
});
});
@@ -4,7 +4,7 @@ import { NO_INTERACTION } from "@vrtmrz/livesync-commonlib/replication";
import { runCommand } from "./runCommand";
import type { CLIOptions } from "./types";
// Mock performFullScan so daemon tests don't require a real CouchDB connection.
// Track explicit scans: database preparation owns the daemon startup scan.
vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", () => ({
performFullScan: vi.fn(async () => true),
}));
@@ -102,6 +102,7 @@ function createDaemonContext(core: ReturnType<typeof createCoreMock>) {
describe("daemon command", () => {
beforeEach(() => {
vi.restoreAllMocks();
vi.mocked(offlineScanner.performFullScan).mockClear();
vi.useFakeTimers();
});
@@ -109,27 +110,16 @@ describe("daemon command", () => {
vi.useRealTimers();
});
it("calls performFullScan during startup", async () => {
it("does not repeat the startup scan after initial replication", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
await runCommand(makeDaemonOptions(), createDaemonContext(core));
expect(await runCommand(makeDaemonOptions(), createDaemonContext(core))).toBe(true);
expect(offlineScanner.performFullScan).toHaveBeenCalledTimes(1);
});
it("returns false when performFullScan fails", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(false);
const result = await runCommand(makeDaemonOptions(), createDaemonContext(core));
expect(result).toBe(false);
expect(offlineScanner.performFullScan).not.toHaveBeenCalled();
});
it("polling mode: calls setTimeout when interval option is set", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
const context = createDaemonContext(core);
@@ -143,7 +133,6 @@ describe("daemon command", () => {
it("polling mode: applies settings with suspendFileWatching=false before setting interval", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
await runCommand(makeDaemonOptions(10), createDaemonContext(core));
@@ -156,7 +145,6 @@ describe("daemon command", () => {
it("liveSync mode: calls applyPartial and applySettings", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
await runCommand(makeDaemonOptions(), createDaemonContext(core));
@@ -176,7 +164,6 @@ describe("daemon command", () => {
liveSync: false,
syncOnStart: false,
}));
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
const result = await runCommand(makeDaemonOptions(), createDaemonContext(core));
@@ -194,7 +181,6 @@ describe("daemon command", () => {
liveSync: true,
syncOnStart: false,
}));
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
await runCommand(makeDaemonOptions(), createDaemonContext(core));
@@ -205,22 +191,21 @@ describe("daemon command", () => {
expect(warningCalls.length).toBe(0);
});
it("calls replicate before performFullScan", async () => {
it("completes initial replication before restoring automatic synchronisation", async () => {
const core = createCoreMock();
const callOrder: string[] = [];
core.services.replication.replicateUnattended = vi.fn(async () => {
callOrder.push("replicate");
return { status: "completed" as const };
});
vi.mocked(offlineScanner.performFullScan).mockImplementation(async () => {
callOrder.push("performFullScan");
return true;
core.services.control.applySettings.mockImplementation(async () => {
callOrder.push("restoreSettings");
});
const context = createDaemonContext(core);
await runCommand(makeDaemonOptions(), context);
expect(callOrder).toEqual(["replicate", "performFullScan"]);
expect(callOrder).toEqual(["replicate", "restoreSettings"]);
expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({
trigger: "daemon",
interaction: NO_INTERACTION,
@@ -234,12 +219,11 @@ describe("daemon command", () => {
status: "failed" as const,
error: new Error("initial replication failed"),
}));
vi.mocked(offlineScanner.performFullScan).mockClear();
const result = await runCommand(makeDaemonOptions(), createDaemonContext(core));
expect(result).toBe(false);
// performFullScan should NOT have been called
expect(core.services.control.applySettings).not.toHaveBeenCalled();
expect(offlineScanner.performFullScan).not.toHaveBeenCalled();
expect(core.services.replication.replicateUnattended).toHaveBeenCalledWith({
trigger: "daemon",
@@ -249,7 +233,6 @@ describe("daemon command", () => {
it("polling mode: registers onUnload handler that clears timeout", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
await runCommand(makeDaemonOptions(10), createDaemonContext(core));
@@ -265,7 +248,6 @@ describe("daemon command", () => {
it("polling backoff: interval escalates on failure, caps at 300000ms, then halves on recovery", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
// startup replicate (call 1) succeeds; poll calls 27 fail; call 8 succeeds.
let callCount = 0;
@@ -320,7 +302,6 @@ describe("daemon command", () => {
it("polling error handling: replicate rejection is caught and written to standard error", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
// Make replicate succeed on the initial call (startup), then fail on the poll.
let callCount = 0;
+6 -22
View File
@@ -15,11 +15,6 @@ import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_b
import type { CLICommandContext, CLIOptions } from "./types";
import { toArrayBuffer, toDatabaseRelativePath } from "./utils";
import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p";
import {
performFullScan,
VaultScanResults,
} from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import { UnresolvedErrorManager } from "@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
import { writeStderrLine, writeStdoutLine } from "@/apps/cli/cliOutput";
@@ -59,9 +54,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
// accept whatever configuration the remote has.
await core.services.setting.applyPartial({ disableCheckingConfigMismatch: true }, true);
// 1. Replicate the configured remote into the local database so the
// mirror scan has content to work with.
log("Replicating from remote...");
// Database preparation has already reconciled the local database and Vault.
// Replicate before restoring automatic synchronisation.
log("Replicating with remote...");
const replResult = await core.services.replication.replicateUnattended({
trigger: "daemon",
interaction: NO_INTERACTION,
@@ -73,17 +68,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
replicationScheduling.markInitialOneShotSatisfied();
log("Initial replication complete");
// 2. Mirror scan to reconcile PouchDB ↔ local filesystem.
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
log("Running mirror scan...");
const scanOk = await performFullScan(core, log, errorManager, false, true);
if (!scanOk) {
writeStderrLine(standardIo, "[Daemon] Mirror scan failed, cannot continue");
return false;
}
log("Mirror scan complete");
// 3. Re-enable sync.
// Re-enable sync.
const restoreSyncSettings = async () => {
await core.services.setting.applyPartial(
{
@@ -530,9 +515,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
if (options.command === "mirror") {
writeStderrLine(standardIo, "[Command] mirror");
const log = (msg: unknown) => writeStderrLine(standardIo, `[Mirror] ${String(msg)}`);
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
return (await performFullScan(core, log, errorManager, false, true)) === VaultScanResults.COMPLETED;
// Database preparation has already completed the mirror scan.
return true;
}
if (options.command === "remote-add") {
+195
View File
@@ -0,0 +1,195 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import * as chokidar from "chokidar";
import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { ServiceFileHandler } from "@/serviceModules/FileHandler";
import { ServiceFileAccessCLI } from "./serviceModules/ServiceFileAccessImpl";
import { runCommand } from "./commands/runCommand";
import { createDefaultCliSettings } from "./cliSettingsDefaults";
import { main, type CliCommandRunner } from "./main";
vi.mock("chokidar", { spy: true });
function createStandardIoMock() {
return {
readStdin: vi.fn(async () => ""),
prompt: vi.fn(async () => ""),
writeStdout: vi.fn(),
writeStderr: vi.fn(),
};
}
describe("CLI database preparation", () => {
const originalArgv = process.argv.slice();
const originalExitCode = process.exitCode;
let directory: string;
let vaultPath: string;
let settingsPath: string;
let signalHandlers: Map<"SIGINT" | "SIGTERM", Set<NodeJS.SignalsListener>>;
let standardIo: ReturnType<typeof createStandardIoMock>;
beforeEach(async () => {
vi.mocked(chokidar.watch).mockClear();
directory = await mkdtemp(join(tmpdir(), "livesync-cli-bootstrap-"));
vaultPath = join(directory, "vault");
settingsPath = join(directory, "settings.json");
await mkdir(join(vaultPath, "notes"), { recursive: true });
await writeFile(join(vaultPath, "notes/local.md"), "local content");
await writeFile(settingsPath, JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true }));
standardIo = createStandardIoMock();
signalHandlers = new Map(
(["SIGINT", "SIGTERM"] as const).map((signal) => [signal, new Set(process.listeners(signal))])
);
process.exitCode = undefined;
vi.spyOn(process, "exit").mockImplementation((code) => {
throw new Error(`__EXIT__:${code ?? 0}`);
});
});
afterEach(async () => {
for (const [signal, originalHandlers] of signalHandlers) {
for (const handler of process.listeners(signal)) {
if (!originalHandlers.has(handler)) process.removeListener(signal, handler);
}
}
process.argv = originalArgv.slice();
process.exitCode = originalExitCode;
vi.restoreAllMocks();
await rm(directory, { recursive: true, force: true });
});
async function start(command: "daemon" | "mirror" | "ls", runner: CliCommandRunner, exitCode = 1) {
process.argv = ["node", "livesync-cli", directory, "--vault", vaultPath, "--settings", settingsPath, command];
// Daemon probes return false so the real core unloads without keeping a daemon alive.
await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`);
}
it.each([
{ command: "daemon" as const, suspendFileWatching: false },
{ command: "mirror" as const, suspendFileWatching: false },
{ command: "mirror" as const, suspendFileWatching: true },
])(
"prepares the Vault before $command (watching suspended: $suspendFileWatching)",
async ({ command, suspendFileWatching }) => {
await writeFile(
settingsPath,
JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching })
);
await mkdir(join(vaultPath, ".livesync"));
await writeFile(join(vaultPath, ".livesync/ignore"), "*.tmp\n");
await writeFile(join(vaultPath, "notes/ignored.tmp"), "ignored");
const storedPaths: string[] = [];
let content: string | undefined;
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) {
storedPaths.push(doc.path);
}
const file = await core.serviceModules.databaseFileAccess.fetch("notes/local.md" as FilePathWithPrefix);
content = file ? await file.body.text() : undefined;
return false;
});
await start(command, runner);
expect(runner).toHaveBeenCalledOnce();
expect(storedPaths).toEqual(["notes/local.md"]);
expect(content).toBe("local content");
expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content");
}
);
it("runs the mirror scan once and exits without starting file watching", async () => {
const enumerate = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles");
const watch = vi.mocked(chokidar.watch);
const runner = vi.fn<CliCommandRunner>(runCommand);
await start("mirror", runner, 0);
expect(runner).toHaveBeenCalledOnce();
expect(enumerate).toHaveBeenCalledOnce();
expect(watch).not.toHaveBeenCalled();
});
it.each([
{ command: "daemon" as const, commandRuns: true },
{ command: "mirror" as const, commandRuns: false },
])("handles an individual file failure during $command preparation", async ({ command, commandRuns }) => {
const store = vi
.spyOn(ServiceFileHandler.prototype, "storeFileToDB")
.mockRejectedValue(new Error("file failed"));
const unload = vi.spyOn(ControlService.prototype, "onUnload");
const runner = vi.fn<CliCommandRunner>(async () => false);
await start(command, runner);
expect(store).toHaveBeenCalledOnce();
expect(runner).toHaveBeenCalledTimes(commandRuns ? 1 : 0);
expect(unload).toHaveBeenCalledOnce();
expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content");
});
it("does not import vault files for standalone database commands", async () => {
const storedPaths: string[] = [];
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) {
storedPaths.push(doc.path);
}
return false;
});
await start("ls", runner);
expect(runner).toHaveBeenCalledOnce();
expect(storedPaths).toEqual([]);
});
it("unloads without starting the command when database preparation fails", async () => {
vi.spyOn(ControlService.prototype, "onReady").mockResolvedValue(false);
const unload = vi.spyOn(ControlService.prototype, "onUnload");
const runner = vi.fn<CliCommandRunner>(async () => false);
const settingsBefore = await readFile(settingsPath, "utf-8");
await start("daemon", runner);
expect(runner).not.toHaveBeenCalled();
expect(unload).toHaveBeenCalledOnce();
expect(unload.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(process.exit).mock.invocationCallOrder[0]);
expect(process.exit).toHaveBeenCalledWith(1);
expect(await readFile(settingsPath, "utf-8")).toBe(settingsBefore);
});
it("stops the daemon when the startup scanner refuses a suspended Vault scan", async () => {
await writeFile(
settingsPath,
JSON.stringify({ ...createDefaultCliSettings(), isConfigured: true, suspendFileWatching: true })
);
const unload = vi.spyOn(ControlService.prototype, "onUnload");
const runner = vi.fn<CliCommandRunner>(async () => false);
await start("daemon", runner);
expect(runner).not.toHaveBeenCalled();
expect(unload).toHaveBeenCalledOnce();
expect(process.exit).toHaveBeenCalledWith(1);
});
it("unloads when database preparation throws", async () => {
const ready = vi.spyOn(ControlService.prototype, "onReady").mockRejectedValue(new Error("scan failed"));
const unload = vi.spyOn(ControlService.prototype, "onUnload");
const runner = vi.fn<CliCommandRunner>(async () => false);
try {
await start("daemon", runner);
expect(runner).not.toHaveBeenCalled();
expect(unload).toHaveBeenCalledOnce();
expect(standardIo.writeStderr.mock.calls.flat().join("")).toContain("scan failed");
} finally {
const control = ready.mock.contexts[0];
if (unload.mock.calls.length === 0 && control instanceof ControlService) await control.onUnload();
}
});
});
+50 -13
View File
@@ -1,6 +1,6 @@
import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub";
import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage";
import { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore";
import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules";
import {
LOG_LEVEL_VERBOSE,
@@ -24,6 +24,7 @@ import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import { IgnoreRules } from "./serviceModules/IgnoreRules";
import { useP2PReplicatorFeature, type UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/p2p";
import { useOfflineScanner } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import type { ReplicationSchedulingControl } from "@/serviceFeatures/replicationScheduling";
import { createNodeStandardIo, fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
@@ -41,6 +42,27 @@ import {
} from "./settingsPersistence";
const SETTINGS_FILE = ".livesync/settings.json";
interface CLIVaultSyncMode {
readonly watchFiles: boolean;
readonly reflectReplicationResults: boolean;
readonly startupDatabaseOptions: StartupDatabaseOptions;
}
// Commands which synchronise a physical Vault with the local database.
const VAULT_SYNC_MODES: Readonly<Partial<Record<CLICommand, CLIVaultSyncMode>>> = {
daemon: {
watchFiles: true,
reflectReplicationResults: true,
startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: true },
},
mirror: {
watchFiles: false,
reflectReplicationResults: false,
startupDatabaseOptions: { ignoreSuspending: true, continueOnFileFailure: false },
},
};
ensureGlobalNodeLocalStorage();
defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG;
@@ -296,6 +318,7 @@ export async function main(
commandRunner: CliCommandRunner = runCommand
) {
const options = parseArgs(standardIo);
const vaultSyncMode = VAULT_SYNC_MODES[options.command];
if (options.interval && options.command !== "daemon") {
writeStderrLine(
standardIo,
@@ -357,9 +380,6 @@ export async function main(
// Resolve vault path: mirror positional argument takes priority,
// then --vault flag, otherwise fall back to databasePath.
// For daemon mode, enable chokidar file watching so the _changes feed picks up events.
// mirror runs a single full scan and doesn't need continuous watching.
const watchEnabled = options.command === "daemon";
const vaultPath =
options.command === "mirror" && options.commandArgs[0]
? path.resolve(options.commandArgs[0])
@@ -385,7 +405,7 @@ export async function main(
infoLog(`Settings: ${settingsPath}`);
infoLog("");
let ignoreRules: IgnoreRules | undefined;
if (options.command === "daemon" || options.command === "mirror") {
if (vaultSyncMode) {
ignoreRules = new IgnoreRules(vaultPath, (message, detail) => {
if (detail === undefined) {
writeStderrLine(standardIo, message);
@@ -426,9 +446,8 @@ export async function main(
}
writeStderrLine(standardIo, prefix, message);
}, true);
// Prevent replication result from being processed automatically in non-daemon commands.
// In daemon mode the default handler must run so changes are applied to the filesystem.
if (options.command !== "daemon") {
// Only modes which reflect replication results use the default filesystem handler.
if (!vaultSyncMode?.reflectReplicationResults) {
serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => {
writeStderrLine(
standardIo,
@@ -489,12 +508,21 @@ export async function main(
const core = new LiveSyncBaseCore(
serviceHubInstance,
(core: LiveSyncBaseCore<NodeServiceContext, never>, serviceHub: InjectableServiceHub<NodeServiceContext>) => {
return initialiseServiceModulesCLI(vaultPath, core, serviceHub, ignoreRules, watchEnabled);
return initialiseServiceModulesCLI(
vaultPath,
core,
serviceHub,
ignoreRules,
vaultSyncMode?.watchFiles ?? false
);
},
(core) => [],
() => [], // No add-ons
(core, coreFeatureViews) => {
replicationScheduling = coreFeatureViews.replicationScheduling;
if (vaultSyncMode) {
useOfflineScanner(core);
}
// Register P2P replicator feature.
p2pReplicator = useP2PReplicatorFeature(core);
// Add target filter to prevent internal files are handled
@@ -512,7 +540,7 @@ export async function main(
return await Promise.resolve(true);
}, -1 /* highest priority */);
// Apply user-defined ignore rules for daemon mode (lower priority, runs after dotfile check).
// Apply user-defined ignore rules after the dotfile check.
if (ignoreRules) {
const rules = ignoreRules;
core.services.vault.isTargetFile.addHandler(async (target) => {
@@ -524,7 +552,8 @@ export async function main(
return true;
}, 0);
}
}
},
vaultSyncMode?.startupDatabaseOptions
);
if (!replicationScheduling) {
throw new Error("Replication scheduling was not provided during core feature composition.");
@@ -577,7 +606,7 @@ export async function main(
: originalSettingsText;
// Capture sync settings before suspendAllSync() clobbers them.
// Used by daemon mode to restore the correct sync behaviour after the mirror scan.
// Used by daemon mode to restore sync behaviour after initial replication.
const settingsBeforeSuspend = cloneSettings(core.services.setting.currentSettings());
const durableSettingsBeforeSuspend = cloneSettings(settingsBeforeSuspend);
applyStoredSetting(durableSettingsBeforeSuspend, settingsAfterLoadText, "useIndexedDBAdapter");
@@ -592,7 +621,15 @@ export async function main(
};
await core.services.setting.suspendAllSync();
const settingsAfterSuspend = cloneSettings(core.services.setting.currentSettings());
await core.services.control.onReady();
let readyResult = false;
try {
readyResult = await core.services.control.onReady();
} finally {
if (!readyResult) await core.services.control.onUnload();
}
if (!readyResult) {
throw new Error("Failed to initialise LiveSync.");
}
const settingsBeforeCommand = cloneSettings(core.services.setting.currentSettings());
const transientSettingKeys = changedSettingKeys(settingsBeforeSuspend, settingsAfterSuspend);
for (const key of CLI_RUNTIME_ONLY_SETTING_KEYS) {
+1 -1
View File
@@ -12,7 +12,7 @@
"buildRun": "npm run build && npm run cli --",
"build:docker": "docker build -f Dockerfile -t livesync-cli ../../..",
"check": "tsc -p tsconfig.json",
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts",
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
"test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh",
"test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh",
"test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh",
+27 -10
View File
@@ -307,10 +307,19 @@ cli_test_wait_for_minio_bucket() {
local delay_sec=2
local i
for ((i = 1; i <= retries; i++)); do
if docker run --rm --network host --entrypoint=/bin/sh minio/mc -c "mc alias set myminio $minio_endpoint $minio_access_key $minio_secret_key >/dev/null 2>&1 && mc ls myminio/$minio_bucket >/dev/null 2>&1"; then
if docker run --rm --network host --entrypoint=/bin/sh \
rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914 \
-c 'set -e
rc alias set myminio "$1" "$2" "$3" >/dev/null 2>&1
rc ls "myminio/$4" >/dev/null 2>&1
' sh "$minio_endpoint" "$minio_access_key" "$minio_secret_key" "$minio_bucket"; then
return 0
fi
bucketName="$minio_bucket" bash "$CLI_DIR/util/minio-init.sh" >/dev/null 2>&1 || true
minioEndpoint="$minio_endpoint" \
accessKey="$minio_access_key" \
secretKey="$minio_secret_key" \
bucketName="$minio_bucket" \
bash "$CLI_DIR/util/minio-init.sh" >/dev/null 2>&1 || true
sleep "$delay_sec"
done
return 1
@@ -323,26 +332,34 @@ cli_test_start_minio() {
local minio_bucket="$4"
local minio_init_ok=0
echo "[INFO] stopping leftover MinIO container if present"
echo "[INFO] stopping leftover RustFS container if present"
cli_test_stop_minio
echo "[INFO] starting MinIO test container"
bucketName="$minio_bucket" bash "$CLI_DIR/util/minio-start.sh"
echo "[INFO] starting RustFS test container"
minioEndpoint="$minio_endpoint" \
accessKey="$minio_access_key" \
secretKey="$minio_secret_key" \
bucketName="$minio_bucket" \
bash "$CLI_DIR/util/minio-start.sh"
echo "[INFO] initialising MinIO test bucket: $minio_bucket"
echo "[INFO] initialising RustFS test bucket: $minio_bucket"
for _ in 1 2 3 4 5; do
if bucketName="$minio_bucket" bash "$CLI_DIR/util/minio-init.sh"; then
if minioEndpoint="$minio_endpoint" \
accessKey="$minio_access_key" \
secretKey="$minio_secret_key" \
bucketName="$minio_bucket" \
bash "$CLI_DIR/util/minio-init.sh"; then
minio_init_ok=1
break
fi
sleep 2
done
if [[ "$minio_init_ok" != "1" ]]; then
echo "[FAIL] could not initialise MinIO bucket after retries: $minio_bucket" >&2
echo "[FAIL] could not initialise RustFS bucket after retries: $minio_bucket" >&2
exit 1
fi
if ! cli_test_wait_for_minio_bucket "$minio_endpoint" "$minio_access_key" "$minio_secret_key" "$minio_bucket"; then
echo "[FAIL] MinIO bucket not ready: $minio_bucket" >&2
echo "[FAIL] RustFS bucket not ready: $minio_bucket" >&2
exit 1
fi
}
@@ -359,4 +376,4 @@ display_test_info(){
if [[ "${LIVESYNC_TEST_DOCKER:-0}" == "1" ]]; then
# shellcheck source=/dev/null
source "$(dirname "${BASH_SOURCE[0]}")/test-helpers-docker.sh"
fi
fi
+1
View File
@@ -5,6 +5,7 @@
"test:p2p:compose": "deno run -A --no-check run-compose-p2p.ts",
"test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts",
"test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts",
"test:daemon-startup": "deno test --env-file=.test.env -A --no-check test-daemon-startup.ts",
"test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts",
"test:remote-commands": "deno test --env-file=.test.env -A --no-check test-remote-commands.ts",
"test:settings-writeback": "deno test -A --no-check test-settings-writeback.ts",
+37 -23
View File
@@ -190,7 +190,7 @@ async function dockerOrFail(...args: string[]): Promise<string> {
async function stopAndRemoveContainer(container: string): Promise<void> {
await docker("stop", container).catch(() => {});
await docker("rm", container).catch(() => {});
await docker("rm", "-v", container).catch(() => {});
}
async function cleanupTrackedContainers(reason: string): Promise<void> {
@@ -327,8 +327,22 @@ const COUCHDB_CONTAINER = "couchdb-test";
const COUCHDB_IMAGE = "couchdb:3.5.0";
const MINIO_CONTAINER = "minio-test";
const MINIO_IMAGE = "minio/minio";
const MINIO_MC_IMAGE = "minio/mc";
// RustFS provides the S3 backend for the existing MINIO test mode.
const S3_IMAGE = "rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035";
const S3_CLIENT_IMAGE = "rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914";
const S3_BUCKET_CORS = `<CORSConfiguration>
<CORSRule>
<AllowedOrigin>*</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>PUT</AllowedMethod>
<AllowedMethod>POST</AllowedMethod>
<AllowedMethod>DELETE</AllowedMethod>
<AllowedMethod>HEAD</AllowedMethod>
<AllowedHeader>*</AllowedHeader>
<AllowedHeader>authorization</AllowedHeader>
<ExposeHeader>ETag</ExposeHeader>
</CORSRule>
</CORSConfiguration>`;
export async function stopCouchdb(): Promise<void> {
await stopAndRemoveContainer(COUCHDB_CONTAINER);
@@ -454,7 +468,7 @@ export async function updateCouchdbDoc(
}
// ---------------------------------------------------------------------------
// MinIO
// S3 (RustFS)
// ---------------------------------------------------------------------------
function shQuote(value: string): string {
@@ -473,9 +487,10 @@ async function initMinioBucket(
bucket: string
): Promise<boolean> {
const cmd =
`mc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` +
`mc mb --ignore-existing myminio/${shQuote(bucket)} >/dev/null 2>&1`;
const r = await docker("run", "--rm", "--network", "host", "--entrypoint", "/bin/sh", MINIO_MC_IMAGE, "-c", cmd);
`rc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` +
`rc mb --ignore-existing myminio/${shQuote(bucket)} >/dev/null 2>&1 && ` +
`printf %s ${shQuote(S3_BUCKET_CORS)} | rc cors set myminio/${shQuote(bucket)} - >/dev/null 2>&1`;
const r = await docker("run", "--rm", "--network", "host", "--entrypoint", "/bin/sh", S3_CLIENT_IMAGE, "-c", cmd);
return r.code === 0;
}
@@ -487,8 +502,8 @@ async function waitForMinioBucket(
): Promise<void> {
for (let i = 0; i < 30; i++) {
const checkCmd =
`mc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` +
`mc ls myminio/${shQuote(bucket)} >/dev/null 2>&1`;
`rc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` +
`rc ls myminio/${shQuote(bucket)} >/dev/null 2>&1`;
const check = await docker(
"run",
"--rm",
@@ -498,7 +513,7 @@ async function waitForMinioBucket(
"host",
"--entrypoint",
"/bin/sh",
MINIO_MC_IMAGE,
S3_CLIENT_IMAGE,
"-c",
checkCmd
);
@@ -508,7 +523,7 @@ async function waitForMinioBucket(
await initMinioBucket(minioEndpoint, accessKey, secretKey, bucket);
await sleep(2000);
}
throw new Error(`MinIO bucket not ready: ${bucket}`);
throw new Error(`S3 bucket not ready: ${bucket}`);
}
export async function startMinio(
@@ -517,10 +532,10 @@ export async function startMinio(
secretKey: string,
bucket: string
): Promise<void> {
console.log("[INFO] stopping leftover MinIO container if present");
console.log("[INFO] stopping leftover S3 test container if present");
await stopMinio().catch(() => {});
console.log("[INFO] starting MinIO test container");
console.log("[INFO] starting RustFS test container");
await dockerOrFail(
"run",
"-d",
@@ -532,20 +547,19 @@ export async function startMinio(
"-p",
"9001:9001",
"-e",
`MINIO_ROOT_USER=${accessKey}`,
`RUSTFS_ACCESS_KEY=${accessKey}`,
"-e",
`MINIO_ROOT_PASSWORD=${secretKey}`,
`RUSTFS_SECRET_KEY=${secretKey}`,
"-e",
`MINIO_SERVER_URL=${minioEndpoint}`,
MINIO_IMAGE,
"server",
"/data",
"--console-address",
":9001"
"RUSTFS_CONSOLE_ENABLE=true",
"-e",
"RUSTFS_CORS_ALLOWED_ORIGINS=*",
S3_IMAGE,
"/data"
);
trackContainer(MINIO_CONTAINER);
console.log(`[INFO] initialising MinIO test bucket: ${bucket}`);
console.log(`[INFO] initialising S3 test bucket: ${bucket}`);
let initialised = false;
for (let i = 0; i < 5; i++) {
if (await initMinioBucket(minioEndpoint, accessKey, secretKey, bucket)) {
@@ -555,7 +569,7 @@ export async function startMinio(
await sleep(2000);
}
if (!initialised) {
throw new Error(`Could not initialise MinIO bucket after retries: ${bucket}`);
throw new Error(`Could not initialise S3 bucket after retries: ${bucket}`);
}
await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket);
+1
View File
@@ -4,6 +4,7 @@ const TASKS = [
"test:setup-put-cat",
"test:mirror",
"test:daemon",
"test:daemon-startup",
"test:push-pull",
"test:decoupled-vault",
"test:sync-two-local",
@@ -0,0 +1,152 @@
import { assertEquals } from "@std/assert";
import { join } from "@std/path";
import { TempDir } from "./helpers/temp.ts";
import { runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts";
import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts";
import { startCliInBackground, type BackgroundCliProcess } from "./helpers/backgroundCli.ts";
import { startCouchdb, stopCouchdb } from "./helpers/docker.ts";
function envOrDefault(keys: string[], fallback: string): string {
for (const key of keys) {
const value = Deno.env.get(key)?.trim();
if (value) return value;
}
return fallback;
}
function waitForTick(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 100));
}
async function waitForText(filePath: string, expected: string, timeoutMs = 45_000): Promise<void> {
const deadline = Date.now() + timeoutMs;
let actual = "";
while (Date.now() < deadline) {
try {
actual = await Deno.readTextFile(filePath);
if (actual === expected) return;
} catch (error) {
if (!(error instanceof Deno.errors.NotFound)) throw error;
}
await waitForTick();
}
throw new Error(
`Timed out waiting for ${filePath} to contain ${JSON.stringify(expected)}; actual=${JSON.stringify(actual)}`
);
}
async function waitForMissing(filePath: string, timeoutMs = 45_000): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
try {
await Deno.stat(filePath);
} catch (error) {
if (error instanceof Deno.errors.NotFound) return;
throw error;
}
await waitForTick();
}
throw new Error(`Timed out waiting for ${filePath} to be removed`);
}
async function stopDaemon(daemon: BackgroundCliProcess | undefined): Promise<void> {
if (!daemon) return;
await daemon.stop().catch(() => {});
}
Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files", async () => {
await using workDir = await TempDir.create("livesync-cli-daemon-startup");
const couchdbUri = envOrDefault(["COUCHDB_URI", "hostname"], "http://127.0.0.1:5989").replace(/\/$/, "");
const couchdbUser = envOrDefault(["COUCHDB_USER", "username"], "admin");
const couchdbPassword = envOrDefault(["COUCHDB_PASSWORD", "password"], "testpassword");
const dbPrefix = envOrDefault(["COUCHDB_DBNAME", "dbname"], "livesync-test-db-ci");
const dbname = `${dbPrefix}-daemon-startup-${Date.now()}-${Math.floor(Math.random() * 1_000_000)}`.toLowerCase();
const databaseA = workDir.join("database-a");
const databaseB = workDir.join("database-b");
const databaseC = workDir.join("database-c");
const vaultA = workDir.join("vault-a");
const vaultB = workDir.join("vault-b");
const vaultC = workDir.join("vault-c");
const settingsA = workDir.join("settings-a.json");
const settingsB = workDir.join("settings-b.json");
const settingsC = workDir.join("settings-c.json");
await Promise.all([
Deno.mkdir(databaseA, { recursive: true }),
Deno.mkdir(databaseB, { recursive: true }),
Deno.mkdir(databaseC, { recursive: true }),
Deno.mkdir(vaultA, { recursive: true }),
Deno.mkdir(vaultB, { recursive: true }),
Deno.mkdir(vaultC, { recursive: true }),
]);
const startupPath = "notes/present-before-start.md";
const deletePath = "notes/deleted-while-stopped.md";
const remoteOnlyPath = "notes/remote-only.md";
const startupFileA = join(vaultA, startupPath);
const deleteFileA = join(vaultA, deletePath);
const startupFileB = join(vaultB, startupPath);
const deleteFileB = join(vaultB, deletePath);
const remoteOnlyFileB = join(vaultB, remoteOnlyPath);
await Deno.mkdir(join(vaultA, "notes"), { recursive: true });
await Deno.writeTextFile(startupFileA, "created before daemon startup\n");
const initialTime = new Date(Date.now() - 10_000);
await Deno.utime(startupFileA, initialTime, initialTime);
await Deno.writeTextFile(deleteFileA, "delete this after the first run\n");
let daemonA: BackgroundCliProcess | undefined;
let daemonB: BackgroundCliProcess | undefined;
try {
await startCouchdb(couchdbUri, couchdbUser, couchdbPassword, dbname);
for (const settings of [settingsA, settingsB, settingsC]) {
await initSettingsFile(settings);
await applyCouchdbSettings(settings, couchdbUri, couchdbUser, couchdbPassword, dbname, true);
}
// A pre-existing local file must be uploaded by the daemon's startup scan.
daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon");
await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000);
// A separate daemon proves that the first startup replication reached CouchDB
// and that remote files are reflected into its filesystem.
daemonB = startCliInBackground(databaseB, "--vault", vaultB, "--settings", settingsB, "daemon");
await daemonB.waitUntilContains("[Daemon] Initial replication complete", 45_000);
await waitForText(startupFileB, "created before daemon startup\n");
await waitForText(deleteFileB, "delete this after the first run\n");
// Changes made while A is stopped must be found by its next startup scan.
assertEquals(await daemonA.stop(), 0, daemonA.combined);
daemonA = undefined;
await Deno.writeTextFile(startupFileA, "edited while daemon was stopped\n");
await Deno.remove(deleteFileA);
daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon");
await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000);
await waitForText(startupFileB, "edited while daemon was stopped\n");
await waitForMissing(deleteFileB);
// Seed a file into a third local database without creating it in vault C.
// After C's finite sync, it exists only remotely from B's point of view.
await runCliWithInputOrFail(
"created in a different local database\n",
databaseC,
"--vault",
vaultC,
"--settings",
settingsC,
"put",
remoteOnlyPath
);
await runCliOrFail(databaseC, "--vault", vaultC, "--settings", settingsC, "sync");
await waitForText(remoteOnlyFileB, "created in a different local database\n");
assertEquals(await Deno.readTextFile(startupFileB), "edited while daemon was stopped\n");
assertEquals((await Deno.stat(remoteOnlyFileB)).isFile, true);
} finally {
await stopDaemon(daemonB);
await stopDaemon(daemonA);
await stopCouchdb().catch(() => {});
}
});
@@ -60,6 +60,32 @@ export async function runScenario(remoteType: RemoteType, encrypt: boolean): Pro
}
try {
if (remoteType === "MINIO") {
// The shared S3 fixture also serves the browser and real Obsidian tests.
const origin = "app://obsidian.md";
const requestedHeaders = ["authorization", "content-type", "x-amz-date", "x-amz-content-sha256"];
const preflight = await fetch(`${minioEndpoint}/${minioBucket}`, {
method: "OPTIONS",
headers: {
Origin: origin,
"Access-Control-Request-Method": "PUT",
"Access-Control-Request-Headers": requestedHeaders.join(","),
},
});
await preflight.body?.cancel();
assert(preflight.ok, "The S3 fixture must accept browser preflight requests");
const allowedOrigin = preflight.headers.get("access-control-allow-origin");
assert(allowedOrigin === "*" || allowedOrigin === origin, "The S3 fixture must allow the Obsidian origin");
const allowedHeaders = (preflight.headers.get("access-control-allow-headers") ?? "")
.toLowerCase()
.split(",")
.map((header) => header.trim());
assert(allowedHeaders.includes("authorization"), "S3 CORS must explicitly allow the Authorization header");
assert(
preflight.headers.get("access-control-allow-methods")?.split(/,\s*/).includes("PUT"),
"S3 CORS must allow browser uploads"
);
}
await initSettingsFile(settingsA);
await initSettingsFile(settingsB);
await applyRemoteSyncSettings(settingsA, {
+4 -3
View File
@@ -99,11 +99,12 @@ This file corresponds to settings helpers in `test-helpers.sh`.
### `helpers/docker.ts`
- Starts, stops, and initialises CouchDB directly from Deno.
- Starts, stops, and initialises CouchDB and RustFS directly from Deno.
- Configures CouchDB via `fetch + retry`.
- Initialises S3 buckets using the RustFS `rc` client, including CORS for signed browser requests.
- Starts and stops the P2P relay through the same Docker runner.
Both CouchDB and P2P relay flows are bash-independent.
These flows do not require Bash on the host. The S3 matrix tasks, environment variables, and container name retain their existing `minio` names for compatibility; RustFS provides the test backend. The RustFS server and `rc` client images are pinned by version and digest.
### `helpers/backgroundCli.ts`
@@ -328,7 +329,7 @@ The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` runs automati
## Current limitations
- MinIO startup and matrix coverage are ported. Current limits are elsewhere, not setup URI generation.
- S3 startup and matrix coverage use RustFS. Current limits are elsewhere, not setup URI generation.
---
+21 -44
View File
@@ -1,47 +1,24 @@
#!/bin/bash
set -e
cat >/tmp/mybucket-rw.json <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation","s3:ListBucket"],
"Resource": ["arn:aws:s3:::$bucketName"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject","s3:PutObject","s3:DeleteObject"],
"Resource": ["arn:aws:s3:::$bucketName/*"]
}
]
}
EOF
# echo "<CORSConfiguration>
# <CORSRule>
# <AllowedOrigin>http://localhost:63315</AllowedOrigin>
# <AllowedOrigin>http://localhost:63316</AllowedOrigin>
# <AllowedOrigin>http://localhost</AllowedOrigin>
# <AllowedMethod>GET</AllowedMethod>
# <AllowedMethod>PUT</AllowedMethod>
# <AllowedMethod>POST</AllowedMethod>
# <AllowedMethod>DELETE</AllowedMethod>
# <AllowedMethod>HEAD</AllowedMethod>
# <AllowedHeader>*</AllowedHeader>
# </CORSRule>
# </CORSConfiguration>" > /tmp/cors.xml
# docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c "
# mc alias set myminio $minioEndpoint $username $password
# mc mb --ignore-existing myminio/$bucketName
# mc admin policy create myminio my-custom-policy /tmp/mybucket-rw.json
# echo 'Creating service account for user $username with access key $accessKey'
# mc admin user svcacct add --access-key '$accessKey' --secret-key '$secretKey' myminio '$username'
# mc admin policy attach myminio my-custom-policy --user '$accessKey'
# echo 'Verifying policy and user creation:'
# mc admin user svcacct info myminio '$accessKey'
# "
docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c "
mc alias set myminio $minioEndpoint $accessKey $secretKey
mc mb --ignore-existing myminio/$bucketName
"
docker run --rm --network host --entrypoint=/bin/sh \
rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914 \
-c 'set -e
rc alias set myminio "$1" "$2" "$3"
rc mb --ignore-existing "myminio/$4"
rc cors set "myminio/$4" - <<CORS
<CORSConfiguration>
<CORSRule>
<AllowedOrigin>*</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>PUT</AllowedMethod>
<AllowedMethod>POST</AllowedMethod>
<AllowedMethod>DELETE</AllowedMethod>
<AllowedMethod>HEAD</AllowedMethod>
<AllowedHeader>*</AllowedHeader>
<AllowedHeader>authorization</AllowedHeader>
<ExposeHeader>ETag</ExposeHeader>
</CORSRule>
</CORSConfiguration>
CORS
' sh "$minioEndpoint" "$accessKey" "$secretKey" "$bucketName"
+7 -1
View File
@@ -1,2 +1,8 @@
#!/bin/bash
docker run -d --name minio-test -p 9000:9000 -p 9001:9001 -e MINIO_ROOT_USER=$accessKey -e MINIO_ROOT_PASSWORD=$secretKey -e MINIO_SERVER_URL=$minioEndpoint minio/minio server /data --console-address ':9001'
docker run -d --name minio-test \
-p 9000:9000 -p 9001:9001 \
-e "RUSTFS_ACCESS_KEY=$accessKey" \
-e "RUSTFS_SECRET_KEY=$secretKey" \
-e "RUSTFS_CONSOLE_ENABLE=true" \
-e 'RUSTFS_CORS_ALLOWED_ORIGINS=*' \
rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035 /data
+1 -1
View File
@@ -1,3 +1,3 @@
#!/bin/bash
docker stop minio-test
docker rm minio-test
docker rm -v minio-test
+7 -1
View File
@@ -45,7 +45,13 @@ export class ModuleLiveSyncMain extends AbstractModule {
}
// Ordinary start-up may continue when individual files could not be
// processed. Explicit Fetch and Rebuild flows retain the strict default.
const initialisationResult = await this.services.databaseEvents.initialiseDatabase(false, false, false, true);
const { ignoreSuspending = false, continueOnFileFailure = true } = this.core.startupDatabaseOptions;
const initialisationResult = await this.services.databaseEvents.initialiseDatabase(
false,
false,
ignoreSuspending,
continueOnFileFailure
);
if (initialisationResult === VaultScanResults.FAILED) {
this._log($msg("Ui.Common.LocalDatabaseInitialisationFailed"), LOG_LEVEL_NOTICE);
//TODO:stop all sync.
@@ -25,6 +25,7 @@ describe("ModuleLiveSyncMain", () => {
const log = vi.fn();
const host = {
core: {
startupDatabaseOptions: {},
services: {
appLifecycle: {
onLayoutReady: vi.fn(async () => true),
@@ -58,6 +59,7 @@ describe("ModuleLiveSyncMain", () => {
};
const host = {
core: {
startupDatabaseOptions: {},
services: { appLifecycle },
},
services: {
@@ -77,4 +79,33 @@ describe("ModuleLiveSyncMain", () => {
expect(result).toBe(true);
expect(log).toHaveBeenCalledWith("Ui.Common.SomeFilesCouldNotBeSynchronised", LOG_LEVEL_NOTICE);
});
it("passes strict startup database options to initialisation", async () => {
const initialiseDatabase = vi.fn(async () => false);
const host = {
core: {
startupDatabaseOptions: {
ignoreSuspending: true,
continueOnFileFailure: false,
},
services: {
appLifecycle: {
onLayoutReady: vi.fn(async () => true),
},
},
},
services: {
databaseEvents: { initialiseDatabase },
},
settings: {
suspendFileWatching: false,
suspendParseReplicationResult: false,
},
_log: vi.fn(),
};
await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never);
expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false);
});
});
+2 -2
View File
@@ -125,7 +125,7 @@ The mobile pass uses Obsidian's `app.emulateMobile(true)`, a 390 by 844 CSS-pixe
`test:e2e:obsidian:p2p-pane` starts one configured CouchDB-only session with no P2P profile and separate configured P2P sessions for desktop and mobile. It proves that the command remains registered while the retired command, automatic pane, and ribbon entry without a P2P configuration are absent. For the configured P2P profiles, it verifies that the desktop ribbon is available, the current status command reaches the pane without it opening at start-up, checks its connection control and horizontal layout, and captures unobstructed desktop and mobile screenshots. The mobile session uses a fresh Vault, profile, and Obsidian process, enters `app.emulateMobile(true)` through `lifecycle.beforePluginStart`, and requires the P2P view to belong to the right drawer rather than inheriting desktop workspace state. It deliberately uses no relay or peer: replacement of the active replicator is covered by focused unit tests, the Deno and Compose CLI P2P lifecycle suite covers the headless transport, and `p2p-setup-uri-workflow` owns the visible transfer path between two real Obsidian sessions.
`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Object Storage upload and Setup URI round-trip, P2P Setup URI round-trip, startup scan, provisioned CouchDB Setup URI, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB, MinIO, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run.
`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Object Storage upload and Setup URI round-trip, P2P Setup URI round-trip, startup scan, provisioned CouchDB Setup URI, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB, RustFS, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run.
`test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, starts from configured plug-in data without the device-local compatibility marker, and verifies the copied-or-restored Vault explanation in the actual compatibility dialogue. It captures the summary and details, resumes explicitly, confirms that the marker was recorded, creates a note in real Obsidian, commits the note into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the metadata document and its chunk documents.
@@ -160,7 +160,7 @@ LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) --
`test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service and starts with isolated Object Storage settings and the device-local compatibility acknowledgement already in place, keeping the scenario focused on upload rather than unconfigured start-up or setup. It confirms those settings through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. Adapter tests separately observe an in-progress SDK command, while this real-runtime workflow verifies the resulting request counters advance and rebalance.
`test:e2e:obsidian:object-storage-setup-uri-workflow` uses the public Commonlib-backed tool to generate the initial Setup URI for a unique MinIO prefix, completes visible initialisation on the first device, and then asks that working real Obsidian device to create a new Setup URI through the registered command. A second real Obsidian device imports only the device-generated URI. The workflow verifies the A-to-B note through explicit replication, then verifies that the B-to-A note arrives through `syncOnStart` after restarting the first device, without requesting manual replication. It captures the documented onboarding choices, and removes the Object Storage prefix only after both sessions have stopped.
`test:e2e:obsidian:object-storage-setup-uri-workflow` uses the public Commonlib-backed tool to generate the initial Setup URI for a unique Object Storage prefix, completes visible initialisation on the first device, and then asks that working real Obsidian device to create a new Setup URI through the registered command. A second real Obsidian device imports only the device-generated URI. The workflow verifies the A-to-B note through explicit replication, then verifies that the B-to-A note arrives through `syncOnStart` after restarting the first device, without requesting manual replication. It captures the documented onboarding choices, and removes the Object Storage prefix only after both sessions have stopped.
`test:e2e:obsidian:p2p-setup-uri-workflow` runs two concurrent isolated real Obsidian sessions against the local Compose Nostr relay fixture. The first device imports a generated initial Setup URI and completes its signalling test with zero peers, creates a Setup URI for the second device through the registered command, and remains online while the second device imports it. The second device must select the expected online source before Fetch can rebuild its local database. The workflow accepts each connection request visibly on the receiving device, verifies the initial A-to-B fetch, checks that the menu for the three persistent per-peer actions remains within the viewport, reconnects both P2P sessions in join order, and verifies the B-to-A return journey. Every started session remains tracked until teardown completes.
+21 -44
View File
@@ -1,47 +1,24 @@
#!/bin/bash
set -e
cat >/tmp/mybucket-rw.json <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation","s3:ListBucket"],
"Resource": ["arn:aws:s3:::$bucketName"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject","s3:PutObject","s3:DeleteObject"],
"Resource": ["arn:aws:s3:::$bucketName/*"]
}
]
}
EOF
# echo "<CORSConfiguration>
# <CORSRule>
# <AllowedOrigin>http://localhost:63315</AllowedOrigin>
# <AllowedOrigin>http://localhost:63316</AllowedOrigin>
# <AllowedOrigin>http://localhost</AllowedOrigin>
# <AllowedMethod>GET</AllowedMethod>
# <AllowedMethod>PUT</AllowedMethod>
# <AllowedMethod>POST</AllowedMethod>
# <AllowedMethod>DELETE</AllowedMethod>
# <AllowedMethod>HEAD</AllowedMethod>
# <AllowedHeader>*</AllowedHeader>
# </CORSRule>
# </CORSConfiguration>" > /tmp/cors.xml
# docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c "
# mc alias set myminio $minioEndpoint $username $password
# mc mb --ignore-existing myminio/$bucketName
# mc admin policy create myminio my-custom-policy /tmp/mybucket-rw.json
# echo 'Creating service account for user $username with access key $accessKey'
# mc admin user svcacct add --access-key '$accessKey' --secret-key '$secretKey' myminio '$username'
# mc admin policy attach myminio my-custom-policy --user '$accessKey'
# echo 'Verifying policy and user creation:'
# mc admin user svcacct info myminio '$accessKey'
# "
docker run --rm --network host -v /tmp/mybucket-rw.json:/tmp/mybucket-rw.json --entrypoint=/bin/sh minio/mc -c "
mc alias set myminio $minioEndpoint $accessKey $secretKey
mc mb --ignore-existing myminio/$bucketName
"
docker run --rm --network host --entrypoint=/bin/sh \
rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914 \
-c 'set -e
rc alias set myminio "$1" "$2" "$3"
rc mb --ignore-existing "myminio/$4"
rc cors set "myminio/$4" - <<CORS
<CORSConfiguration>
<CORSRule>
<AllowedOrigin>*</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>PUT</AllowedMethod>
<AllowedMethod>POST</AllowedMethod>
<AllowedMethod>DELETE</AllowedMethod>
<AllowedMethod>HEAD</AllowedMethod>
<AllowedHeader>*</AllowedHeader>
<AllowedHeader>authorization</AllowedHeader>
<ExposeHeader>ETag</ExposeHeader>
</CORSRule>
</CORSConfiguration>
CORS
' sh "$minioEndpoint" "$accessKey" "$secretKey" "$bucketName"
+7 -1
View File
@@ -1,2 +1,8 @@
#!/bin/bash
docker run -d --name minio-test -p 9000:9000 -p 9001:9001 -e MINIO_ROOT_USER=$accessKey -e MINIO_ROOT_PASSWORD=$secretKey -e MINIO_SERVER_URL=$minioEndpoint minio/minio server /data --console-address ':9001'
docker run -d --name minio-test \
-p 9000:9000 -p 9001:9001 \
-e "RUSTFS_ACCESS_KEY=$accessKey" \
-e "RUSTFS_SECRET_KEY=$secretKey" \
-e "RUSTFS_CONSOLE_ENABLE=true" \
-e 'RUSTFS_CORS_ALLOWED_ORIGINS=*' \
rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035 /data
+1 -1
View File
@@ -1,3 +1,3 @@
#!/bin/bash
docker stop minio-test
docker rm minio-test
docker rm -v minio-test
+5
View File
@@ -12,6 +12,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
## Unreleased
### Fixed
- CLI: daemon and mirror now scan the Vault during database initialisation, following the Obsidian startup sequence. The daemon completes this scan before replication; mirror runs the scan once and still exits with an error if any file cannot be processed.
- CLI: file enumeration now includes current files even after individual path lookups or earlier scans.
## 1.0.28
9th September, 2026