Verify NFC scan failure preservation and recovery

src/apps/cli/adapters/NodeFileSystemAdapter.integration.spec.ts

@@ -0,0 +1,180 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { NodeStorageAdapter, fsPromises, os, path } from "@vrtmrz/livesync-commonlib/node";
+import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
+import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
+import { createDefaultCliSettings } from "@/apps/cli/cliSettingsDefaults";
+import { runCommand } from "@/apps/cli/commands/runCommand";
+import type { CLICommand, CLICommandContext } from "@/apps/cli/commands/types";
+import { main, type CliCommandRunner } from "@/apps/cli/main";
+
+type StoredFileSnapshot = {
+    id: string;
+    revision: string;
+    path: string;
+    size: number;
+    children: string[];
+    content: string;
+};
+
+describe("CLI NFC scan failure recovery", () => {
+    const originalArgv = process.argv.slice();
+    const originalExitCode = process.exitCode;
+    const signalHandlers = new Map<"SIGINT" | "SIGTERM", Set<Parameters<typeof process.removeListener>[1]>>();
+    let directory: string;
+    let vaultPath: string;
+    let settingsPath: string;
+    let standardIo: StandardIo;
+
+    afterEach(async () => {
+        for (const [signal, originalHandlers] of signalHandlers) {
+            for (const handler of process.listeners(signal)) {
+                if (!originalHandlers.has(handler)) process.removeListener(signal, handler);
+            }
+        }
+        signalHandlers.clear();
+        process.argv = originalArgv.slice();
+        process.exitCode = originalExitCode;
+        vi.restoreAllMocks();
+        if (directory) await fsPromises.rm(directory, { recursive: true, force: true });
+    });
+
+    async function start(command: CLICommand, runner: CliCommandRunner, exitCode: number) {
+        process.exitCode = undefined;
+        process.argv = [
+            "node",
+            "livesync-cli",
+            path.join(directory, "db"),
+            "--vault",
+            vaultPath,
+            "--settings",
+            settingsPath,
+            command,
+        ];
+        await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`);
+    }
+
+    async function snapshotFile(context: CLICommandContext, targetPath: string): Promise<StoredFileSnapshot> {
+        for await (const doc of context.core.services.database.localDatabase.findAllNormalDocs({ conflicts: true }) || []) {
+            if (stripAllPrefixes(doc.path) !== targetPath) continue;
+            const file = await context.core.serviceModules.databaseFileAccess.fetch(targetPath as FilePathWithPrefix);
+            if (!file || file.deleted) throw new Error(`Expected readable database content for ${targetPath}`);
+            const body = file.body;
+            if (body.type !== "text/plain") throw new Error(`Expected text database content for ${targetPath}`);
+            return {
+                id: doc._id,
+                revision: doc._rev ?? "",
+                path: doc.path,
+                size: doc.size,
+                children: [...doc.children],
+                content: await body.text(),
+            };
+        }
+        throw new Error(`Expected a stored Metadata document for ${targetPath}`);
+    }
+
+    it("does not accept a partially enumerated Vault as complete and keeps its PouchDB Metadata", async () => {
+        directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-cli-nfc-scan-integration-"));
+        vaultPath = path.join(directory, "vault");
+        settingsPath = path.join(directory, "settings.json");
+        const databasePath = path.join(directory, "db");
+        const knownPath = "notes/known.md";
+        const blockedPath = "notes/blocked/new.md";
+        const knownContent = "stored before the failed inventory\n";
+        const blockedContent = "available after recovery\n";
+
+        await fsPromises.mkdir(path.join(vaultPath, "notes"), { recursive: true });
+        await fsPromises.mkdir(databasePath);
+        await fsPromises.writeFile(path.join(vaultPath, knownPath), knownContent);
+        await fsPromises.writeFile(
+            settingsPath,
+            JSON.stringify({
+                ...createDefaultCliSettings(),
+                isConfigured: true,
+                pathNormalizationVersion: 1,
+            })
+        );
+        standardIo = {
+            readStdin: vi.fn(async () => ""),
+            prompt: vi.fn(async () => ""),
+            writeStdout: vi.fn(),
+            writeStderr: vi.fn(),
+        };
+        for (const signal of ["SIGINT", "SIGTERM"] as const) {
+            signalHandlers.set(signal, new Set(process.listeners(signal)));
+        }
+        vi.spyOn(process, "exit").mockImplementation((code) => {
+            throw new Error(`__EXIT__:${code ?? 0}`);
+        });
+
+        let seeded: StoredFileSnapshot | undefined;
+        const seedMirror = vi.fn<CliCommandRunner>(async (options, context) => {
+            const result = await runCommand(options, context);
+            seeded = await snapshotFile(context, knownPath);
+            return result;
+        });
+        await start("mirror", seedMirror, 0);
+        expect(seedMirror).toHaveBeenCalledOnce();
+        expect(seeded).toMatchObject({ path: knownPath, content: knownContent });
+
+        await fsPromises.mkdir(path.join(vaultPath, "notes", "blocked"));
+        await fsPromises.writeFile(path.join(vaultPath, blockedPath), blockedContent);
+
+        const scanFailure = Object.assign(new Error("Storage permission denied"), { code: "EACCES" });
+        const directoryReads: string[] = [];
+        const parentDirectoryEntries: string[] = [];
+        // Retain the prototype implementation before the spy replaces it; invoke it with each real adapter below.
+        const originalReadDirectory = Reflect.get(
+            NodeStorageAdapter.prototype,
+            "readDirectory"
+        );
+        const readDirectory = vi.spyOn(NodeStorageAdapter.prototype, "readDirectory").mockImplementation(
+            async function (this: NodeStorageAdapter, relativePath) {
+                directoryReads.push(relativePath);
+                if (relativePath === "notes/blocked") throw scanFailure;
+                const entries = await originalReadDirectory.call(this, relativePath);
+                if (relativePath === "notes") {
+                    parentDirectoryEntries.push(...(entries ?? []).map((entry) => entry.name));
+                }
+                return entries;
+            }
+        );
+        const failedMirror = vi.fn<CliCommandRunner>(async (options, context) => await runCommand(options, context));
+        try {
+            await start("mirror", failedMirror, 1);
+        } finally {
+            readDirectory.mockRestore();
+        }
+
+        expect(failedMirror).not.toHaveBeenCalled();
+        expect(directoryReads).toContain("");
+        expect(directoryReads).toContain("notes");
+        expect(directoryReads).toContain("notes/blocked");
+        expect(parentDirectoryEntries).toContain("known.md");
+        expect(parentDirectoryEntries).toContain("blocked");
+
+        let afterFailure: StoredFileSnapshot | undefined;
+        const inspectAfterFailure = vi.fn<CliCommandRunner>(async (options, context) => {
+            const result = await runCommand(options, context);
+            afterFailure = await snapshotFile(context, knownPath);
+            return result;
+        });
+        await start("ls", inspectAfterFailure, 0);
+        expect(afterFailure).toEqual(seeded);
+        expect(await fsPromises.readFile(path.join(vaultPath, knownPath), "utf8")).toBe(knownContent);
+        expect(await fsPromises.readFile(path.join(vaultPath, blockedPath), "utf8")).toBe(blockedContent);
+
+        let recoveredKnown: StoredFileSnapshot | undefined;
+        let recoveredNewFile: StoredFileSnapshot | undefined;
+        const recoverMirror = vi.fn<CliCommandRunner>(async (options, context) => {
+            const result = await runCommand(options, context);
+            recoveredKnown = await snapshotFile(context, knownPath);
+            recoveredNewFile = await snapshotFile(context, blockedPath);
+            return result;
+        });
+        await start("mirror", recoverMirror, 0);
+        expect(recoverMirror).toHaveBeenCalledOnce();
+        expect(recoveredKnown).toEqual(seeded);
+        expect(recoveredNewFile).toMatchObject({ path: blockedPath, content: blockedContent });
+    });
+});

src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts

@@ -268,6 +268,49 @@ describe("NodeFileSystemAdapter Unicode path resolution", () => {
         expect(report).not.toHaveBeenCalled();
     });
 
+    it("does not return or cache a partial NFC listing after a nested directory read fails", async () => {
+        const directory = await createVault();
+        const knownPath = "Known.md";
+        const initialContent = "initial content";
+        await fs.writeFile(path.join(directory, knownPath), initialContent);
+        let normalizationVersion = 1;
+        const adapter = new NodeFileSystemAdapter(directory, undefined, () => normalizationVersion, () => true);
+        await adapter.scanDirectory();
+
+        const changedContent = "changed content after cache";
+        await fs.writeFile(path.join(directory, knownPath), changedContent);
+        await fs.mkdir(path.join(directory, "late-directory"));
+        await fs.writeFile(path.join(directory, "late-directory", "later.md"), "later content");
+
+        const error = Object.assign(new Error("Storage permission denied"), { code: "EACCES" });
+        const readDirectory = adapter.storage.readDirectory.bind(adapter.storage);
+        const readDirectorySpy = vi.spyOn(adapter.storage, "readDirectory").mockImplementation(async (relativePath) => {
+            if (relativePath === "late-directory") throw error;
+            const entries = await readDirectory(relativePath);
+            return relativePath === "" && entries !== null
+                ? [...entries].sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0))
+                : entries;
+        });
+
+        try {
+            await expect(adapter.getFiles()).rejects.toBe(error);
+            await expect(adapter.scanDirectory()).rejects.toBe(error);
+        } finally {
+            readDirectorySpy.mockRestore();
+        }
+
+        // Compatibility-mode case-insensitive lookup exposes the cache without
+        // triggering another scan, so it can verify that the failed NFC scan did
+        // not publish its already-enumerated file metadata.
+        normalizationVersion = 0;
+        const scanDirectory = vi.spyOn(adapter, "scanDirectory");
+        await expect(adapter.getAbstractFileByPathInsensitive("known.md")).resolves.toMatchObject({
+            path: knownPath,
+            stat: { size: initialContent.length },
+        });
+        expect(scanDirectory).not.toHaveBeenCalled();
+    });
+
     it("uses one comparison policy snapshot for the whole NFC scan", async () => {
         const directory = await createVault();
         await fs.mkdir(path.join(directory, "Folder"));

test/e2e-cli/test-mirror.ts

@@ -21,10 +21,10 @@
 
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { copyFile, mkdir, readFile, readdir, stat, utimes, writeFile } from "node:fs/promises";
+import { copyFile, mkdir, readFile, readdir, rm, stat, utimes, writeFile } from "node:fs/promises";
 
 import { TempDir } from "./helpers/temp.ts";
-import { runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts";
+import { runCli, runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts";
 import { initSettingsFile, markSettingsConfigured } from "./helpers/settings.ts";
 
 test("mirror: storage <-> DB synchronisation", async (t) => {
@@ -286,3 +286,62 @@ test("mirror: NFC identity retains physical Unicode spelling", async () => {
     assert.deepEqual(await readdir(vault), [parent]);
     assert.deepEqual(await readdir(workDir.join("vault", parent)), [leaf]);
 });
+
+test("mirror: rejects an ambiguous NFC inventory without changing prior data and recovers", async (t) => {
+    await using workDir = await TempDir.create("livesync-cli-nfc-scan-failure");
+    const vault = workDir.join("vault");
+    const database = workDir.join("db");
+    const settings = workDir.join("settings.json");
+    const existingContent = "retained before failed scan\n";
+    const existingFile = workDir.join("vault", "known.md");
+    const nfcParent = "Café";
+    const nfdParent = nfcParent.normalize("NFD");
+    const nfcFile = workDir.join("vault", nfcParent, "from-nfc.md");
+    const nfdFile = workDir.join("vault", nfdParent, "from-nfd.md");
+
+    await mkdir(vault, { recursive: true });
+    await mkdir(database);
+    await initSettingsFile(settings);
+    await markSettingsConfigured(settings);
+    const data = JSON.parse(await readFile(settings, "utf8"));
+    data.pathNormalizationVersion = 1;
+    await writeFile(settings, JSON.stringify(data));
+    await writeFile(existingFile, existingContent);
+
+    const runMirror = () => runCliOrFail(database, "--settings", settings, "mirror", vault);
+    const runDatabase = (...args: string[]) => runCliOrFail(database, "--settings", settings, ...args);
+
+    await runMirror();
+    const before = JSON.parse(await runDatabase("info", "known.md"));
+    assert.equal(await runDatabase("cat-rev", "known.md", before.revision), existingContent);
+
+    await mkdir(workDir.join("vault", nfcParent), { recursive: true });
+    await mkdir(workDir.join("vault", nfdParent), { recursive: true });
+    await writeFile(nfcFile, "NFC spelling content\n");
+    await writeFile(nfdFile, "NFD spelling content\n");
+    const inventoryNames = await readdir(vault);
+    if (!inventoryNames.includes(nfcParent) || !inventoryNames.includes(nfdParent)) {
+        t.skip("This filesystem cannot represent both NFC and NFD directory entries separately");
+        return;
+    }
+
+    const failedMirror = await runCli(database, "--settings", settings, "mirror", vault);
+    assert.notEqual(failedMirror.code, 0, "An ambiguous inventory must fail the mirror command");
+    assert(
+        !failedMirror.combined.includes("[Done] Command 'mirror' completed"),
+        "A failed inventory must not claim successful mirror completion"
+    );
+    assert.equal(await readFile(existingFile, "utf8"), existingContent);
+    assert.equal(await readFile(nfcFile, "utf8"), "NFC spelling content\n");
+    assert.equal(await readFile(nfdFile, "utf8"), "NFD spelling content\n");
+    assert.deepEqual(JSON.parse(await runDatabase("info", "known.md")), before);
+    assert.equal(await runDatabase("cat-rev", "known.md", before.revision), existingContent);
+
+    await rm(workDir.join("vault", nfdParent), { recursive: true });
+    const recoveredMirror = await runMirror();
+    assert(recoveredMirror.includes("[Done] Command 'mirror' completed"));
+    const recovered = JSON.parse(await runDatabase("info", `${nfcParent}/from-nfc.md`));
+    assert.equal(await runDatabase("cat-rev", `${nfcParent}/from-nfc.md`, recovered.revision), "NFC spelling content\n");
+    assert.equal(await readFile(existingFile, "utf8"), existingContent);
+    assert.deepEqual(JSON.parse(await runDatabase("info", "known.md")), before);
+});

vitest.config.unit.ts

@@ -31,6 +31,7 @@ export default mergeConfig(
                     "**/*.test.ts",
                     "**/*unit.test.ts",
                     "**/*.unit.spec.ts",
+                    "**/*.integration.spec.ts",
                     "test/**",
                     "**/_*",
                     "**/*_obsolete.ts",