fix(cli): enumerate current Vault files independently of cache

src/apps/cli/adapters/NodeFileSystemAdapter.ts

@@ -92,10 +92,9 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
     }
 
     async getFiles(): Promise<NodeFile[]> {
-        if (this.fileCache.size === 0) {
-            await this.scanDirectory();
-        }
-        return Array.from(this.fileCache.values());
+        const files = new Map<string, NodeFile>();
+        await this.scanDirectoryInto("", files);
+        return Array.from(files.values());
     }
 
     async renameFile(file: NodeFile, newPath: string): Promise<NodeFile> {
@@ -147,6 +146,10 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
      * Helper method to recursively scan directory and populate file cache
      */
     async scanDirectory(relativePath: string = ""): Promise<void> {
+        await this.scanDirectoryInto(relativePath, this.fileCache);
+    }
+
+    private async scanDirectoryInto(relativePath: string, files: Map<string, NodeFile>): Promise<void> {
         const fullPath = this.resolvePath(relativePath);
         try {
             const directoryStat = await this.storage.stat(relativePath);
@@ -160,10 +163,10 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
                     path: entryPath as FilePath,
                     stat,
                 };
-                this.fileCache.set(entryPath, file);
+                files.set(entryPath, file);
             }
             for (const entryPath of entries.folders) {
-                await this.scanDirectory(entryPath);
+                await this.scanDirectoryInto(entryPath, files);
             }
         } catch (error) {
             // Directory doesn't exist or is not readable

src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts

@@ -0,0 +1,118 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node";
+import { NodeFileSystemAdapter } from "./NodeFileSystemAdapter";
+
+describe("NodeFileSystemAdapter file enumeration", () => {
+    const tempDirs: string[] = [];
+    const paths = ["a.md", "folder/b.md", "folder/sub/c.md"];
+
+    async function createVault() {
+        const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-enumeration-"));
+        tempDirs.push(directory);
+        for (const file of paths) {
+            await fs.mkdir(path.dirname(path.join(directory, file)), { recursive: true });
+            await fs.writeFile(path.join(directory, file), `content of ${file}`);
+        }
+        return { directory, adapter: new NodeFileSystemAdapter(directory) };
+    }
+
+    afterEach(async () => {
+        await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })));
+    });
+
+    it("lists every file when one file was refreshed before the first enumeration", async () => {
+        const { adapter } = await createVault();
+
+        expect(await adapter.refreshFile("folder/b.md")).not.toBeNull();
+
+        expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
+    });
+
+    it("lists every file after a path lookup without any replication", async () => {
+        const { adapter } = await createVault();
+
+        expect((await adapter.getAbstractFileByPath("folder/b.md"))?.path).toBe("folder/b.md");
+
+        expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
+    });
+
+    it("lists every file on the first enumeration without a prior path lookup", async () => {
+        const { adapter } = await createVault();
+
+        expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
+    });
+
+    it("excludes a deleted file after its cache entry is refreshed", async () => {
+        const { directory, adapter } = await createVault();
+        await adapter.getFiles();
+
+        await fs.rm(path.join(directory, "folder/b.md"));
+        expect(await adapter.refreshFile("folder/b.md")).toBeNull();
+
+        expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(["a.md", "folder/sub/c.md"]);
+    });
+
+    it("reflects files added and deleted between enumerations", async () => {
+        const { directory, adapter } = await createVault();
+
+        expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(paths);
+
+        await fs.rm(path.join(directory, "folder/b.md"));
+        const updatedContent = "updated content of a.md";
+        await fs.writeFile(path.join(directory, "a.md"), updatedContent);
+        await fs.writeFile(path.join(directory, "later.md"), "content of later.md");
+
+        const files = await adapter.getFiles();
+        expect(files.map((file) => file.path).sort()).toEqual(["a.md", "folder/sub/c.md", "later.md"]);
+        expect(files.find((file) => file.path === "a.md")?.stat.size).toBe(updatedContent.length);
+    });
+
+    it("returns complete listings from simultaneous calls", async () => {
+        const { adapter } = await createVault();
+
+        const originalStat = adapter.storage.stat.bind(adapter.storage);
+        let releaseFolderStat!: () => void;
+        const folderStatReleased = new Promise<void>((resolve) => {
+            releaseFolderStat = resolve;
+        });
+        let folderStatStarted!: () => void;
+        const folderStatStartedPromise = new Promise<void>((resolve) => {
+            folderStatStarted = resolve;
+        });
+        let pauseFolderStat = true;
+        const statSpy = vi.spyOn(adapter.storage, "stat").mockImplementation(async (relativePath) => {
+            const stat = await originalStat(relativePath);
+            if (pauseFolderStat && relativePath === "folder") {
+                pauseFolderStat = false;
+                folderStatStarted();
+                await folderStatReleased;
+            }
+            return stat;
+        });
+
+        const firstListing = adapter.getFiles();
+        let listings: Awaited<ReturnType<typeof adapter.getFiles>>[] | undefined;
+        try {
+            await folderStatStartedPromise;
+            const secondListing = adapter.getFiles();
+            const secondFiles = await secondListing;
+            releaseFolderStat();
+            const firstFiles = await firstListing;
+            listings = [secondFiles, firstFiles];
+        } finally {
+            releaseFolderStat();
+            statSpy.mockRestore();
+        }
+
+        if (!listings) throw new Error("Expected both concurrent listings to complete");
+        expect(listings.map((files) => files.map((file) => file.path).sort())).toEqual([paths, paths]);
+    });
+
+    it("returns an empty listing for an empty vault", async () => {
+        const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-enumeration-empty-"));
+        tempDirs.push(directory);
+        const adapter = new NodeFileSystemAdapter(directory);
+
+        await expect(adapter.getFiles()).resolves.toEqual([]);
+    });
+});

src/apps/cli/package.json

@@ -12,7 +12,7 @@
         "buildRun": "npm run build && npm run cli --",
         "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..",
         "check": "tsc -p tsconfig.json",
-        "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts",
+        "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
         "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh",
         "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh",
         "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh",