From 16a61e3f1c1c555a732256ad0a1b4b9848ae2f3e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 7 Nov 2026 11:37:10 +0000 Subject: [PATCH] Fix CLI detection of files in newly created directories --- package-lock.json | 35 ++- src/apps/cli/README.md | 4 + ...ventManagerAdapter.real-watch.unit.spec.ts | 211 ++++++++++++++++++ src/apps/cli/package.json | 7 +- updates.md | 7 + 5 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts diff --git a/package-lock.json b/package-lock.json index e6be3386..277973d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4742,6 +4742,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, "license": "MIT", "dependencies": { "readdirp": "^4.0.1" @@ -9576,6 +9577,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 14.18.0" @@ -11663,7 +11665,7 @@ "version": "1.0.46-cli", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", - "chokidar": "^4.0.0", + "chokidar": "6.0.0", "minimatch": "^11.0.3", "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", @@ -11682,6 +11684,37 @@ "typescript": "6.0.3", "vite": "^8.1.6", "vitest": "^4.1.12" + }, + "engines": { + "node": ">=22.22" + } + }, + "src/apps/cli/node_modules/chokidar": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-6.0.0.tgz", + "integrity": "sha512-Ec2Z7wpcKBiMpR7DxteoqS/sq5G5d2A4fpxzPv6+fD7xNNdrTghn7L11YwV5EN1B3i72Erc3XVT5FkWIDx90+A==", + "license": "MIT", + "dependencies": { + "readdirp": "^5.1.1" + }, + "engines": { + "node": ">= 22.22" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "src/apps/cli/node_modules/readdirp": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.1.1.tgz", + "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" } }, "src/apps/webapp": { diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 07aa5794..de254214 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -1,6 +1,10 @@ # Self-hosted LiveSync CLI Command-line version of Self-hosted LiveSync plug-in for syncing vaults without Obsidian. +## Requirements + +The CLI requires Node.js 22.22 or later. The Docker image includes its own Node.js runtime. + ## Features - ✅ Sync Obsidian vaults using CouchDB without running Obsidian diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts new file mode 100644 index 00000000..a5513585 --- /dev/null +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts @@ -0,0 +1,211 @@ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readdir, rename, rm, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { setTimeout as delay } from "node:timers/promises"; +import { describe, expect, it } from "vitest"; +import type { IStorageEventWatchHandlers } from "@vrtmrz/livesync-commonlib/compat/managers/adapters"; +import { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules"; +import { CLIStorageEventManagerAdapter } from "./CLIStorageEventManagerAdapter"; +import { NodeTypeGuardAdapter } from "@/apps/cli/adapters/NodeTypeGuardAdapter"; + +const run = promisify(execFile); + +type WatchEvent = { + type: "create" | "change" | "delete"; + path: string; + folder: boolean; +}; + +function makeHandlers(events: WatchEvent[], onRename: () => void = () => undefined): IStorageEventWatchHandlers { + const typeGuard = new NodeTypeGuardAdapter(); + const record = (type: WatchEvent["type"], file: unknown) => { + if (!typeGuard.isFile(file) && !typeGuard.isFolder(file)) { + throw new TypeError("Expected a CLI file or folder event"); + } + events.push({ type, path: file.path, folder: typeGuard.isFolder(file) }); + }; + return { + onCreate: (file) => record("create", file), + onChange: (file) => record("change", file), + onDelete: (file) => record("delete", file), + onRename, + onRaw: () => undefined, + }; +} + +async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) return false; + await delay(25); + } + return true; +} + +const directoryFirstWriter = ` +const fs = require("node:fs"); +const path = require("node:path"); +const root = process.argv[1]; + +for (let directoryIndex = 0; directoryIndex < 40; directoryIndex++) { + fs.mkdirSync(path.join(root, "book-" + directoryIndex)); +} + +for (let directoryIndex = 0; directoryIndex < 40; directoryIndex++) { + const directory = path.join(root, "book-" + directoryIndex); + for (let fileIndex = 0; fileIndex < 10; fileIndex++) { + const name = "image-" + fileIndex + ".png"; + const temporary = path.join(directory, "." + name + ".tmp"); + const target = path.join(directory, name); + fs.writeFileSync(temporary, "payload:" + directoryIndex + ":" + fileIndex); + fs.renameSync(temporary, target); + } +} +`; + +describe("CLI filesystem watcher with a real filesystem", () => { + it("reports all 400 files when another process creates directories before dot-temp renames", async () => { + const root = await mkdtemp(join(tmpdir(), "livesync-issue1328-burst-")); + const events: WatchEvent[] = []; + const adapter = new CLIStorageEventManagerAdapter(root, undefined, true); + const expectedPaths = Array.from({ length: 40 }, (_, directoryIndex) => + Array.from({ length: 10 }, (_, fileIndex) => `book-${directoryIndex}/image-${fileIndex}.png`) + ).flat(); + + try { + await writeFile(join(root, "present-before-watch.md"), "initial file"); + await adapter.watch.beginWatch(makeHandlers(events)); + + await run(process.execPath, ["-e", directoryFirstWriter, root], { timeout: 15_000 }); + + const diskPaths = (await readdir(root, { recursive: true })).filter((path) => path.endsWith(".png")); + expect(diskPaths).toHaveLength(400); + + await waitFor( + () => + expectedPaths.every((path) => + events.some((event) => event.type === "create" && !event.folder && event.path === path) + ), + 12_000 + ); + + const createdPaths = new Set( + events + .filter((event) => event.type === "create" && !event.folder && event.path.startsWith("book-")) + .map((event) => event.path) + ); + const missingPaths = expectedPaths.filter((path) => !createdPaths.has(path)); + + expect(events.some((event) => event.path === "present-before-watch.md")).toBe(false); + expect( + missingPaths.length, + `Expected create events for all 400 files; received ${createdPaths.size}, missing sample: ${missingPaths + .slice(0, 8) + .join(", ")}` + ).toBe(0); + } finally { + await adapter.close(); + await rm(root, { recursive: true, force: true }); + } + }, 30_000); + + it("ignores initial, hidden, and custom-ignored files while watching visible files", async () => { + const root = await mkdtemp(join(tmpdir(), "livesync-real-watch-ignore-")); + const events: WatchEvent[] = []; + const ignoreDirectory = join(root, ".livesync"); + await mkdir(ignoreDirectory); + await writeFile(join(ignoreDirectory, "ignore"), "private/\n"); + const ignoreRules = new IgnoreRules(root); + await ignoreRules.load(); + await mkdir(join(root, "private")); + await writeFile(join(root, "present-before-watch.md"), "initial file"); + await writeFile(join(root, ".hidden-before-watch.md"), "initial hidden file"); + const adapter = new CLIStorageEventManagerAdapter(root, ignoreRules, true); + + try { + await adapter.watch.beginWatch(makeHandlers(events)); + await writeFile(join(root, "visible-after-watch.md"), "visible"); + await writeFile(join(root, ".hidden-after-watch.md"), "hidden"); + await writeFile(join(root, "private", "ignored-after-watch.md"), "custom ignored"); + + expect( + await waitFor(() => + events.some((event) => event.type === "create" && event.path === "visible-after-watch.md") + ) + ).toBe(true); + await delay(700); + + expect(events.filter((event) => !event.folder)).toEqual([ + { type: "create", path: "visible-after-watch.md", folder: false }, + ]); + } finally { + await adapter.close(); + await rm(root, { recursive: true, force: true }); + } + }, 15_000); + + it("reports changes, deletions, rename paths, and stops after close", async () => { + const root = await mkdtemp(join(tmpdir(), "livesync-real-watch-lifecycle-")); + const events: WatchEvent[] = []; + const adapter = new CLIStorageEventManagerAdapter(root, undefined, true); + let closed = false; + let renameCalls = 0; + + try { + await adapter.watch.beginWatch(makeHandlers(events, () => renameCalls++)); + + const originalPath = join(root, "original.md"); + const renamedPath = join(root, "renamed.md"); + await writeFile(originalPath, "first version"); + expect( + await waitFor(() => events.some((event) => event.type === "create" && event.path === "original.md")) + ).toBe(true); + events.length = 0; + + await writeFile(originalPath, "updated version"); + expect( + await waitFor(() => events.some((event) => event.type === "change" && event.path === "original.md")) + ).toBe(true); + events.length = 0; + + await unlink(originalPath); + expect( + await waitFor(() => events.some((event) => event.type === "delete" && event.path === "original.md")) + ).toBe(true); + events.length = 0; + + await writeFile(originalPath, "rename source"); + expect( + await waitFor(() => + events.some((event) => event.type === "create" && event.path === "original.md" && !event.folder) + ) + ).toBe(true); + events.length = 0; + await rename(originalPath, renamedPath); + expect( + await waitFor(() => { + const sourceDeleted = events.some( + (event) => event.type === "delete" && event.path === "original.md" && !event.folder + ); + const destinationCreated = events.some( + (event) => event.type === "create" && event.path === "renamed.md" && !event.folder + ); + return sourceDeleted && destinationCreated; + }) + ).toBe(true); + expect(renameCalls).toBe(0); + + await adapter.close(); + closed = true; + const eventCountAtClose = events.length; + await writeFile(join(root, "after-close.md"), "watcher stopped"); + await delay(700); + expect(events).toHaveLength(eventCountAtClose); + } finally { + if (!closed) await adapter.close(); + await rm(root, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 2c92dbd6..7e7bc5b2 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -4,6 +4,9 @@ "version": "1.0.46-cli", "main": "dist/index.cjs", "type": "module", + "engines": { + "node": ">=22.22" + }, "scripts": { "dev": "vite", "build": "vite build", @@ -12,7 +15,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/runtimeState.unit.spec.ts src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/services/storageCaseSensitivity.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/journalSkip.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/runtimeState.unit.spec.ts src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/services/storageCaseSensitivity.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/journalSkip.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", @@ -38,7 +41,7 @@ }, "dependencies": { "@aws-crypto/sha256-js": "5.2.0", - "chokidar": "^4.0.0", + "chokidar": "6.0.0", "minimatch": "^11.0.3", "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", diff --git a/updates.md b/updates.md index f779c23c..f643dd27 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### CLI + +#### Fixed + +- The daemon now detects files written into newly created folders during a burst of writes, including files renamed from temporary names, without requiring a restart. (#1328) + - The CLI requires Node.js 22.22 or later. The Docker image includes its own runtime. + ## 1.0.46 7th November, 2026