From 116c2dfbfb7eadbd11ccb05e8ee678916201f66e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Thu, 12 Nov 2026 08:51:11 +0000 Subject: [PATCH] fix(cli): share Vault-relative path exclusions Apply watcher and logical-target exclusions within the selected Vault so hidden or database-like parent directories do not suppress monitoring. Protect primary, previous, and view databases, and provide a runtime opt-in for ordinary database-like directories without changing database ownership. Cover hidden roots, symlink restarts, exclusion parity, and encrypted CouchDB create, modify, and delete operations with regression tests. Refs #1340 --- docs/README.md | 1 + .../design_docs/cli_watcher_path_filtering.md | 303 +++++++++ src/apps/cli/README.md | 26 + src/apps/cli/cliPathFilter.ts | 92 +++ src/apps/cli/cliPathFilter.unit.spec.ts | 138 ++++ src/apps/cli/commands/types.ts | 2 + src/apps/cli/main.bootstrap.unit.spec.ts | 146 ++++- src/apps/cli/main.ts | 41 +- src/apps/cli/main.unit.spec.ts | 16 + ...ventManagerAdapter.real-watch.unit.spec.ts | 184 +++++- .../managers/CLIStorageEventManagerAdapter.ts | 92 ++- ...CLIStorageEventManagerAdapter.unit.spec.ts | 63 ++ .../cli/managers/StorageEventManagerCLI.ts | 18 +- src/apps/cli/package.json | 2 +- .../cli/serviceModules/CLIServiceModules.ts | 19 +- test/README.md | 35 + test/e2e-cli/ci-tasks.json | 1 + test/e2e-cli/tasks.json | 3 + test/e2e-cli/test-daemon-watch-paths.ts | 616 ++++++++++++++++++ updates.md | 12 + 20 files changed, 1703 insertions(+), 107 deletions(-) create mode 100644 docs/design_docs/cli_watcher_path_filtering.md create mode 100644 src/apps/cli/cliPathFilter.unit.spec.ts create mode 100644 test/e2e-cli/test-daemon-watch-paths.ts diff --git a/docs/README.md b/docs/README.md index af2de393..a81f200e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,7 @@ Use these references for the area being changed, together with its relevant ADR. | P2P connection preparation | [TURN connection settings](design_docs/renewable_turn_credentials.md) | | Journal storage internals | [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) | | CLI database selection and interrupted reconstruction | [CLI database continuity](design_docs/cli_database_continuity.md) | +| CLI filesystem monitoring | [CLI watcher path filtering](design_docs/cli_watcher_path_filtering.md): accepted implementation; locally validated, not yet released | The installed `@vrtmrz/livesync-commonlib` package is authoritative for shared runtime contracts. Use the exact dependency selected by [package.json](../package.json) and its lockfile when checking behaviour. The [Commonlib documentation](https://github.com/vrtmrz/livesync-commonlib/tree/main/docs) is maintained in that repository; this index does not replace its package contracts. diff --git a/docs/design_docs/cli_watcher_path_filtering.md b/docs/design_docs/cli_watcher_path_filtering.md new file mode 100644 index 00000000..fff4e4fd --- /dev/null +++ b/docs/design_docs/cli_watcher_path_filtering.md @@ -0,0 +1,303 @@ +--- +date: 2026-11-12 +commonlib-version: "0.4.3" +self-hosted-livesync-version: "1.0.49" +status: accepted +--- + +# CLI watcher path filtering + +Implementation note: the correction is implemented and locally validated, but +has not been released. The reviewed baseline remains Self-hosted LiveSync +`78ebc2d2` and Commonlib `0.4.3`. The original failure was reproduced against +Self-hosted LiveSync `71893f2e` (`1.0.47`) with Commonlib `0.4.1` and Chokidar `6.0.0`. +Reporter confirmation remains pending. + +## Problem and required behaviour + +[#1340](https://github.com/vrtmrz/obsidian-livesync/issues/1340) reports that new +Vault files never reach the remote after Fetch. In the focused reproduction, +the real CLI watch adapter accepts a normal Vault and a visible symlink to a +hidden-parent Vault, but emits no new-file event after that symlink is resolved +to its physical path. The file exists, watcher readiness completes, and no +watcher diagnostic is reported. This reproduces the monitoring failure without +Fetch, Metadata collisions, or a negative timeout. + +Before this correction, the watcher applied its default exclusion expressions +to the full filesystem path. A dot-prefixed directory above the selected Vault +could therefore exclude the entire watch root. + +The selected Vault must remain watchable regardless of the spelling of its root +or parent directories. Default exclusions and user ignore rules must apply to +entries within that Vault. The correction must preserve the canonical physical +root used by [CLI database continuity](cli_database_continuity.md). + +## Scope + +The implementation uses one CLI-local refinement, `VaultRelativePath`, and one +conversion boundary. It shares the watcher and logical target exclusion policy and adds a runtime +compatibility option, while retaining `main.ts` path resolution, database +selection, runtime scopes, Commonlib APIs, and the existing symlink-following +policy. + +The type expresses a checked lexical path relative to the supplied Vault root. +It does not establish filesystem existence, Unicode or case resolution, +physical containment through descendant symlinks, or continued validity after +filesystem changes. It does not encode which Vault instance produced the value. + +## Type and conversion boundary + +The CLI-local `src/apps/cli/cliPathFilter.ts` retains `isHiddenPath(string)` +for its existing callers and provides these internal CLI contracts: + +```ts +import type { FilePath } from "@vrtmrz/livesync-commonlib/storage"; +import type { IgnoreRules } from "./serviceModules/IgnoreRules"; + +declare const vaultRelativePathBrand: unique symbol; + +export type VaultRelativePath = FilePath & { + readonly [vaultRelativePathBrand]: true; +}; + +export interface CLIPathFilterOptions { + readonly includeDbLikeDirs?: boolean; + readonly databaseRoot?: VaultRelativePath; +} + +export declare function toVaultRelativePath(vaultRoot: string, candidatePath: string): VaultRelativePath | undefined; + +export declare function isIgnoredFilePath( + relativePath: FilePath, + rules?: Pick, + options?: CLIPathFilterOptions +): boolean; + +export declare function isIgnoredVaultPath( + relativePath: VaultRelativePath, + rules?: Pick, + options?: CLIPathFilterOptions +): boolean; +``` + +Use a distinct local symbol to refine `FilePath`. Both the existing +`FilePath` and a new `TaggedType` use the same underlying tag +property; assigning a different tag to `FilePath` would conflict with its +existing tag. The refinement above remains assignable to the existing +`NodeFile.path` and `NodeFolder.path` contracts without another assertion. + +Only `toVaultRelativePath` creates the local refinement: + +1. Require absolute native filesystem paths for both inputs. The caller retains + the root obtained by the existing start-up `fs.realpath` call. +2. Compute `path.relative(vaultRoot, candidatePath)`, using the existing native + `path` implementation. +3. Retain the CLI's existing conversion of separators to `/`. +4. Pass the result through Commonlib's `parseFilePath`. It accepts the empty + root path and rejects absolute, drive-qualified, backslash, and traversal + input without changing spelling. +5. Return the validated `FilePath` with the additional local brand. Return + `undefined` for invalid input or a result outside the lexical root. + +Keep any catch around the pure parser call only. This conversion performs no +filesystem reads, existence checks, Unicode conversion, or case conversion. +The parser is responsible for relative-path syntax; do not duplicate its +validation expressions. Different-drive and sibling-prefix cases must fail +validation rather than acquire the brand. + +A deleted file still has a convertible event path. Do not call `realpath` or +`stat` for each event. The canonical root is retained as a separate value; +computing a relative path does not replace it. + +## Exclusion policy + +`isIgnoredFilePath` owns the policy for validated logical paths. +`isIgnoredVaultPath` retains the narrower watcher contract and delegates to it. +`main.ts` validates logical target paths with `parseFilePath` before applying the +same policy. Its default and user-rule handler priorities remain unchanged. + +| Input or condition | Decision | +| ----------------------------------------------------------------- | ------------------------------------------------ | +| Empty string, denoting the selected root | Keep the root watchable | +| A dot-prefixed component within the Vault | Exclude | +| A DB-like entry directly within the configured database directory | Exclude, including with the compatibility option | +| Any DB-like component elsewhere, with the default options | Exclude | +| An entry matched by `IgnoreRules` | Exclude, including with the compatibility option | +| Any other valid entry | Monitor or admit as a target | + +DB-like names end in Commonlib's `SuffixDatabaseName` (`-livesync-v2`) or contain +`-livesync-v2-mrview-`, the prefix PouchDB uses for dependent view databases. +The CLI's PouchDB factory produces both primary and view directories; a primary +name-only check misses the latter. Previous database suffixes remain protected +in the configured database directory. + +The database root is converted from the canonical `databasePath` to a +`VaultRelativePath`. When it is outside the Vault, it creates no protected +storage namespace inside the Vault. When both roots coincide, the empty string +identifies that namespace. Only its direct DB-like entries and their descendants +receive unconditional protection; the database root itself remains traversable. +This is a lexical ownership rule, not inspection of arbitrary directory contents. + +### Compatibility option + +`--include-db-like-dirs` permits ordinary paths otherwise excluded only by the +DB-like naming rule. It applies consistently to initial scans, live events, and +logical target checks. Hidden entries, custom ignore rules, and DB-like entries +in the configured database storage namespace remain excluded. + +For example, with both the database and Vault rooted at ``: + +| Vault-relative path | Default | With the option | +| ---------------------------------------- | ------- | --------------- | +| `local-livesync-v2/CURRENT` | Exclude | Exclude | +| `local-livesync-v2-mrview-index/CURRENT` | Exclude | Exclude | +| `notes/archive-livesync-v2/note.md` | Exclude | Admit | +| `.livesync/settings.json` | Exclude | Exclude | + +With the database root at `/state`, the first two protected entries are +under `state/` instead. With a database root outside the Vault, matching user +directories in the Vault can be admitted at any depth by the option. + +The option is runtime-only and must be supplied on each invocation that needs +it. It adds no stored setting, migration, or ownership ledger. The policy cannot +identify an unrelated database merely from its contents; an opt-in can admit +such directories outside the configured storage namespace. Existing descendant +symlink behaviour and its physical-containment limits remain unchanged. + +Reuse `isHiddenPath` and `IgnoreRules.shouldIgnore`; a hidden parent outside the +selected Vault must not affect these decisions. A user rule can exclude every +entry while the watcher root remains watchable. + +## Watcher integration and rejected candidates + +The Chokidar `ignored` callback converts its candidate before calling the typed +policy. If conversion returns `undefined`, the candidate is excluded without +throwing or reporting an error. Chokidar may evaluate ancestors of its root; +version `6.0.0` separately keeps root ancestors watchable. Do not manufacture a +Vault-relative value for those paths. + +Bind conversion to the adapter's stored `basePath`: watch callbacks supply only +the candidate path to an adapter-local wrapper. Keep the pure two-input +converter available for unit tests, and do not derive a new root from an event. + +Use the same converter in the adapter's file and folder event conversion, so +those paths are produced at the same boundary. A successful value flows into +the existing `FilePath` fields directly. Preserve folder-root event semantics. + +An emitted event that cannot be converted is unexpected: use the existing CLI +diagnostic reporter and omit that event. It must not create a pending storage +operation, reset state, terminate the daemon, or stop unrelated valid events. +Expected ancestor candidates in the `ignored` callback are not diagnostics. +Diagnostics must describe the rejected path and the selected root without +claiming that all monitoring has stopped. + +The adapter uses the typed policy rather than calling its string-based +exclusion helpers directly. Do not add assertions at callback call sites to +bypass conversion. + +## Compatibility and bounded alternatives + +The canonical root continues to identify scanner state, reflection provenance, +storage-event snapshots, and replication checkpoints. This correction adds no +persistent state or migration. Existing legacy-state admission still applies +when upgrading an older database. + +Keeping an unresolved symlink name only for watching would leave direct hidden +roots broken and introduce another root representation. Removing `realpath` +would also affect runtime ownership. Neither is required by this design. + +Commonlib's `StoragePath` includes resolved physical spelling, which this lexical +converter does not prove. Use its existing `FilePath` and `parseFilePath` +contracts instead of asserting `StoragePath`. A complete absolute/canonical +path type system and per-Vault generic types remain outside this correction. + +Metadata identity collisions, existing `document-id-mismatch` entries, and +negative timeout warnings remain separate investigations. Monitoring recovery +must not be described as repairing those entries. + +## Implementation and verification + +| Location | Responsibility | +| -------------------------------------------------------- | ---------------------------------------------------------------------- | +| `src/apps/cli/cliPathFilter.ts` | The refinement, checked conversion, and typed exclusion policy | +| `src/apps/cli/managers/CLIStorageEventManagerAdapter.ts` | Connect ignored candidates and emitted events to that boundary | +| CLI path-filter and watcher unit tests | Conversion, policy, type compatibility, and event conversion | +| Existing real-filesystem watcher tests | Preserve the original failure and verify actual events | +| A focused `test/e2e-cli` scenario | Fetch, live daemon monitoring, remote contents, and restart continuity | + +Acceptance requires the following evidence: + +- Port the failing `1.0.47` scenario to the implementation baseline and confirm + failure for the missing CREATE event before changing production code. +- Normal and hidden-parent roots, visible symlinks, and root names resembling + internal database directories must report create, modify, and delete events + with correct relative paths. +- Dot-prefixed entries, internal databases, custom ignore patterns, and existing + symlink behaviour must retain their exclusions. Exercise Chokidar's ancestor + candidates as well as the root itself. +- Cover invalid or relative inputs, `..`, sibling prefixes, platform-native + separators, and different drives where supported. Rejected emitted events + must not prevent later valid events. +- Type checks must accept `VaultRelativePath` as `FilePath` and reject both a + plain string and an unrefined `FilePath` at the typed policy. Vitest execution + alone does not perform this check; run the CLI TypeScript check explicitly. +- A disposable CouchDB test must complete Fetch, start the real daemon, and + verify created and edited contents and deletion on the remote. Restart through + an alias and its physical path and verify the same database selection and + runtime scope. The existing `test-daemon-linux.sh` mirror tests alone do not + exercise this live monitoring boundary. +- Complete the repository checks, unit tests, production build, CLI check and + build, and the focused CLI E2E required by the implementation. Run broad + validation serially with the configured heap limit. + +Describe local verification and reporter confirmation separately when publishing +the correction. Verify default/option parity and protected primary, old, and +derived databases when changing the shared policy. +The original focused reproduction establishes the failure. Evidence for the +correction, including the CouchDB test, is recorded below. + +## Local verification and remaining coverage + +The current-main regression first failed for hidden roots and parents and for +roots or parents matching the internal-database exclusion. Normal roots and +descendant symlinks remained operational. After the correction, the same +real-filesystem cases pass for creation, modification, and deletion. Hidden +entries, internal databases, and user patterns remain excluded. + +Local verification used Linux ARM64, Node.js 26.13.0, Commonlib 0.4.3, and +Chokidar 6.0.0. The repository checks, full unit suite, CLI and production +builds, E2E TypeScript check, and iOS bundle compatibility check passed. + +The disposable CouchDB test uses E2EE, online Chunk retrieval, and an independent +replicating observer. It verifies Fetch, live create/modify/delete, exclusions +in the observer database, and restart from the symlink to the physical path +with unchanged database selection and runtime scope. + +Baseline current Metadata identity and content remain unchanged on both peers. +The source retains its original revision body, and CouchDB retains that +revision in the baseline document's ancestry. The test does not require peers +to download unavailable historical revision bodies. + +The Windows-only different-drive case was skipped on Linux; this local run +does not establish Windows filesystem behaviour. Reporter confirmation and +investigation of the separate Metadata identity and timeout warnings remain +outstanding. + +### Shared-policy update + +The shared policy and `--include-db-like-dirs` extend the initial watcher-only +correction. Before implementation, a bootstrap regression demonstrated that a +nested DB-like path passed the logical target filter while the watcher excluded +it. A real PouchDB inventory also demonstrated the separate view directory. +The updated tests cover these cases and the compatibility option. Local +verification passed the CLI TypeScript check, repository checks, full unit +suite (2,157 passed and two skipped), CLI and production builds, E2E TypeScript +check, and the encrypted CouchDB scenario. The latter confirms default exclusion +from the independent observer database, then initial contents, modifications, +and deletion after restarting both clients with the option. Bootstrap tests +also create a real PouchDB view index and verify that current, previous, and +view databases remain excluded with the option. + +The two skipped unit cases are platform-dependent: a Windows different-drive +case and an existing host-filesystem alias case. Windows runtime behaviour and +reporter confirmation remain unverified. The correction remains unreleased. diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 29c66016..c7fc26b1 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -71,6 +71,7 @@ livesync-cli [database-path] [command] [args...] - `--vault ` / `-V `: (daemon/mirror/rebuild/fetch) Path to the Vault directory containing `.md` files. - Allows the PouchDB database directory and the actual vault directory to be different locations. - For `mirror` command, the positional `[vault-path]` argument takes precedence over `--vault`. +- `--include-db-like-dirs`: Include ordinary directories whose names match the default database-name exclusions. See [database-like directory exclusions](#database-like-directory-exclusions). Hidden entries and user ignore rules remain excluded. - `--force` / `-f`: Required by `rebuild` and `fetch` because they replace a database. It also retains its existing `init-settings` overwrite behaviour. - `--write-settings`: Write setting migrations and other lasting changes after the command succeeds. - `init-settings` writes its target file. `setup`, `remote-add`, `remote-rm`, `remote-set`, and `remote-activate` write their settings changes without this option. @@ -675,6 +676,31 @@ livesync-cli /path/to/vault livesync-cli /path/to/vault --interval 60 ``` +### Database-like directory exclusions + +By default, Vault synchronisation excludes path components ending in +`-livesync-v2` or containing `-livesync-v2-mrview-` at every depth. These names +cover the CLI's primary PouchDB directories and their view indexes. The same +policy applies during startup scans, live monitoring, and target checks for +received files. + +If an ordinary directory uses one of these names, specify `--include-db-like-dirs` +on each invocation that should synchronise it: + +```bash +livesync-cli ./database --vault ./vault --include-db-like-dirs daemon +``` + +The option does not override hidden-entry exclusions or `.livesync/ignore`. +Direct database-like entries inside the configured `database-path` also remain +excluded, including previous databases and view indexes. When `database-path` +and the Vault are the same directory, the option can include matching names +below ordinary subdirectories, but direct database-like entries remain protected. + +The option changes a naming rule; it does not inspect arbitrary directory +contents. An unrelated database outside the configured database storage area +can also be included by opting in. The option is not saved to settings. + ### .livesync/ignore Place a `.livesync/ignore` file in your vault root to exclude files from sync in both directions (local → CouchDB and CouchDB → local). diff --git a/src/apps/cli/cliPathFilter.ts b/src/apps/cli/cliPathFilter.ts index 41447bc5..81bc8df2 100644 --- a/src/apps/cli/cliPathFilter.ts +++ b/src/apps/cli/cliPathFilter.ts @@ -1,3 +1,95 @@ +import { path } from "@vrtmrz/livesync-commonlib/node"; +import { SuffixDatabaseName } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { parseFilePath, type FilePath } from "@vrtmrz/livesync-commonlib/storage"; +import type { IgnoreRules } from "./serviceModules/IgnoreRules"; + +declare const vaultRelativePathBrand: unique symbol; + +/** + * A checked lexical path relative to the selected Vault. Filesystem existence, + * descendant symlink targets, and the identity of the Vault require separate checks. + */ +export type VaultRelativePath = FilePath & { + readonly [vaultRelativePathBrand]: true; +}; + +export interface CLIPathFilterOptions { + /** Relax the naming heuristic while retaining storage protection and user exclusions. */ + readonly includeDbLikeDirs?: boolean; + /** Configured DB root relative to the Vault: shared roots use ''; an outside root is undefined. */ + readonly databaseRoot?: VaultRelativePath; +} + export function isHiddenPath(relativePath: string): boolean { return relativePath.split("/").some((part) => part.startsWith(".")); } + +/** + * The sole constructor for watcher-relative paths. Rebase before filtering so + * names above the selected Vault cannot exclude the entire watch root. + */ +export function toVaultRelativePath(vaultRoot: string, candidatePath: string): VaultRelativePath | undefined { + // Relative inputs would make the result depend on the process working directory. + if (!path.isAbsolute(vaultRoot) || !path.isAbsolute(candidatePath)) { + return undefined; + } + + const relativePath = path.relative(vaultRoot, candidatePath).replace(/\\/g, "/"); + try { + // Preserve spelling and the empty root; reject outside/traversal and drive-qualified results. + return parseFilePath(relativePath) as VaultRelativePath; + } catch { + return undefined; + } +} + +// PouchDB creates view indexes beside the primary DB using its name plus '-mrview-'. +// Both forms are conservative name matches, rather than proof of a directory's contents. +function isDbLikeDirectoryName(name: string): boolean { + return name.endsWith(SuffixDatabaseName) || name.includes(`${SuffixDatabaseName}-mrview-`); +} + +/** Protect current, previous, and view DB entries even when the naming heuristic is relaxed. */ +function isDatabaseStoragePath(relativePath: FilePath, databaseRoot: VaultRelativePath | undefined): boolean { + if (databaseRoot === undefined) { + return false; + } + // Include the separator so a sibling such as 'state-copy' cannot enter the 'state' namespace. + const prefix = databaseRoot === "" ? "" : `${databaseRoot}/`; + if (!relativePath.startsWith(prefix)) { + return false; + } + // Protect the direct DB entry and all descendants. Nested user-directory names + // elsewhere still belong to the optional naming rule. + const entry = relativePath.slice(prefix.length).split("/")[0]; + return isDbLikeDirectoryName(entry); +} + +/** Shared exclusions for logical targets and watcher paths already rebased to the Vault. */ +export function isIgnoredFilePath( + relativePath: FilePath, + rules?: Pick, + options: CLIPathFilterOptions = {} +): boolean { + // Keep the root traversable even when a user rule excludes every entry beneath it. + if (relativePath === "") { + return false; + } + + // The opt-in bypasses only the naming heuristic; protected storage and explicit ignores still apply. + return ( + isHiddenPath(relativePath) || + isDatabaseStoragePath(relativePath, options.databaseRoot) || + (!options.includeDbLikeDirs && relativePath.split("/").some(isDbLikeDirectoryName)) || + (rules?.shouldIgnore(relativePath) ?? false) + ); +} + +/** Keep raw filesystem candidates out of the shared policy until conversion has validated them. */ +export function isIgnoredVaultPath( + relativePath: VaultRelativePath, + rules?: Pick, + options: CLIPathFilterOptions = {} +): boolean { + return isIgnoredFilePath(relativePath, rules, options); +} diff --git a/src/apps/cli/cliPathFilter.unit.spec.ts b/src/apps/cli/cliPathFilter.unit.spec.ts new file mode 100644 index 00000000..92b7554a --- /dev/null +++ b/src/apps/cli/cliPathFilter.unit.spec.ts @@ -0,0 +1,138 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { path } from "@vrtmrz/livesync-commonlib/node"; +import { parseFilePath, type FilePath } from "@vrtmrz/livesync-commonlib/storage"; +import { + isHiddenPath, + isIgnoredFilePath, + isIgnoredVaultPath, + toVaultRelativePath, + type VaultRelativePath, +} from "./cliPathFilter"; + +const vaultRoot = path.resolve(path.sep, "vault"); + +function checkedRelativePath(relativePath: string): VaultRelativePath { + const converted = toVaultRelativePath(vaultRoot, path.join(vaultRoot, relativePath)); + if (converted === undefined) { + throw new Error(`Expected a valid Vault-relative fixture: ${relativePath}`); + } + return converted; +} + +describe("toVaultRelativePath", () => { + it.each(["", "note.md", "notes/note.md", "..draft.md", "Notes/e\u0301.md"])( + "retains the relative spelling of %j without requiring a filesystem entry", + (relativePath) => { + expect(toVaultRelativePath(vaultRoot, path.join(vaultRoot, relativePath))).toBe(relativePath); + } + ); + + it("uses the selected root even when it or its parents have excluded names", () => { + const root = path.resolve(path.sep, ".hidden", "archive-livesync-v2", ".vault"); + const converted = toVaultRelativePath(root, path.join(root, "note.md")); + expect(converted).toBe("note.md"); + expect(converted !== undefined && isIgnoredVaultPath(converted)).toBe(false); + }); + + it.each([ + [".", path.join(vaultRoot, "note.md")], + [vaultRoot, "note.md"], + [vaultRoot, path.resolve(vaultRoot, "..", "outside.md")], + [vaultRoot, path.join(vaultRoot + "-sibling", "note.md")], + [vaultRoot, path.dirname(vaultRoot)], + ])("rejects a non-Vault candidate (%s, %s)", (root, candidate) => { + expect(toVaultRelativePath(root, candidate)).toBeUndefined(); + }); + + it.skipIf(process.platform !== "win32")("rejects a candidate on another drive", () => { + expect(toVaultRelativePath("C:\\vault", "D:\\note.md")).toBeUndefined(); + }); + + it("keeps the refinement compatible with existing file paths and rejects unconverted inputs", () => { + expectTypeOf().toMatchTypeOf(); + expectTypeOf[0]>().toEqualTypeOf(); + expectTypeOf().not.toMatchTypeOf[0]>(); + expectTypeOf().not.toMatchTypeOf[0]>(); + expectTypeOf[0]>().toEqualTypeOf(); + expectTypeOf().not.toMatchTypeOf[0]>(); + }); +}); + +describe("isIgnoredVaultPath", () => { + it.each([ + [".hidden.md", true], + ["notes/.hidden.md", true], + [".git/config", true], + ["local-livesync-v2/state", true], + ["notes/local-livesync-v2/state", true], + ["notes/local-livesync-v2-mrview-0123456789abcdef/CURRENT", true], + ["-livesync-v2", true], + ["notes/note.md", false], + ["local-livesync-v2.md", false], + ["local-livesync-v2-copy/state", false], + ])("filters %s within the selected Vault", (candidate, ignored) => { + expect(isIgnoredVaultPath(checkedRelativePath(candidate))).toBe(ignored); + expect(isIgnoredFilePath(parseFilePath(candidate))).toBe(ignored); + }); + + it("keeps the root watchable even when user rules exclude every entry", () => { + const rules = { shouldIgnore: vi.fn(() => true) }; + expect(isIgnoredVaultPath(checkedRelativePath(""), rules)).toBe(false); + expect(rules.shouldIgnore).not.toHaveBeenCalled(); + expect(isIgnoredVaultPath(checkedRelativePath("note.md"), rules)).toBe(true); + }); + + it("passes only the relative path to user rules", () => { + const rules = { shouldIgnore: vi.fn((candidate: string) => candidate === "private/note.md") }; + expect(isIgnoredVaultPath(checkedRelativePath("private/note.md"), rules)).toBe(true); + expect(rules.shouldIgnore).toHaveBeenCalledWith("private/note.md"); + expect(isIgnoredVaultPath(checkedRelativePath("public/note.md"), rules)).toBe(false); + }); + + it("includes DB-like entries outside the configured database namespace when requested", () => { + const options = { includeDbLikeDirs: true, databaseRoot: checkedRelativePath("state") }; + for (const candidate of ["archive-livesync-v2/note.md", "notes/archive-livesync-v2/note.md"]) { + expect(isIgnoredVaultPath(checkedRelativePath(candidate), undefined, options)).toBe(false); + expect(isIgnoredFilePath(parseFilePath(candidate), undefined, options)).toBe(false); + } + for (const candidate of [ + "state/current-livesync-v2/CURRENT", + "state/previous-livesync-v2/CURRENT", + "state/current-livesync-v2-mrview-index/CURRENT", + "state/previous-livesync-v2-mrview-temp/CURRENT", + ]) { + expect(isIgnoredVaultPath(checkedRelativePath(candidate), undefined, options)).toBe(true); + } + expect(isIgnoredVaultPath(checkedRelativePath("state"), undefined, options)).toBe(false); + expect( + isIgnoredVaultPath(checkedRelativePath("state-copy/archive-livesync-v2/note.md"), undefined, options) + ).toBe(false); + }); + + it("protects colocated database entries while allowing nested DB-like user directories", () => { + const options = { includeDbLikeDirs: true, databaseRoot: checkedRelativePath("") }; + expect(isIgnoredVaultPath(checkedRelativePath("local-livesync-v2/CURRENT"), undefined, options)).toBe(true); + expect( + isIgnoredVaultPath(checkedRelativePath("local-livesync-v2-mrview-index/CURRENT"), undefined, options) + ).toBe(true); + expect(isIgnoredVaultPath(checkedRelativePath("notes/local-livesync-v2/note.md"), undefined, options)).toBe( + false + ); + expect(isIgnoredVaultPath(checkedRelativePath(""), undefined, options)).toBe(false); + }); + + it("keeps hidden entries and user rules excluded when DB-like entries are included", () => { + const options = { includeDbLikeDirs: true }; + const rules = { shouldIgnore: vi.fn((candidate: string) => candidate.endsWith(".tmp")) }; + expect(isIgnoredVaultPath(checkedRelativePath(".livesync/settings.json"), rules, options)).toBe(true); + expect(isIgnoredVaultPath(checkedRelativePath("notes/.private/note.md"), rules, options)).toBe(true); + expect(isIgnoredVaultPath(checkedRelativePath("archive-livesync-v2/note.tmp"), rules, options)).toBe(true); + expect(rules.shouldIgnore).toHaveBeenCalledWith("archive-livesync-v2/note.tmp"); + expect(isIgnoredVaultPath(checkedRelativePath("archive-livesync-v2/note.md"), rules, options)).toBe(false); + }); + + it("preserves the existing hidden-path predicate for its current callers", () => { + expect(isHiddenPath("notes/.draft.md")).toBe(true); + expect(isHiddenPath("notes/draft.md")).toBe(false); + }); +}); diff --git a/src/apps/cli/commands/types.ts b/src/apps/cli/commands/types.ts index bd04d4bb..c935620f 100644 --- a/src/apps/cli/commands/types.ts +++ b/src/apps/cli/commands/types.ts @@ -47,6 +47,8 @@ export interface CLIOptions { writeSettings?: boolean; /** Restore the former zero exit code after a returned remote-administration verification failure. */ compatRemoteAdminExitZero?: boolean; + /** Per-invocation opt-in; configured DB protection remains effective and no setting is saved. */ + includeDbLikeDirs?: boolean; command: CLICommand; commandArgs: string[]; interval?: number; diff --git a/src/apps/cli/main.bootstrap.unit.spec.ts b/src/apps/cli/main.bootstrap.unit.spec.ts index 558d3fd3..94755d2f 100644 --- a/src/apps/cli/main.bootstrap.unit.spec.ts +++ b/src/apps/cli/main.bootstrap.unit.spec.ts @@ -1,9 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { appendFile, mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { appendFile, mkdtemp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import * as chokidar from "chokidar"; import { path } from "@vrtmrz/livesync-commonlib/node"; +import { parseFilePath } from "@vrtmrz/livesync-commonlib/storage"; import * as offlineScanner from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; import { ReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/base/ReplicatorService"; import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; @@ -85,7 +86,8 @@ describe("CLI database preparation", () => { runner: CliCommandRunner, exitCode = 1, force = false, - writeSettings = false + writeSettings = false, + extraArgs: readonly string[] = [] ) { process.exitCode = undefined; process.argv = [ @@ -98,6 +100,7 @@ describe("CLI database preparation", () => { vaultPath, "--settings", settingsPath, + ...extraArgs, command, ]; // Daemon probes return false so the real core unloads without keeping a daemon alive. @@ -114,28 +117,31 @@ describe("CLI database preparation", () => { expect(restore).not.toHaveBeenCalled(); }); - it.each(["fetch", "rebuild"] as const)("invalidates old reflection work after the %s database reset", async (command) => { - const snapshotKey = "replicationResultProcessorSnapshot"; - const oldSnapshot = { queued: [{ _id: "old", _rev: "1-old" }], processing: [] }; - let retainedBeforeReset: unknown; - let retainedAfterReset: unknown; - let reset: boolean | undefined; - const runner = vi.fn(async (_options, { core }) => { - await core.services.keyValueDB.kvDB.set(snapshotKey, oldSnapshot); - retainedBeforeReset = await core.services.keyValueDB.kvDB.get(snapshotKey); - reset = await core.services.database.resetDatabase(); - retainedAfterReset = await core.services.keyValueDB.kvDB.get(snapshotKey); - return false; - }); + it.each(["fetch", "rebuild"] as const)( + "invalidates old reflection work after the %s database reset", + async (command) => { + const snapshotKey = "replicationResultProcessorSnapshot"; + const oldSnapshot = { queued: [{ _id: "old", _rev: "1-old" }], processing: [] }; + let retainedBeforeReset: unknown; + let retainedAfterReset: unknown; + let reset: boolean | undefined; + const runner = vi.fn(async (_options, { core }) => { + await core.services.keyValueDB.kvDB.set(snapshotKey, oldSnapshot); + retainedBeforeReset = await core.services.keyValueDB.kvDB.get(snapshotKey); + reset = await core.services.database.resetDatabase(); + retainedAfterReset = await core.services.keyValueDB.kvDB.get(snapshotKey); + return false; + }); - await start(command, runner, 1, true); + await start(command, runner, 1, true); - expect(runner).toHaveBeenCalledOnce(); - expect(reset).toBe(true); - expect(retainedBeforeReset).toEqual(oldSnapshot); - expect(retainedAfterReset).toEqual({ queued: [], processing: [] }); - expect(await readFile(join(vaultPath, "notes/local.md"), "utf8")).toBe("local content"); - }); + expect(runner).toHaveBeenCalledOnce(); + expect(reset).toBe(true); + expect(retainedBeforeReset).toEqual(oldSnapshot); + expect(retainedAfterReset).toEqual({ queued: [], processing: [] }); + expect(await readFile(join(vaultPath, "notes/local.md"), "utf8")).toBe("local content"); + } + ); it.each([ { handleFilenameCaseSensitive: true, pathNormalizationVersion: 0 as const }, @@ -355,6 +361,102 @@ describe("CLI database preparation", () => { } ); + it.each([false, true])( + "uses the same database-directory exclusions for scans and watching (include: %s)", + async (includeDbLikeDirs) => { + const databaseEntry = parseFilePath("notes/archive-livesync-v2/state"); + await mkdir(join(vaultPath, "notes/archive-livesync-v2")); + await writeFile(join(vaultPath, databaseEntry), "database-directory fixture"); + const storedPaths: string[] = []; + let isTarget: boolean | undefined; + let watcherIgnored: boolean | undefined; + const runner = vi.fn(async (_options, { core }) => { + isTarget = await core.services.vault.isTargetFile(databaseEntry); + const watcherCalls = vi.mocked(chokidar.watch).mock.calls; + const ignored = watcherCalls[watcherCalls.length - 1]?.[1]?.ignored; + if (typeof ignored !== "function") { + throw new Error("Expected the CLI watcher path policy"); + } + watcherIgnored = ignored(join(vaultPath, databaseEntry)); + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + return false; + }); + + await start("daemon", runner, 1, false, false, includeDbLikeDirs ? ["--include-db-like-dirs"] : []); + + expect(runner).toHaveBeenCalledOnce(); + expect(watcherIgnored).toBe(!includeDbLikeDirs); + expect(isTarget).toBe(includeDbLikeDirs); + expect(storedPaths.sort()).toEqual( + includeDbLikeDirs ? [databaseEntry, "notes/local.md"] : ["notes/local.md"] + ); + } + ); + + it("keeps actual, old, and derived database files excluded with the compatibility option", async () => { + const databaseRoot = join(vaultPath, "state"); + // Retained entries simulate older suffixes; the opt-in must protect them + // alongside the active DB and the real view index created below. + for (const entry of ["previous-livesync-v2", "previous-livesync-v2-mrview-index"]) { + await mkdir(join(databaseRoot, entry), { recursive: true }); + await writeFile(join(databaseRoot, entry, "CURRENT"), "retained database fixture"); + } + const protectedEntries: { path: string; isTarget: boolean; ignored: boolean }[] = []; + const storedPaths: string[] = []; + let hasDerivedDatabase = false; + const runner = vi.fn(async (_options, { core }) => { + const database = core.services.database.localDatabase.localDatabase; + await database.createIndex({ index: { fields: ["path"], ddoc: "path-protection", name: "by-path" } }); + const entries = (await readdir(databaseRoot)).filter((entry) => entry !== ".livesync"); + hasDerivedDatabase = entries.some( + (entry) => entry.includes("-mrview-") && entry !== "previous-livesync-v2-mrview-index" + ); + const watcherCalls = vi.mocked(chokidar.watch).mock.calls; + const ignored = watcherCalls[watcherCalls.length - 1]?.[1]?.ignored; + if (typeof ignored !== "function") { + throw new Error("Expected the CLI watcher path policy"); + } + for (const entry of entries) { + const candidate = parseFilePath(`state/${entry}/CURRENT`); + protectedEntries.push({ + path: candidate, + isTarget: await core.services.vault.isTargetFile(candidate), + ignored: ignored(join(vaultPath, candidate)), + }); + } + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + return false; + }); + process.argv = [ + "node", + "livesync-cli", + databaseRoot, + "--vault", + vaultPath, + "--settings", + settingsPath, + "--include-db-like-dirs", + "daemon", + ]; + + await expect(main(standardIo, runner)).rejects.toThrow("__EXIT__:1"); + + expect(runner).toHaveBeenCalledOnce(); + expect(hasDerivedDatabase).toBe(true); + expect(protectedEntries).toEqual( + expect.arrayContaining([ + { path: "state/previous-livesync-v2/CURRENT", isTarget: false, ignored: true }, + { path: "state/previous-livesync-v2-mrview-index/CURRENT", isTarget: false, ignored: true }, + ]) + ); + expect(protectedEntries.every((entry) => !entry.isTarget && entry.ignored)).toBe(true); + expect(storedPaths).toEqual(["notes/local.md"]); + }); + it("retains filesystem creation time in database Metadata after an in-place edit", async (context) => { const note = join(vaultPath, "notes/local.md"); const initial = await stat(note); diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 35c5038a..1d90bbf4 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -24,7 +24,8 @@ import { getForceRequirementError } from "./commands/utils"; import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils"; import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { IgnoreRules } from "./serviceModules/IgnoreRules"; -import { isHiddenPath } from "./cliPathFilter"; +import { isIgnoredFilePath, toVaultRelativePath, type CLIPathFilterOptions } from "./cliPathFilter"; +import { parseFilePath, type FilePath } from "@vrtmrz/livesync-commonlib/storage"; import { useP2PReplicatorFeature, type UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/p2p"; import { useOfflineScanner } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; import type { ReplicationSchedulingControl } from "@/serviceFeatures/replicationScheduling"; @@ -162,6 +163,8 @@ Options: --force, -f Required for rebuild and fetch because they replace databases --interval , -i (daemon only) Poll CouchDB every N seconds instead of using the _changes feed --write-settings Write setting changes after a successful command + --include-db-like-dirs Include user directories whose names resemble database directories + CLI database storage entries, hidden entries, and user ignore rules remain excluded --compat-remote-admin-exit-zero Preserve the former zero exit code when remote-administration verification fails @@ -215,6 +218,7 @@ export function parseArgs(standardIo: StandardIo = createNodeStandardIo()): CLIO let force = false; let writeSettings = false; let compatRemoteAdminExitZero = false; + let includeDbLikeDirs = false; let interval: number | undefined; let command: CLICommand = "daemon"; const commandArgs: string[] = []; @@ -277,6 +281,9 @@ export function parseArgs(standardIo: StandardIo = createNodeStandardIo()): CLIO case "--compat-remote-admin-exit-zero": compatRemoteAdminExitZero = true; break; + case "--include-db-like-dirs": + includeDbLikeDirs = true; + break; default: { if (!databasePath) { if (command === "daemon" && isCLICommand(token)) { @@ -319,6 +326,7 @@ export function parseArgs(standardIo: StandardIo = createNodeStandardIo()): CLIO force, writeSettings, compatRemoteAdminExitZero, + includeDbLikeDirs, command, commandArgs, interval, @@ -469,6 +477,13 @@ export async function main( await ignoreRules.load(); } + // Derive protection from the canonical startup paths and share it with both + // target checks and watching. The runtime opt-in never changes the protected DB root. + const pathFilterOptions: CLIPathFilterOptions = { + includeDbLikeDirs: options.includeDbLikeDirs, + databaseRoot: toVaultRelativePath(vaultPath, databasePath), + }; + // Create service context and hub const context = new NodeServiceContext(databasePath, standardIo, vaultPath); const serviceHubInstance = new NodeServiceHub(databasePath, context); @@ -566,7 +581,8 @@ export async function main( core, serviceHub, ignoreRules, - vaultSyncMode?.watchFiles ?? false + vaultSyncMode?.watchFiles ?? false, + pathFilterOptions ); }, (core) => [], @@ -580,21 +596,20 @@ export async function main( p2pReplicator = useP2PReplicatorFeature(core, undefined, undefined, { prepareP2PSettings: useP2PSettingsPreparation(core.services.API.webCompatFetch.bind(core.services.API)), }); - // Add target filter to prevent internal files are handled + // Logical targets are already Vault-relative. Validate without filesystem + // resolution, then use the same exclusions as startup scans and live events. core.services.vault.isTargetFile.addHandler(async (target) => { const targetPath = stripAllPrefixes(getPathFromUXFileInfo(target)); - const parts = targetPath.split("/"); - if (isHiddenPath(targetPath)) { - return await Promise.resolve(false); + let relativePath: FilePath; + try { + relativePath = parseFilePath(targetPath); + } catch { + return false; } - // PouchDB LevelDB database directory lives in the vault directory. - if (parts[0]?.endsWith("-livesync-v2")) { - return await Promise.resolve(false); - } - return await Promise.resolve(true); + return !isIgnoredFilePath(relativePath, undefined, pathFilterOptions); }, -1 /* highest priority */); - // Apply user-defined ignore rules after the dotfile check. + // Keep user rules as a later veto; they cannot re-admit paths rejected by the shared policy. if (ignoreRules) { const rules = ignoreRules; core.services.vault.isTargetFile.addHandler(async (target) => { @@ -602,7 +617,7 @@ export async function main( if (rules.shouldIgnore(targetPath)) { return false; } - // At least this handler think it is a target file, but other handlers may still veto it. + // This handler admits the path, but later handlers may still veto it. return true; }, 0); } diff --git a/src/apps/cli/main.unit.spec.ts b/src/apps/cli/main.unit.spec.ts index 29d6fc78..e19ba1ae 100644 --- a/src/apps/cli/main.unit.spec.ts +++ b/src/apps/cli/main.unit.spec.ts @@ -72,9 +72,25 @@ describe("CLI parseArgs", () => { expect(combined).toContain("Usage:"); expect(combined).toContain("livesync-cli [options] [command-args]"); expect(combined).toContain("--compat-remote-admin-exit-zero"); + expect(combined).toContain("--include-db-like-dirs"); expect(combined).toContain("journal-skip "); }); + it.each([false, true])("parses the DB-like directory compatibility option (%s)", (enabled) => { + process.argv = [ + "node", + "livesync-cli", + "./databasePath", + ...(enabled ? ["--include-db-like-dirs"] : []), + "daemon", + ]; + mockProcessExit(); + const parsed = parseArgs(); + expect(parsed.includeDbLikeDirs).toBe(enabled); + expect(parsed.command).toBe("daemon"); + expect(parsed.commandArgs).toEqual([]); + }); + it("parses p2p-peers command and timeout", () => { process.argv = ["node", "livesync-cli", "./databasePath", "p2p-peers", "5"]; const parsed = parseArgs(); diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts index a5513585..f4f1ba16 100644 --- a/src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdir, mkdtemp, readdir, rename, rm, unlink, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, symlink, unlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -66,6 +66,118 @@ for (let directoryIndex = 0; directoryIndex < 40; directoryIndex++) { `; describe("CLI filesystem watcher with a real filesystem", () => { + it.each([ + { name: "an ordinary Vault", parent: "notes", vault: "vault", alias: false, resolveAlias: true }, + { + name: "an ordinary Vault through a symlink", + parent: "notes", + vault: "vault", + alias: true, + resolveAlias: true, + }, + { + name: "a hidden-parent Vault through a preserved symlink", + parent: ".obsidian", + vault: "vault", + alias: true, + resolveAlias: false, + }, + { + name: "a hidden-parent Vault by physical path", + parent: ".obsidian", + vault: "vault", + alias: false, + resolveAlias: true, + }, + { + name: "a hidden-parent Vault through a resolved symlink", + parent: ".obsidian", + vault: "vault", + alias: true, + resolveAlias: true, + }, + { name: "a dot-prefixed Vault root", parent: "notes", vault: ".vault", alias: false, resolveAlias: true }, + { + name: "a Vault below a database-like parent", + parent: "archive-livesync-v2", + vault: "vault", + alias: false, + resolveAlias: true, + }, + { + name: "a database-like Vault root", + parent: "notes", + vault: "vault-livesync-v2", + alias: false, + resolveAlias: true, + }, + ])( + "keeps monitoring files in $name", + async ({ parent, vault, alias, resolveAlias }) => { + const fixture = await mkdtemp(join(tmpdir(), "livesync-watch-root-")); + const physicalVault = join(fixture, parent, vault); + const aliasPath = join(fixture, "livesync-vault"); + const events: WatchEvent[] = []; + const diagnostics: unknown[] = []; + await mkdir(physicalVault, { recursive: true }); + await symlink(physicalVault, aliasPath, "dir"); + const suppliedPath = alias ? aliasPath : physicalVault; + const watchRoot = resolveAlias ? await realpath(suppliedPath) : suppliedPath; + const adapter = new CLIStorageEventManagerAdapter(watchRoot, undefined, true, (message, detail) => + diagnostics.push({ message, detail }) + ); + + try { + await adapter.watch.beginWatch(makeHandlers(events)); + const noteName = "new-note.md"; + const notePath = join(physicalVault, noteName); + await writeFile(notePath, "created after watcher readiness"); + const created = await waitFor(() => + events.some((event) => event.type === "create" && !event.folder && event.path === noteName) + ); + expect(await readFile(notePath, "utf8")).toBe("created after watcher readiness"); + expect(diagnostics).toEqual([]); + expect(created, `Expected a new-file event below ${watchRoot}`).toBe(true); + + await writeFile(notePath, "modified after receiving the creation event"); + expect( + await waitFor(() => events.some((event) => event.type === "change" && event.path === noteName)) + ).toBe(true); + await unlink(notePath); + expect( + await waitFor(() => events.some((event) => event.type === "delete" && event.path === noteName)) + ).toBe(true); + } finally { + await adapter.close(); + await rm(fixture, { recursive: true, force: true }); + } + }, + 20_000 + ); + + it("preserves monitoring through a descendant symlink to another directory", async () => { + const fixture = await mkdtemp(join(tmpdir(), "livesync-watch-child-link-")); + const root = join(fixture, "vault"); + const target = join(fixture, "external"); + await mkdir(root); + await mkdir(target); + await symlink(target, join(root, "linked"), "dir"); + const events: WatchEvent[] = []; + const adapter = new CLIStorageEventManagerAdapter(root, undefined, true); + try { + await adapter.watch.beginWatch(makeHandlers(events)); + await writeFile(join(target, "linked-note.md"), "written through the physical target"); + expect( + await waitFor(() => + events.some((event) => event.type === "create" && event.path === "linked/linked-note.md") + ) + ).toBe(true); + } finally { + await adapter.close(); + await rm(fixture, { recursive: true, force: true }); + } + }, 15_000); + it("reports all 400 files when another process creates directories before dot-temp renames", async () => { const root = await mkdtemp(join(tmpdir(), "livesync-issue1328-burst-")); const events: WatchEvent[] = []; @@ -111,40 +223,48 @@ describe("CLI filesystem watcher with a real filesystem", () => { } }, 30_000); - it("ignores initial, hidden, and custom-ignored files while watching visible files", async () => { - const root = await mkdtemp(join(tmpdir(), "livesync-real-watch-ignore-")); - const events: WatchEvent[] = []; - const ignoreDirectory = join(root, ".livesync"); - await mkdir(ignoreDirectory); - await writeFile(join(ignoreDirectory, "ignore"), "private/\n"); - const ignoreRules = new IgnoreRules(root); - await ignoreRules.load(); - await mkdir(join(root, "private")); - await writeFile(join(root, "present-before-watch.md"), "initial file"); - await writeFile(join(root, ".hidden-before-watch.md"), "initial hidden file"); - const adapter = new CLIStorageEventManagerAdapter(root, ignoreRules, true); + it.each(["visible-parent", ".hidden-parent"])( + "ignores hidden, internal, and custom-ignored entries below %s", + async (parent) => { + const fixture = await mkdtemp(join(tmpdir(), "livesync-real-watch-ignore-")); + const root = join(fixture, parent, "vault"); + await mkdir(root, { recursive: true }); + const events: WatchEvent[] = []; + const ignoreDirectory = join(root, ".livesync"); + await mkdir(ignoreDirectory); + await writeFile(join(ignoreDirectory, "ignore"), "private/\n"); + const ignoreRules = new IgnoreRules(root); + await ignoreRules.load(); + await mkdir(join(root, "private")); + await mkdir(join(root, "local-livesync-v2")); + await writeFile(join(root, "present-before-watch.md"), "initial file"); + await writeFile(join(root, ".hidden-before-watch.md"), "initial hidden file"); + const adapter = new CLIStorageEventManagerAdapter(root, ignoreRules, true); - try { - await adapter.watch.beginWatch(makeHandlers(events)); - await writeFile(join(root, "visible-after-watch.md"), "visible"); - await writeFile(join(root, ".hidden-after-watch.md"), "hidden"); - await writeFile(join(root, "private", "ignored-after-watch.md"), "custom ignored"); + try { + await adapter.watch.beginWatch(makeHandlers(events)); + await writeFile(join(root, "visible-after-watch.md"), "visible"); + await writeFile(join(root, ".hidden-after-watch.md"), "hidden"); + await writeFile(join(root, "private", "ignored-after-watch.md"), "custom ignored"); + await writeFile(join(root, "local-livesync-v2", "state"), "internal database"); - expect( - await waitFor(() => - events.some((event) => event.type === "create" && event.path === "visible-after-watch.md") - ) - ).toBe(true); - await delay(700); + expect( + await waitFor(() => + events.some((event) => event.type === "create" && event.path === "visible-after-watch.md") + ) + ).toBe(true); + await delay(700); - expect(events.filter((event) => !event.folder)).toEqual([ - { type: "create", path: "visible-after-watch.md", folder: false }, - ]); - } finally { - await adapter.close(); - await rm(root, { recursive: true, force: true }); - } - }, 15_000); + expect(events.filter((event) => !event.folder)).toEqual([ + { type: "create", path: "visible-after-watch.md", folder: false }, + ]); + } finally { + await adapter.close(); + await rm(fixture, { recursive: true, force: true }); + } + }, + 15_000 + ); it("reports changes, deletions, rename paths, and stops after close", async () => { const root = await mkdtemp(join(tmpdir(), "livesync-real-watch-lifecycle-")); diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts index 94e26165..bea7e5d3 100644 --- a/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts @@ -16,6 +16,12 @@ import { fsPromises as fs, getNodeFileCreationTime, path, type Stats } from "@vr import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput"; import { NodeTypeGuardAdapter } from "@/apps/cli/adapters/NodeTypeGuardAdapter"; import { writeFileAtomically } from "@/apps/cli/runtimeState"; +import { + isIgnoredVaultPath, + toVaultRelativePath, + type VaultRelativePath, + type CLIPathFilterOptions, +} from "@/apps/cli/cliPathFilter"; /** * CLI-specific persistence adapter (file-based snapshot) @@ -86,18 +92,43 @@ class CLIConverterAdapter implements IStorageEventConverterAdapter { * CLI-specific watch adapter using chokidar for real-time filesystem monitoring. */ class CLIWatchAdapter implements IStorageEventWatchAdapter { + /** Retain the handle owned by this adapter so normal unload can close its watcher. */ private _watcher: FSWatcher | undefined; constructor( - private basePath: string, + private readonly basePath: string, private ignoreRules?: IgnoreRules, private watchEnabled: boolean = false, - private reportDiagnostic: CliDiagnosticReporter = () => undefined + private reportDiagnostic: CliDiagnosticReporter = () => undefined, + private readonly pathFilterOptions: CLIPathFilterOptions = {} ) {} - private _toNodeFile(filePath: string, stats: Stats | undefined): NodeFile { + // Bind conversion to the selected root; event callbacks cannot substitute a different root. + private _toVaultRelativePath(candidatePath: string): VaultRelativePath | undefined { + return toVaultRelativePath(this.basePath, candidatePath); + } + + // Unlike an ignored-callback probe, an invalid emitted event is unexpected. + // Report it and let the caller omit that event while later valid events continue. + private _getEventPath(candidatePath: string): VaultRelativePath | undefined { + const relativePath = this._toVaultRelativePath(candidatePath); + if (relativePath === undefined) { + this.reportDiagnostic( + "[CLIWatchAdapter] File watcher event was ignored because its path is not valid within the selected Vault.", + { path: candidatePath, vaultPath: this.basePath } + ); + } + return relativePath; + } + + // Deleted files still have a lexical event path, so conversion must not require filesystem existence. + private _toNodeFile(filePath: string, stats: Stats | undefined): NodeFile | false { + const relativePath = this._getEventPath(filePath); + if (relativePath === undefined) { + return false; + } return { - path: path.relative(this.basePath, filePath).replace(/\\/g, "/") as FilePath, + path: relativePath, stat: { // Floor to integer milliseconds; Linux fs.Stats.*Ms carry sub-millisecond // precision, and timestamps are stored as integer ms everywhere else. @@ -109,25 +140,28 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter { }; } - private _toNodeFolder(dirPath: string): NodeFolder { + private _toNodeFolder(dirPath: string): NodeFolder | false { + const relativePath = this._getEventPath(dirPath); + // The empty relative path is a valid root-folder event, so test the failure sentinel explicitly. + if (relativePath === undefined) { + return false; + } return { - path: path.relative(this.basePath, dirPath).replace(/\\/g, "/") as FilePath, + path: relativePath, isFolder: true, }; } async beginWatch(handlers: IStorageEventWatchHandlers): Promise { if (!this.watchEnabled) return; - const baseIgnored: Array boolean)> = [ - /(^|[/\\])\./, - /(^|[/\\])[^/\\]*-livesync-v2([/\\]|$)/, - ]; - // Bind rules to a local const before the closure — chokidar v4 requires a - // MatchFunction, not glob strings, for custom patterns. - const rules = this.ignoreRules; - const ignored = rules - ? [...baseIgnored, (p: string) => rules.shouldIgnore(path.relative(this.basePath, p))] - : baseIgnored; + // Chokidar may probe ancestors of the root; exclude those candidates quietly. + // Entry exclusions see only the rebased path, preserving Vaults beneath hidden parents. + const ignored = (candidatePath: string): boolean => { + const relativePath = this._toVaultRelativePath(candidatePath); + return ( + relativePath === undefined || isIgnoredVaultPath(relativePath, this.ignoreRules, this.pathFilterOptions) + ); + }; const watcher = chokidarWatch(this.basePath, { ignored, @@ -141,27 +175,37 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter { watcher.on("add", (filePath, stats) => { const nodeFile = this._toNodeFile(filePath, stats); - handlers.onCreate(nodeFile); + if (nodeFile !== false) { + handlers.onCreate(nodeFile); + } }); watcher.on("change", (filePath, stats) => { const nodeFile = this._toNodeFile(filePath, stats); - handlers.onChange(nodeFile); + if (nodeFile !== false) { + handlers.onChange(nodeFile); + } }); watcher.on("unlink", (filePath) => { const nodeFile = this._toNodeFile(filePath, undefined); - handlers.onDelete(nodeFile); + if (nodeFile !== false) { + handlers.onDelete(nodeFile); + } }); watcher.on("addDir", (dirPath) => { const nodeFolder = this._toNodeFolder(dirPath); - handlers.onCreate(nodeFolder); + if (nodeFolder !== false) { + handlers.onCreate(nodeFolder); + } }); watcher.on("unlinkDir", (dirPath) => { const nodeFolder = this._toNodeFolder(dirPath); - handlers.onDelete(nodeFolder); + if (nodeFolder !== false) { + handlers.onDelete(nodeFolder); + } }); watcher.on("error", (err) => { @@ -173,6 +217,7 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter { process.exit(1); }); + // Complete watch startup only after Chokidar has finished its initial discovery. await new Promise((resolve) => watcher.once("ready", resolve)); this._watcher = watcher; } @@ -200,11 +245,12 @@ export class CLIStorageEventManagerAdapter implements IStorageEventManagerAdapte ignoreRules?: IgnoreRules, watchEnabled: boolean = false, reportDiagnostic: CliDiagnosticReporter = () => undefined, - snapshotPath?: () => string + snapshotPath?: () => string, + pathFilterOptions?: CLIPathFilterOptions ) { this.typeGuard = new NodeTypeGuardAdapter(); this.persistence = new CLIPersistenceAdapter(basePath, reportDiagnostic, snapshotPath); - this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled, reportDiagnostic); + this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled, reportDiagnostic, pathFilterOptions); this.status = new CLIStatusAdapter(); this.converter = new CLIConverterAdapter(); } diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts index 9437e74b..6b3a1342 100644 --- a/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts @@ -67,6 +67,69 @@ describe("CLIStorageEventManagerAdapter", () => { } }); + it("filters only checked Vault-relative candidates and keeps its root watchable", async () => { + const base = path.resolve(path.sep, ".parent", "vault-livesync-v2"); + const reporter = vi.fn(); + const adapter = new CLIStorageEventManagerAdapter(base, undefined, true, reporter); + await adapter.watch.beginWatch(makeHandlers()); + const options = vi.mocked(chokidar.watch).mock.calls[0][1]; + const ignored = options?.ignored; + if (typeof ignored !== "function") { + throw new Error("Expected the typed watcher filter"); + } + expect(ignored(base)).toBe(false); + expect(ignored(path.join(base, "note.md"))).toBe(false); + expect(ignored(path.join(base, ".hidden.md"))).toBe(true); + expect(ignored(path.join(base, "local-livesync-v2", "state"))).toBe(true); + expect(ignored(path.dirname(base))).toBe(true); + expect(ignored(path.join(base + "-sibling", "note.md"))).toBe(true); + expect(ignored("relative.md")).toBe(true); + expect(reporter).not.toHaveBeenCalled(); + }); + + it.each([ + ["add", "onCreate"], + ["change", "onChange"], + ["unlink", "onDelete"], + ["addDir", "onCreate"], + ["unlinkDir", "onDelete"], + ] as const)("omits an invalid %s event and still delivers the next valid event", async (event, handler) => { + const base = path.resolve(path.sep, "vault"); + const reporter = vi.fn(); + const adapter = new CLIStorageEventManagerAdapter(base, undefined, true, reporter); + const handlers = makeHandlers(); + await adapter.watch.beginWatch(handlers); + const registration = mockWatcher.on.mock.calls.find(([name]) => name === event); + const callback = registration?.[1] as ((candidate: string) => void) | undefined; + if (!callback) { + throw new Error(`Expected the ${event} handler`); + } + const outside = path.resolve(base, "..", "outside"); + callback(outside); + expect(handlers[handler]).not.toHaveBeenCalled(); + expect(reporter).toHaveBeenCalledWith(expect.stringContaining("ignored"), { + path: outside, + vaultPath: base, + }); + + callback(path.join(base, "valid")); + expect(handlers[handler]).toHaveBeenCalledTimes(1); + expect(handlers[handler]).toHaveBeenCalledWith(expect.objectContaining({ path: "valid" })); + expect(reporter).toHaveBeenCalledTimes(1); + }); + + it("preserves folder events for the Vault root", async () => { + const base = path.resolve(path.sep, "vault"); + const adapter = new CLIStorageEventManagerAdapter(base, undefined, true); + const handlers = makeHandlers(); + await adapter.watch.beginWatch(handlers); + const callback = mockWatcher.on.mock.calls.find(([event]) => event === "unlinkDir")?.[1] as ( + candidate: string + ) => void; + callback(base); + expect(handlers.onDelete).toHaveBeenCalledWith(expect.objectContaining({ path: "", isFolder: true })); + }); + it("beginWatch is no-op when watchEnabled=false", async () => { const adapter = new CLIStorageEventManagerAdapter("/base", undefined, false); const handlers = makeHandlers(); diff --git a/src/apps/cli/managers/StorageEventManagerCLI.ts b/src/apps/cli/managers/StorageEventManagerCLI.ts index 90bba6af..8da276b2 100644 --- a/src/apps/cli/managers/StorageEventManagerCLI.ts +++ b/src/apps/cli/managers/StorageEventManagerCLI.ts @@ -6,11 +6,11 @@ import { CLIStorageEventManagerAdapter } from "./CLIStorageEventManagerAdapter"; import type { IMinimumLiveSyncCommands, LiveSyncBaseCore } from "@/LiveSyncBaseCore"; import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules"; +import type { CLIPathFilterOptions } from "@/apps/cli/cliPathFilter"; import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; import { runtimeScopeDirectory } from "@/apps/cli/runtimeState"; import type { NodeServiceContext } from "@/apps/cli/services/NodeServiceContext"; import { path as nodePath } from "@vrtmrz/livesync-commonlib/node"; -// import type { IMinimumLiveSyncCommands } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; export class StorageEventManagerCLI extends StorageEventManagerBase { core: LiveSyncBaseCore; @@ -20,7 +20,8 @@ export class StorageEventManagerCLI extends StorageEventManagerBase, dependencies: StorageEventManagerBaseDependencies, ignoreRules?: IgnoreRules, - watchEnabled?: boolean + watchEnabled?: boolean, + pathFilterOptions?: CLIPathFilterOptions ) { const context = core.services.context as NodeServiceContext; const adapter = new CLIStorageEventManagerAdapter( @@ -33,23 +34,22 @@ export class StorageEventManagerCLI extends StorageEventManagerBase nodePath.join( runtimeScopeDirectory(context.databasePath, dependencies.vaultService.getVaultName(), basePath), "storage-events.json" - ) + ), + pathFilterOptions ); super(adapter, dependencies); this.core = core; } - /** - * Override _watchVaultRawEvents for CLI-specific logic - * In CLI, we don't have internal files like Obsidian's .obsidian folder - */ + /** The CLI uses shared exclusions instead of the plug-in's raw internal-file event processing. */ protected override async _watchVaultRawEvents(path: string) { - // No-op in CLI version - // Internal file handling is not needed + // Hidden and internal entries are intentionally not queued as raw Vault events. } /** diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index fa0dd251..c51fe744 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -15,7 +15,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/runtimeState.unit.spec.ts src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/services/storageCaseSensitivity.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/journalSkip.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/cliPathFilter.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/runtimeState.unit.spec.ts src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/services/storageCaseSensitivity.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.real-watch.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/journalSkip.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", diff --git a/src/apps/cli/serviceModules/CLIServiceModules.ts b/src/apps/cli/serviceModules/CLIServiceModules.ts index 53cdf12f..0120285b 100644 --- a/src/apps/cli/serviceModules/CLIServiceModules.ts +++ b/src/apps/cli/serviceModules/CLIServiceModules.ts @@ -8,23 +8,27 @@ import { ServiceDatabaseFileAccessCLI } from "./DatabaseFileAccess"; import { StorageEventManagerCLI } from "@/apps/cli/managers/StorageEventManagerCLI"; import type { ServiceModules } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; import type { IgnoreRules } from "./IgnoreRules"; +import type { CLIPathFilterOptions } from "@/apps/cli/cliPathFilter"; import { composeSharedServiceModules } from "@/apps/serviceModules/composeSharedServiceModules"; /** - * Initialize service modules for CLI version - * This is the CLI equivalent of ObsidianLiveSyncPlugin.initialiseServiceModules + * Compose CLI storage, watching, and database access with the shared service modules. * - * @param basePath - The base path of the vault directory + * @param basePath - Selected Vault root used for storage and event paths * @param core - The LiveSyncBaseCore instance * @param services - The service hub - * @returns ServiceModules containing all initialized service modules + * @param ignoreRules - User exclusions loaded before watch startup + * @param watchEnabled - Whether this command owns a live filesystem watcher + * @param pathFilterOptions - The same runtime exclusions used by logical target checks + * @returns CLI service modules bound to this Vault */ export function initialiseServiceModulesCLI( basePath: string, core: LiveSyncBaseCore, services: InjectableServiceHub, ignoreRules?: IgnoreRules, - watchEnabled: boolean = false + watchEnabled: boolean = false, + pathFilterOptions?: CLIPathFilterOptions ): ServiceModules { const storageAccessManager = new StorageAccessManager(); @@ -37,7 +41,7 @@ export function initialiseServiceModulesCLI( pathService: services.path, }); - // CLI-specific storage event manager + // Live monitoring receives the same runtime policy as the core target checks. const storageEventManager = new StorageEventManagerCLI( basePath, core, @@ -49,7 +53,8 @@ export function initialiseServiceModulesCLI( APIService: services.API, }, ignoreRules, - watchEnabled + watchEnabled, + pathFilterOptions ); // Close the file watcher during graceful shutdown so the process can exit cleanly. diff --git a/test/README.md b/test/README.md index 241eb969..21382b87 100644 --- a/test/README.md +++ b/test/README.md @@ -77,6 +77,41 @@ npm run test:e2e:docker:mirror --workspace self-hosted-livesync-cli The expected result is `PASS=7 FAIL=0`. These mirror suites use temporary local databases and need no CouchDB service. The complete `test:e2e:docker:all` command also runs the other Docker CLI suites and manages a disposable CouchDB fixture; use it to check the full Docker CI gate. Keep the ordinary-edit and unknown-provenance scenarios aligned between the Node.js and Bash suites. +## CLI daemon watcher path regression + +Build the CLI, then run the focused task with Node.js 26 or later and Docker +available: + +```bash +NODE_OPTIONS=--max-old-space-size=3072 npm run build --workspace self-hosted-livesync-cli +node test/e2e-cli/run-task.ts test:daemon-watch-paths +``` + +The task starts the disposable `couchdb-test` container, creates a unique +database, and removes the container during teardown. Docker, the `couchdb-test` +container name, and host port `5989` must be available. The CLI `.test.env` file supplies +the same CouchDB connection variables used by the other CLI CouchDB tests; +local test defaults apply when they are absent. + +The test seeds two revisions for a baseline, runs `fetch` into a Vault under a +hidden `.obsidian` parent through a visible symlink, and starts the actual CLI +daemon. An independent CLI client confirms exact remote file bytes and deletion +after create, modify, and delete events. The current baseline Metadata revision +and bytes are checked on the fetched client and observer; its original revision +remains readable on the source and is checked in CouchDB's revision history. The +test also checks the default hidden-entry and `.livesync/ignore` exclusions, +then restarts the daemon with the physical Vault path and verifies the saved +database suffix, runtime scope, and further monitoring. + +All three CLI clients use the same encrypted CouchDB profile and explicit test +passphrase. Chunk retrieval is configured to permit remote reads. + +The test also creates an ordinary nested `archive-livesync-v2` directory. It +checks that the default policy keeps its file out of the independently +replicated database, then restarts both daemons with `--include-db-like-dirs` +and verifies its initial bytes, later modifications, and deletion. Local +database inspection occurs only after the corresponding daemon has stopped. + ## CLI Fetch with on-demand chunks Build the CLI, then run `node test/e2e-cli/run-task.ts test:fetch-on-demand-chunks` diff --git a/test/e2e-cli/ci-tasks.json b/test/e2e-cli/ci-tasks.json index 1c998856..205a62b6 100644 --- a/test/e2e-cli/ci-tasks.json +++ b/test/e2e-cli/ci-tasks.json @@ -6,6 +6,7 @@ "test:mirror", "test:daemon", "test:daemon-startup", + "test:daemon-watch-paths", "test:push-pull", "test:decoupled-vault", "test:sync-two-local", diff --git a/test/e2e-cli/tasks.json b/test/e2e-cli/tasks.json index d1cfdd91..07f3d803 100644 --- a/test/e2e-cli/tasks.json +++ b/test/e2e-cli/tasks.json @@ -20,6 +20,9 @@ "test:daemon-startup": { "test": ["test-daemon-startup.ts"] }, + "test:daemon-watch-paths": { + "test": ["test-daemon-watch-paths.ts"] + }, "test:decoupled-vault": { "test": ["test-decoupled-vault.ts"] }, diff --git a/test/e2e-cli/test-daemon-watch-paths.ts b/test/e2e-cli/test-daemon-watch-paths.ts new file mode 100644 index 00000000..9e696251 --- /dev/null +++ b/test/e2e-cli/test-daemon-watch-paths.ts @@ -0,0 +1,616 @@ +import assert from "node:assert/strict"; +import { mkdir, readFile, readdir, realpath, stat, symlink, unlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { test } from "node:test"; +import { startCliInBackground, type BackgroundCliProcess } from "./helpers/backgroundCli.ts"; +import { runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; +import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; +import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; +import { TempDir } from "./helpers/temp.ts"; + +const INITIAL_REPLICATION_COMPLETE = "[Daemon] Initial replication complete"; +const LIVESYNC_ACTIVE = "LiveSync active"; +const TEST_PASSPHRASE = "issue-1340-couchdb-test-passphrase"; + +interface MetadataInfo { + id: string; + revision: string; +} + +function envOrDefault(keys: string[], fallback: string): string { + for (const key of keys) { + const value = process.env[key]?.trim(); + if (value) return value; + } + return fallback; +} + +function waitForTick(): Promise { + return new Promise((resolve) => setTimeout(resolve, 100)); +} + +async function setChunkRetrievalSettings(settingsPath: string): Promise { + const settings = JSON.parse(await readFile(settingsPath, "utf8")) as Record; + settings.readChunksOnline = true; + settings.useOnlyLocalChunk = false; + await writeFile(settingsPath, JSON.stringify(settings, null, 2)); +} + +async function readMetadataInfo( + databasePath: string, + vaultPath: string, + settingsPath: string, + filePath: string +): Promise { + const infoText = await runCliOrFail( + databasePath, + "--vault", + vaultPath, + "--settings", + settingsPath, + "info", + filePath + ); + const info = JSON.parse(infoText) as { id?: unknown; revision?: unknown }; + assert.equal(typeof info.id, "string", "Metadata ID is missing for " + filePath + ": " + infoText); + assert.equal(typeof info.revision, "string", "Metadata revision is missing for " + filePath + ": " + infoText); + return { + id: info.id as string, + revision: info.revision as string, + }; +} + +async function readDatabaseSuffix(settingsPath: string): Promise { + const settings = JSON.parse(await readFile(settingsPath, "utf8")) as Record; + const suffix = settings.additionalSuffixOfDatabaseName; + assert.equal(typeof suffix, "string", "Fetch did not save a selected local database suffix"); + assert.notEqual(suffix, "", "Fetch saved an empty local database suffix"); + return suffix as string; +} + +async function waitForFileBytes(filePath: string, expected: Uint8Array, timeoutMs = 45_000): Promise { + const deadline = Date.now() + timeoutMs; + let actual: Uint8Array | undefined; + while (Date.now() < deadline) { + try { + actual = await readFile(filePath); + if (Buffer.compare(actual, expected) === 0) return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await waitForTick(); + } + throw new Error( + "Timed out waiting for exact bytes at " + filePath + "; actual=" + JSON.stringify(actual?.toString()) + ); +} + +async function waitForMissing(filePath: string, timeoutMs = 45_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + await stat(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + await waitForTick(); + } + throw new Error("Timed out waiting for " + filePath + " to be removed from the observer Vault"); +} + +async function assertMissing(filePath: string, message: string): Promise { + try { + await stat(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + assert.fail(message); +} + +/** Return DB/Vault owner identities so restart checks can detect a second state scope. */ +async function readRuntimeScopePaths(databasePath: string): Promise { + const scopesRoot = join(databasePath, ".livesync", "runtime", "scopes"); + let databaseDirectories; + try { + databaseDirectories = await readdir(scopesRoot, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + + const scopes: string[] = []; + for (const databaseDirectory of databaseDirectories) { + if (!databaseDirectory.isDirectory()) continue; + const databaseScopePath = join(scopesRoot, databaseDirectory.name); + const vaultDirectories = await readdir(databaseScopePath, { withFileTypes: true }); + for (const vaultDirectory of vaultDirectories) { + if (vaultDirectory.isDirectory()) { + scopes.push(join(databaseDirectory.name, vaultDirectory.name)); + } + } + } + return scopes.sort(); +} + +async function assertDatabaseBytes( + databasePath: string, + vaultPath: string, + settingsPath: string, + filePath: string, + expected: Uint8Array, + message: string, + revision?: string +): Promise { + const command = revision ? ["cat-rev", filePath, revision] : ["cat", filePath]; + const output = await runCliOrFail(databasePath, "--vault", vaultPath, "--settings", settingsPath, ...command); + assert.deepEqual(Buffer.from(sanitiseCatStdout(output), "utf8"), expected, message); +} + +// Call only after stopping the daemon. Inspect stored Metadata so reflection +// exclusions cannot hide an accidental upload from a Vault-only assertion. +async function readDatabasePaths(databasePath: string, vaultPath: string, settingsPath: string): Promise> { + const listing = await runCliOrFail(databasePath, "--vault", vaultPath, "--settings", settingsPath, "ls"); + return new Set( + listing + .split(/\r?\n/) + .map((line) => line.split("\t", 1)[0]) + .filter((filePath) => filePath !== undefined && filePath !== "") + ); +} + +async function assertCouchdbRevisionAncestor( + couchdbUri: string, + couchdbUser: string, + couchdbPassword: string, + dbname: string, + documentId: string, + currentRevision: string, + ancestorRevision: string +): Promise { + const response = await fetch( + couchdbUri + "/" + encodeURIComponent(dbname) + "/" + encodeURIComponent(documentId) + "?revs=true", + { + headers: { + Authorization: "Basic " + Buffer.from(couchdbUser + ":" + couchdbPassword).toString("base64"), + }, + } + ); + const responseText = await response.text(); + assert.equal(response.status, 200, "CouchDB must return the seeded baseline Metadata: " + responseText); + const document = JSON.parse(responseText) as { + _rev?: unknown; + _revisions?: { start?: unknown; ids?: unknown }; + }; + assert.equal(document._rev, currentRevision, "CouchDB must retain the selected baseline revision"); + + const ancestorMatch = /^(\d+)-(.+)$/.exec(ancestorRevision); + assert.ok(ancestorMatch, "Original revision must include its generation and hash"); + const ancestorGeneration = Number(ancestorMatch[1]); + const history = document._revisions; + assert.ok(history, "CouchDB response must include the revision history"); + assert.equal(typeof history.start, "number", "CouchDB revision history must include its starting generation"); + assert.ok(Array.isArray(history.ids), "CouchDB revision history must include its revision hashes"); + const ancestorIndex = (history.start as number) - ancestorGeneration; + assert.ok( + Number.isSafeInteger(ancestorIndex) && ancestorIndex >= 0 && ancestorIndex < history.ids.length, + "CouchDB revision history must include the original baseline generation" + ); + assert.equal( + history.ids[ancestorIndex], + ancestorMatch[2], + "CouchDB revision history must retain the original baseline revision" + ); +} + +async function stopDaemon(daemon: BackgroundCliProcess | undefined): Promise { + if (!daemon) return; + await daemon.stop().catch(() => {}); +} + +test( + "daemon: monitors a hidden-parent Vault through a visible symlink and retains its scope after restart", + { timeout: 240_000 }, + async () => { + await using workDir = await TempDir.create("livesync-cli-daemon-watch-paths"); + + const couchdbUri = envOrDefault(["COUCHDB_URI", "hostname"], "http://127.0.0.1:5989").replace(/\/$/, ""); + const couchdbUser = envOrDefault(["COUCHDB_USER", "username"], "admin"); + const couchdbPassword = envOrDefault(["COUCHDB_PASSWORD", "password"], "testpassword"); + const dbPrefix = envOrDefault(["COUCHDB_DBNAME", "dbname"], "livesync-test-db-ci"); + const dbname = ( + dbPrefix + + "-daemon-watch-paths-" + + Date.now() + + "-" + + Math.floor(Math.random() * 1_000_000) + ).toLowerCase(); + + const sourceDatabase = workDir.join("source-database"); + const targetDatabase = workDir.join("target-database"); + const observerDatabase = workDir.join("observer-database"); + const sourceVault = workDir.join("source-vault"); + const physicalVault = workDir.join(".obsidian", "physical-vault"); + const visibleVault = workDir.join("visible-vault"); + const observerVault = workDir.join("observer-vault"); + const sourceSettings = workDir.join("source-settings.json"); + const targetSettings = workDir.join("target-settings.json"); + const observerSettings = workDir.join("observer-settings.json"); + + const baselinePath = "notes/baseline.md"; + const watchedPath = "notes/created-after-fetch.md"; + const restartedPath = "notes/created-after-restart.md"; + const dbLikePath = "notes/archive-livesync-v2/note.md"; + const hiddenPath = "notes/.private.md"; + const ignoredPath = "notes/ignored.ignored.md"; + const initialBytes = Buffer.from("Baseline revision one.\n", "utf8"); + const baselineBytes = Buffer.from("Baseline revision two with preserved ancestry.\n", "utf8"); + const createdBytes = Buffer.from("Created through the visible symlink.\n", "utf8"); + const modifiedBytes = Buffer.from("Modified through the visible symlink.\n", "utf8"); + const restartedBytes = Buffer.from("Created after restarting with the physical root.\n", "utf8"); + const dbLikeInitialBytes = Buffer.from("Nested DB-like path before option restart.\n", "utf8"); + const dbLikeModifiedBytes = Buffer.from("Nested DB-like path modified with the option.\n", "utf8"); + const targetBaseline = join(physicalVault, baselinePath); + const observerBaseline = join(observerVault, baselinePath); + const observerWatched = join(observerVault, watchedPath); + const observerRestarted = join(observerVault, restartedPath); + const observerDbLike = join(observerVault, dbLikePath); + const observerHidden = join(observerVault, hiddenPath); + const observerIgnored = join(observerVault, ignoredPath); + + await Promise.all([ + mkdir(sourceDatabase, { recursive: true }), + mkdir(targetDatabase, { recursive: true }), + mkdir(observerDatabase, { recursive: true }), + mkdir(sourceVault, { recursive: true }), + mkdir(physicalVault, { recursive: true }), + mkdir(observerVault, { recursive: true }), + ]); + await mkdir(join(physicalVault, ".livesync"), { recursive: true }); + await mkdir(join(physicalVault, "notes"), { recursive: true }); + await writeFile(join(physicalVault, ".livesync", "ignore"), "*.ignored.md\n"); + await symlink(physicalVault, visibleVault, process.platform === "win32" ? "junction" : "dir"); + assert.equal( + await realpath(visibleVault), + await realpath(physicalVault), + "Visible Vault path must resolve to the hidden physical Vault" + ); + + let targetDaemon: BackgroundCliProcess | undefined; + let observerDaemon: BackgroundCliProcess | undefined; + try { + await startCouchdb(couchdbUri, couchdbUser, couchdbPassword, dbname); + for (const settingsPath of [sourceSettings, targetSettings, observerSettings]) { + await initSettingsFile(settingsPath); + } + const encryptedCouchdbSettings = { + remoteType: "COUCHDB" as const, + couchdbUri, + couchdbUser, + couchdbPassword, + couchdbDbname: dbname, + encrypt: true, + passphrase: TEST_PASSPHRASE, + }; + await Promise.all([ + applyRemoteSyncSettings(sourceSettings, encryptedCouchdbSettings), + applyRemoteSyncSettings(targetSettings, encryptedCouchdbSettings), + applyRemoteSyncSettings(observerSettings, encryptedCouchdbSettings), + ]); + await Promise.all([ + setChunkRetrievalSettings(sourceSettings), + setChunkRetrievalSettings(targetSettings), + setChunkRetrievalSettings(observerSettings), + ]); + + await runCliWithInputOrFail( + initialBytes, + sourceDatabase, + "--vault", + sourceVault, + "--settings", + sourceSettings, + "put", + baselinePath + ); + await runCliOrFail(sourceDatabase, "--vault", sourceVault, "--settings", sourceSettings, "sync"); + // Check the old body on its retaining source. Fresh peers need the + // current contents and ancestry, without a guarantee of every historical body. + const originalInfo = await readMetadataInfo(sourceDatabase, sourceVault, sourceSettings, baselinePath); + await assertDatabaseBytes( + sourceDatabase, + sourceVault, + sourceSettings, + baselinePath, + initialBytes, + "Source must retain the original baseline revision before its update", + originalInfo.revision + ); + + await runCliWithInputOrFail( + baselineBytes, + sourceDatabase, + "--vault", + sourceVault, + "--settings", + sourceSettings, + "put", + baselinePath + ); + await runCliOrFail(sourceDatabase, "--vault", sourceVault, "--settings", sourceSettings, "sync"); + + const seedInfo = await readMetadataInfo(sourceDatabase, sourceVault, sourceSettings, baselinePath); + assert.equal( + seedInfo.id, + originalInfo.id, + "Second put must update the original baseline Metadata document" + ); + await assertDatabaseBytes( + sourceDatabase, + sourceVault, + sourceSettings, + baselinePath, + baselineBytes, + "Source must retain the current baseline revision bytes", + seedInfo.revision + ); + + await runCliOrFail( + targetDatabase, + "--vault", + visibleVault, + "--settings", + targetSettings, + "fetch", + "--force", + "--write-settings" + ); + assert.deepEqual(await readFile(targetBaseline), baselineBytes, "Fetch must preserve exact baseline bytes"); + assert.deepEqual( + await readMetadataInfo(targetDatabase, visibleVault, targetSettings, baselinePath), + seedInfo, + "Fetch must preserve the current baseline document and revision identity" + ); + await assertDatabaseBytes( + targetDatabase, + visibleVault, + targetSettings, + baselinePath, + baselineBytes, + "Fetch must preserve the current baseline revision bytes", + seedInfo.revision + ); + + targetDaemon = startCliInBackground( + targetDatabase, + "--vault", + visibleVault, + "--settings", + targetSettings, + "daemon" + ); + await targetDaemon.waitUntilContains(INITIAL_REPLICATION_COMPLETE, 45_000); + await targetDaemon.waitUntilContains(LIVESYNC_ACTIVE, 45_000); + + observerDaemon = startCliInBackground( + observerDatabase, + "--vault", + observerVault, + "--settings", + observerSettings, + "daemon" + ); + await observerDaemon.waitUntilContains(INITIAL_REPLICATION_COMPLETE, 45_000); + await observerDaemon.waitUntilContains(LIVESYNC_ACTIVE, 45_000); + await waitForFileBytes(observerBaseline, baselineBytes); + + const targetNotesViaSymlink = join(visibleVault, "notes"); + const targetNotesViaPhysicalRoot = join(physicalVault, "notes"); + await writeFile(join(targetNotesViaSymlink, ".private.md"), "Default hidden-entry exclusion.\n"); + await writeFile(join(targetNotesViaSymlink, "ignored.ignored.md"), "User ignore rule exclusion.\n"); + await mkdir(join(targetNotesViaSymlink, "archive-livesync-v2"), { recursive: true }); + await writeFile(join(targetNotesViaSymlink, "archive-livesync-v2", "note.md"), dbLikeInitialBytes); + await writeFile(join(targetNotesViaSymlink, "created-after-fetch.md"), createdBytes); + await waitForFileBytes(observerWatched, createdBytes); + assert.deepEqual( + await readFile(observerWatched), + createdBytes, + "Created note must reach the remote observer exactly" + ); + await assertMissing(observerHidden, "A dot-prefixed Vault entry must remain excluded"); + await assertMissing(observerIgnored, "An entry matched by .livesync/ignore must remain excluded"); + + await writeFile(join(targetNotesViaSymlink, "created-after-fetch.md"), modifiedBytes); + await waitForFileBytes(observerWatched, modifiedBytes); + assert.deepEqual( + await readFile(observerWatched), + modifiedBytes, + "Modified note must reach the remote observer exactly" + ); + + await unlink(join(targetNotesViaSymlink, "created-after-fetch.md")); + await waitForMissing(observerWatched); + + assert.equal(await targetDaemon.stop(), 0, targetDaemon.combined); + targetDaemon = undefined; + await stopDaemon(observerDaemon); + observerDaemon = undefined; + // Drain remote changes before inspecting the stopped observer's DB. + // A missing reflected file alone would not establish that upload was excluded. + await runCliOrFail(observerDatabase, "--vault", observerVault, "--settings", observerSettings, "sync"); + const defaultObserverPaths = await readDatabasePaths(observerDatabase, observerVault, observerSettings); + assert.equal( + defaultObserverPaths.has(dbLikePath), + false, + "Default policy must not upload a nested DB-like path to the remote database" + ); + assert.equal( + defaultObserverPaths.has(hiddenPath), + false, + "Default policy must exclude dot-prefixed entries" + ); + assert.equal( + defaultObserverPaths.has(ignoredPath), + false, + "Default policy must exclude entries matched by .livesync/ignore" + ); + + // Reuse the existing owners while changing both the root spelling and + // the naming-rule opt-in; the next phase must not create a new dataset or scope. + const selectedSuffix = await readDatabaseSuffix(targetSettings); + const scopesBeforeRestart = await readRuntimeScopePaths(targetDatabase); + assert.ok(scopesBeforeRestart.length > 0, "Alias startup must initialise a Vault runtime scope"); + + targetDaemon = startCliInBackground( + targetDatabase, + "--vault", + physicalVault, + "--settings", + targetSettings, + "--include-db-like-dirs", + "daemon" + ); + await targetDaemon.waitUntilContains(INITIAL_REPLICATION_COMPLETE, 45_000); + await targetDaemon.waitUntilContains(LIVESYNC_ACTIVE, 45_000); + assert.equal( + await readDatabaseSuffix(targetSettings), + selectedSuffix, + "Physical-root restart must retain the selected database suffix" + ); + + observerDaemon = startCliInBackground( + observerDatabase, + "--vault", + observerVault, + "--settings", + observerSettings, + "--include-db-like-dirs", + "daemon" + ); + await observerDaemon.waitUntilContains(INITIAL_REPLICATION_COMPLETE, 45_000); + await observerDaemon.waitUntilContains(LIVESYNC_ACTIVE, 45_000); + await waitForFileBytes(observerDbLike, dbLikeInitialBytes); + assert.deepEqual( + await readFile(observerDbLike), + dbLikeInitialBytes, + "Include option must replicate exact bytes for the previously excluded nested DB-like path" + ); + + await writeFile(join(targetNotesViaPhysicalRoot, "archive-livesync-v2", "note.md"), dbLikeModifiedBytes); + await waitForFileBytes(observerDbLike, dbLikeModifiedBytes); + assert.deepEqual( + await readFile(observerDbLike), + dbLikeModifiedBytes, + "Include option must monitor and replicate modifications under a nested DB-like directory" + ); + + await unlink(join(targetNotesViaPhysicalRoot, "archive-livesync-v2", "note.md")); + await waitForMissing(observerDbLike); + + await writeFile(join(targetNotesViaPhysicalRoot, "created-after-restart.md"), restartedBytes); + await waitForFileBytes(observerRestarted, restartedBytes); + assert.deepEqual( + await readFile(observerRestarted), + restartedBytes, + "Physical-root restart must continue monitoring and replicate exact bytes" + ); + assert.deepEqual( + await readRuntimeScopePaths(targetDatabase), + scopesBeforeRestart, + "Physical-root restart must reuse the existing Vault runtime scope" + ); + + assert.equal(await targetDaemon.stop(), 0, targetDaemon.combined); + targetDaemon = undefined; + await stopDaemon(observerDaemon); + observerDaemon = undefined; + await runCliOrFail(observerDatabase, "--vault", observerVault, "--settings", observerSettings, "sync"); + + for (const [databasePath, vaultPath, settingsPath] of [ + [targetDatabase, visibleVault, targetSettings], + [observerDatabase, observerVault, observerSettings], + ]) { + assert.deepEqual( + await readMetadataInfo(databasePath, vaultPath, settingsPath, baselinePath), + seedInfo, + "Live monitoring must preserve the current baseline document and revision identity" + ); + await assertDatabaseBytes( + databasePath, + vaultPath, + settingsPath, + baselinePath, + baselineBytes, + "Live monitoring must preserve the current baseline revision bytes", + seedInfo.revision + ); + } + assert.deepEqual( + await readMetadataInfo(sourceDatabase, sourceVault, sourceSettings, baselinePath), + seedInfo, + "Source must retain the selected baseline document and revision" + ); + await assertDatabaseBytes( + sourceDatabase, + sourceVault, + sourceSettings, + baselinePath, + baselineBytes, + "Source must retain the selected baseline revision bytes", + seedInfo.revision + ); + await assertDatabaseBytes( + sourceDatabase, + sourceVault, + sourceSettings, + baselinePath, + initialBytes, + "Source must retain the original baseline revision contents", + originalInfo.revision + ); + await assertCouchdbRevisionAncestor( + couchdbUri, + couchdbUser, + couchdbPassword, + dbname, + seedInfo.id, + seedInfo.revision, + originalInfo.revision + ); + await assertDatabaseBytes( + observerDatabase, + observerVault, + observerSettings, + restartedPath, + restartedBytes, + "Restarted daemon content must exist in the independently replicated database" + ); + const observerPaths = await readDatabasePaths(observerDatabase, observerVault, observerSettings); + assert.equal( + observerPaths.has(watchedPath), + false, + "Deleted note must be absent from the independently replicated database" + ); + assert.equal( + observerPaths.has(dbLikePath), + false, + "Deleted DB-like path must be absent from the independently replicated database" + ); + assert.equal( + observerPaths.has(hiddenPath), + false, + "Dot-prefixed entry must be absent from the independently replicated database" + ); + assert.equal( + observerPaths.has(ignoredPath), + false, + "Entry matched by .livesync/ignore must be absent from the independently replicated database" + ); + } finally { + await stopDaemon(observerDaemon); + await stopDaemon(targetDaemon); + await stopCouchdb().catch(() => {}); + } + } +); diff --git a/updates.md b/updates.md index d3218ea8..438196df 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,18 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### CLI + +#### Fixed + +- We can synchronise CLI Vaults whose root or parent directories have dot-prefixed names, including through symlinks that resolve to those locations. (#1340) + - Watch exclusions now apply within the selected Vault. Startup scans and live monitoring share the same exclusions, including PouchDB view indexes. + - Hidden entries, the configured database storage entries, and `.livesync/ignore` rules remain excluded. + +#### Added + +- We can include ordinary directories with database-like names by specifying `--include-db-like-dirs` on each invocation. The CLI's database storage entries, hidden entries, and user ignore rules remain excluded. + ## 1.0.49 11th November, 2026