From 60b093612b121dcb37331aa3d54727cfb9d52154 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 4 Sep 2026 09:33:36 +0000 Subject: [PATCH] Extract optional-file synchronisation workflow owners --- ...misation_and_hidden_file_sync_ownership.md | 95 +- .../optional_file_sync_architecture.md | 110 +- .../ConfigSync/applicationOperations.ts | 346 ++++++ .../applicationOperations.unit.spec.ts | 306 +++++ src/features/ConfigSync/catalogueMigration.ts | 126 ++ .../ConfigSync/catalogueOperations.ts | 216 ++++ .../catalogueOperations.unit.spec.ts | 222 ++++ ...yncCatalogueState.ts => catalogueState.ts} | 11 +- ...it.spec.ts => catalogueState.unit.spec.ts} | 14 +- src/features/ConfigSync/catalogueV1.ts | 50 + src/features/ConfigSync/catalogueV2.ts | 123 ++ ...tomisationSyncContext.command.unit.spec.ts | 22 +- ...tomisationSyncContext.routing.unit.spec.ts | 6 +- ...ationSyncContext.scan-routing.unit.spec.ts | 112 +- .../ConfigSync/customisationSyncContext.ts | 1073 ++--------------- ...isationSyncContext.v2Manifest.unit.spec.ts | 43 +- ...customisationSyncContext.view.unit.spec.ts | 25 +- src/features/ConfigSync/scanOperations.ts | 197 +++ .../ConfigSync/scanOperations.unit.spec.ts | 327 +++++ src/features/ConfigSync/snapshotOperations.ts | 82 ++ .../snapshotOperations.unit.spec.ts | 137 +++ .../ConfigSync/snapshotPersistence.ts | 402 ++++++ .../snapshotPersistence.unit.spec.ts | 219 ++++ ...ddenFileSyncContext.ownership.unit.spec.ts | 9 +- .../HiddenFileSync/hiddenFileSyncContext.ts | 622 +--------- .../hiddenFileSyncContext.unit.spec.ts | 32 +- src/features/HiddenFileSync/reconciliation.ts | 683 +++++++++++ .../reconciliation.unit.spec.ts | 168 +++ .../scripts/hidden-file-snippet-sync.ts | 7 +- 29 files changed, 4017 insertions(+), 1768 deletions(-) create mode 100644 src/features/ConfigSync/applicationOperations.ts create mode 100644 src/features/ConfigSync/applicationOperations.unit.spec.ts create mode 100644 src/features/ConfigSync/catalogueMigration.ts create mode 100644 src/features/ConfigSync/catalogueOperations.ts create mode 100644 src/features/ConfigSync/catalogueOperations.unit.spec.ts rename src/features/ConfigSync/{customisationSyncCatalogueState.ts => catalogueState.ts} (94%) rename src/features/ConfigSync/{customisationSyncCatalogueState.unit.spec.ts => catalogueState.unit.spec.ts} (90%) create mode 100644 src/features/ConfigSync/catalogueV1.ts create mode 100644 src/features/ConfigSync/catalogueV2.ts create mode 100644 src/features/ConfigSync/scanOperations.ts create mode 100644 src/features/ConfigSync/scanOperations.unit.spec.ts create mode 100644 src/features/ConfigSync/snapshotOperations.ts create mode 100644 src/features/ConfigSync/snapshotOperations.unit.spec.ts create mode 100644 src/features/ConfigSync/snapshotPersistence.ts create mode 100644 src/features/ConfigSync/snapshotPersistence.unit.spec.ts create mode 100644 src/features/HiddenFileSync/reconciliation.ts create mode 100644 src/features/HiddenFileSync/reconciliation.unit.spec.ts diff --git a/docs/adr/2026_09_customisation_and_hidden_file_sync_ownership.md b/docs/adr/2026_09_customisation_and_hidden_file_sync_ownership.md index c1f35e0c..05f3af2b 100644 --- a/docs/adr/2026_09_customisation_and_hidden_file_sync_ownership.md +++ b/docs/adr/2026_09_customisation_and_hidden_file_sync_ownership.md @@ -20,13 +20,15 @@ one writer for raw events, scheduled scans, and automatic database-to-local reflection. The constructor-name add-on lookup and the `ConfigSync` and `HiddenFileSync` add-on identities have been retired. -The Customisation Sync private context coordinates its snapshot operations, -scan queues, periodic work, and focused path and transient-state owners. The -Hidden File Sync private context coordinates reconciliation, periodic work, -and the lifetimes of focused path-admission, notification, processed-state, -change-processing, and conflict-resolution owners. Each receives live settings -and database projections, focused storage, path, and exact-revision -capabilities, and explicit host effects rather than `LiveSyncCore`. +The Customisation Sync private context coordinates lifecycle, raw-event +admission, configuration and periodic policy, view composition, and the +lifetimes of focused path, snapshot, application, scan, and catalogue owners. +The Hidden File Sync private context coordinates lifecycle and handler +admission around focused path-admission, notification, processed-state, +change-processing, conflict-resolution, and reconciliation owners. Each +receives live settings and database projections, focused storage, path, and +exact-revision capabilities, and explicit host effects rather than +`LiveSyncCore`. The corresponding implemented topology is documented in [Optional-file synchronisation architecture](../design_docs/optional_file_sync_architecture.md). @@ -203,14 +205,35 @@ operations receive this capability through one narrow `processedState` port; they do not receive bundles of individual state callbacks or implement state keys themselves. -`CustomisationSyncCatalogueState` separately owns the transient catalogue -rows, manifest lookup and mtime cache, their reactive stores, and catalogue -update progress. A small recent-event deduplicator owns raw-event admission -history. These Customisation Sync owners are deliberately not implementations -of a shared Hidden File Sync state abstraction: the former state is a derived, -in-memory projection, while Hidden File Sync markers are persisted operational -reconciliation state with different identity, invalidation, and deletion -rules. +`CatalogueState` separately owns the transient catalogue rows, manifest lookup +and mtime cache, their reactive stores, and catalogue update progress. A small +recent-event deduplicator owns raw-event admission history. These Customisation +Sync owners are deliberately not implementations of a shared Hidden File Sync +state abstraction: the former state is a derived, in-memory projection, while +Hidden File Sync markers are persisted operational reconciliation state with +different identity, invalidation, and deletion rules. + +`CatalogueOperations` owns catalogue enumeration, one queue and its progress +subscription, and composes `CatalogueV1`, `CatalogueV2`, and +`CatalogueMigration`. V1 and V2 are mutually exclusive as the selected write +format, but both persisted formats can coexist during migration. Queue work +therefore reads the live setting when it starts, and the shared state continues +to recognise V2 rows independently of that setting. `CatalogueMigration` +remains the distinct bridge from grouped V1 binders to per-file V2 documents. + +`SnapshotPersistence` owns the host-neutral V1 grouped and V2 per-file writes +and logical deletion. It returns explicit mutation and refresh outcomes, so it +neither owns catalogue state nor calls back through the context. +`SnapshotOperations` applies those outcomes with the inherited awaited V1 or +fire-and-forget V2 timing. `ApplicationOperations` owns compare, apply, +duplicate, and delete workflows, while `ScanOperations` owns configuration-file +enumeration and V1/V2 reconciliation. Both depend on narrow snapshot and +catalogue ports instead of the context. + +The newly extracted catalogue, snapshot, application, scan, and reconciliation +modules omit a Customisation Sync or Hidden File Sync prefix because their +feature directories already supply that scope. Public contexts and views retain +their domain names for compatibility. `CustomisationSyncPathOperations` binds live configuration-directory, mode, and device-name projections to the pure category and V1/V2 key functions. It @@ -232,6 +255,12 @@ This boundary keeps event concurrency and settlement directly testable without giving the processor full scan, initialisation, notification, or host responsibilities. +`Reconciliation` owns storage and database enumeration, full scans, offline +comparison, rebuild direction and ordering, processed-state adoption, +initialisation sequencing, and the scoped rebuild interceptor used by maintained +real-Obsidian tests. Storage and database scans remain together because every +offline and initialisation path coordinates both sides. + The joint composition may return several views backed by those contexts: - a Customisation Sync catalogue and operation view for its dialogue; @@ -372,11 +401,17 @@ to the non-owner. The private context, path module, codec module, focused presentation view, and resource teardown are implemented. A focused path capability binds live -settings and device identity to the pure path functions. A focused -catalogue-state owner holds the rows, manifests, manifest mtime cache, reactive -stores, and update progress, while a bounded deduplicator owns recent raw-event -keys. Enumeration and scan queues remain with the orchestrating context. The -context accepts only narrow, live projections and explicit effects; an +settings and device identity to the pure path functions. A focused catalogue +owner holds one queue, its progress subscription, and shared state, while +separate V1, V2, and migration modules hold format-specific behaviour. A +bounded deduplicator owns recent raw-event keys. Host-neutral snapshot +persistence owns V1 grouped and V2 per-file writes, unchanged-content checks, +and logical deletion. A snapshot coordinator applies its explicit refresh +outcomes without a catalogue-to-context callback cycle. Focused application +and scan owners contain selected-snapshot workflows and full local/database +reconciliation, respectively. The context retains raw-event admission and +scheduling, configuration and periodic policy, owner lifetime, and view +composition. It accepts only narrow, live projections and explicit effects; an Obsidian adapter at the composition edge owns dialogues, Notices, plug-in reload, restart, lifecycle, Vault access, and compatibility scan telemetry. @@ -397,9 +432,10 @@ the ownership, static-path, pattern, and ignore-file sequence. A focused change notifier owns folder batching, delayed delivery, and teardown of its scheduled work and Notice effect. A focused change processor owns storage and database event processing, bounded concurrency, per-path serialisation, activity -publication, and compatibility settlement order. The context retains scan, -initialisation, and reconciliation orchestration. A focused -conflict-resolution owner owns pending-path admission, +publication, and compatibility settlement order. A focused reconciliation +owner owns storage and database scans, offline comparison, rebuilds, +processed-state adoption, initialisation direction and ordering, and its scoped +testing interceptor. A focused conflict-resolution owner owns pending-path admission, the parallel classification and serial interaction queues, automatic merge, newer-revision selection, interactive JSON application, settlement, and queue disposal. An Obsidian adapter owns JSON conflict dialogue instances, progress @@ -499,12 +535,13 @@ migration without first establishing narrow dependencies. - The legacy add-on identity and constructor-name lookup are removed. - The two contexts coordinate separate synchronisation workflows, but their dependency surfaces are explicit and do not include the complete core. - Customisation Sync delegates its path binding, derived catalogue, and - recent-event state, while Hidden File Sync delegates path admission, - notification, processed-state, change-processing, and conflict lifecycles, - to focused owners. Further extraction should follow a concrete behavioural - boundary rather than create additional serviceFeatures for private - operations. + Customisation Sync delegates its path binding, snapshot persistence and + refresh sequencing, selected-snapshot application, scan reconciliation, + derived catalogue, catalogue queue, and recent-event state, while Hidden File + Sync delegates path admission, notification, processed-state, + change-processing, reconciliation, and conflict lifecycles to focused owners. + Further extraction should follow a concrete behavioural boundary rather than + create additional serviceFeatures for private operations. ## References diff --git a/docs/design_docs/optional_file_sync_architecture.md b/docs/design_docs/optional_file_sync_architecture.md index 47cbd4cd..610d2e69 100644 --- a/docs/design_docs/optional_file_sync_architecture.md +++ b/docs/design_docs/optional_file_sync_architecture.md @@ -32,9 +32,19 @@ Obsidian composition (`main.ts`) | | | +--> `CustomisationSyncContext` | | +-- `CustomisationSyncPathOperations` - | | +-- `CustomisationSyncCatalogueState` + | | +-- `SnapshotPersistence` + | | +-- `SnapshotOperations` + | | +-- `ApplicationOperations` + | | +-- `ScanOperations` | | +-- recent-event deduplicator | | +-- immutable service-handler and testing views + | | | + | | +--> `CatalogueOperations` + | | +-- `CatalogueState` + | | +-- one catalogue queue and progress lifecycle + | | +-- `CatalogueV1` + | | +-- `CatalogueV2` + | | +-- `CatalogueMigration` | | ^ | | +-- narrow dependencies from | | `customisationSyncObsidianAdapter` @@ -64,6 +74,10 @@ Obsidian composition (`main.ts`) | | +-- changed-folder batching and scheduled delivery | | +-- suppression and Notice-effect teardown | | + | +--> `Reconciliation` + | | +-- storage and database scans + | | +-- offline reconciliation, rebuilds, and initialisation + | | | +-- immutable service-handler, command, repair, and testing views | +--> `useCustomisationSyncUI` @@ -84,37 +98,65 @@ application core. ## Ownership -| Owner | Owns | Does not own | -| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `useOptionalFileSync` | Construction of both contexts, handler registration and removal, local-owner selection, namespace dispatch, compatibility callback order, and context disposal order. | Persisted feature state, synchronisation algorithms, commands, dialogues, or Notices. | -| `CustomisationSyncContext` | The `ix:` scan and snapshot workflow, snapshot storage and application, periodic scan state, and the lifetimes of its focused owners. | Path derivation, catalogue mutations, recent-event history, Obsidian dialogues, ribbon actions, or handler registration. | -| `CustomisationSyncPathOperations` | Binding live configuration-directory, mode, and device-name projections to the pure category, target-path, V1 key, V2 key, and device-prefix functions. | I/O, mutable state, local-owner selection, or persistence. | -| `CustomisationSyncCatalogueState` | Transient catalogue rows, manifest lookup and mtime cache, reactive catalogue and manifest stores, and catalogue update progress. | Database or storage I/O, scan scheduling, routing, or persistence. | -| Customisation Sync event deduplicator | The bounded, newest-first keys used to admit raw configuration events once. | Scheduling, path ownership, catalogue state, or persisted data. | -| `HiddenFileSyncContext` | The `i:` scan and reconciliation workflow, exact-revision repair composition, periodic scan state, and focused-owner lifetimes. | Path admission state, change-event serialisation, processed-state representation, notification batching, conflict queue state, Obsidian dialogues, or service handler registration. | -| `HiddenFileSyncProcessedState` | Three device-local autosaved maps, exact storage and database keys, retained known mtime, reset behaviour, and storage/database settlement effects. | File transfer, scan scheduling, conflict handling, or presentation. | -| `HiddenFileSyncChangeProcessor` | Storage and database change processing, same-path event serialisation, bounded concurrency, activity counts, and the inherited event-consumption and settlement order. | Full scans, initialisation policy, notification presentation, or conflict interaction. | -| `HiddenFileSyncConflictResolution` | Conflict admission and deduplication, pending paths, the parallel classification and serial interaction queues, automatic merge, newer-revision policy, interactive JSON resolution, and conflict settlement. | Obsidian dialogue instances, general Hidden File Sync scans, processed-state caches, or Service handler registration. | -| `HiddenFileSyncPathAdmission` | The ownership-first eligibility sequence, hidden-path and pattern policy, asynchronous ignore-file check, and the per-context parsed-pattern cache. | Composition-level owner selection, scans, transfer, or persistence. | -| `HiddenFileSyncChangeNotifier` | Changed-folder deduplication, delayed delivery, live suppression and configuration-directory checks, scheduled-task cancellation, and the host Notice show/hide effects. | The Obsidian Notice instance, file extraction, or scan policy. | -| Customisation Sync Obsidian adapter | Obsidian conflict selection, Notice presentation, plug-in reload, restart requests, Vault enumeration, progress telemetry, and platform-derived fallback device names. | Catalogue state, routing, or persisted document operations. | -| Hidden File Sync Obsidian adapter | JSON conflict dialogue lifetime, progress presentation, grouped change Notices, plug-in reload actions, restart scheduling, Vault enumeration, and compatibility activity publication. | Transfer, reconciliation, processed-state, or conflict decisions. | -| `useCustomisationSyncUI` | Command, ribbon, dialogue, open-request subscription, and their unload teardown. | Synchronisation state or Hidden File Sync initialisation behaviour. | -| `useHiddenFileSyncCommands` | Hidden File Sync command registration, setting-change subscription, and their unload teardown. | Synchronisation state or command implementation. | +| Owner | Owns | Does not own | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `useOptionalFileSync` | Construction of both contexts, handler registration and removal, local-owner selection, namespace dispatch, compatibility callback order, and context disposal order. | Persisted feature state, synchronisation algorithms, commands, dialogues, or Notices. | +| `CustomisationSyncContext` | Lifecycle and view composition, raw-event admission and scheduling, configuration transitions, periodic scan policy, and focused-owner lifetimes. | Scan reconciliation, snapshot writes or refresh sequencing, catalogue internals, application and comparison algorithms, Obsidian dialogues, ribbon actions, or handler registration. | +| `CustomisationSyncPathOperations` | Binding live configuration-directory, mode, and device-name projections to the pure category, target-path, V1 key, V2 key, and device-prefix functions. | I/O, mutable state, local-owner selection, or persistence. | +| `SnapshotPersistence` | V1 grouped and V2 per-file local-to-database writes, unchanged-content checks, logical deletion, and explicit catalogue-refresh outcomes. | Catalogue state or refresh execution, scans, lifecycle policy, application, dialogues, or plug-in reload. | +| `SnapshotOperations` | Applying persistence outcomes to the catalogue with the inherited awaited V1 and fire-and-forget V2 refresh timing. | Snapshot encoding, catalogue state, scans, lifecycle policy, or host effects. | +| `ApplicationOperations` | Comparing, applying, duplicating, and deleting selected Customisation Sync snapshots through narrow storage, snapshot, and catalogue ports. | Catalogue enumeration, raw-event admission, periodic scheduling, or view composition. | +| `ScanOperations` | Configuration-file enumeration and V1/V2 reconciliation with local and database state. | Periodic scheduling, raw-event admission, snapshot persistence details, or catalogue state. | +| `CatalogueOperations` | Catalogue enumeration and publication, one format-dispatching queue and its progress lifecycle, and composition of the state, V1, V2, and migration modules. | Local-file scanning, snapshot application, raw-event routing, dialogues, or handler registration. | +| `CatalogueState` | Transient catalogue rows, manifest lookup and mtime cache, reactive catalogue and manifest stores, and catalogue update progress. | Database or storage I/O, scan scheduling, routing, or persistence. | +| `CatalogueV1` | Loading and publishing grouped V1 catalogue rows. | V2 decoding, migration, queue lifetime, or local-file scanning. | +| `CatalogueV2` | Building and updating per-file V2 rows and manifests. | V1 loading, migration, queue lifetime, or local-file scanning. | +| `CatalogueMigration` | Translating a grouped V1 binder into V2 per-file documents, deleting the migrated binder, and applying its required V1 refresh. | General catalogue enumeration, queue ownership, or local-file scanning. | +| Customisation Sync event deduplicator | The bounded, newest-first keys used to admit raw configuration events once. | Scheduling, path ownership, catalogue state, or persisted data. | +| `HiddenFileSyncContext` | Lifecycle and view composition, handler admission, configuration transitions, periodic scan state, exact-revision repair composition, and focused-owner lifetimes. | Scan and rebuild algorithms, path admission state, change-event serialisation, processed-state representation, notification batching, conflict queue state, Obsidian dialogues, or service handler registration. | +| `HiddenFileSyncProcessedState` | Three device-local autosaved maps, exact storage and database keys, retained known mtime, reset behaviour, and storage/database settlement effects. | File transfer, scan scheduling, conflict handling, or presentation. | +| `HiddenFileSyncChangeProcessor` | Storage and database change processing, same-path event serialisation, bounded concurrency, activity counts, and the inherited event-consumption and settlement order. | Full scans, initialisation policy, notification presentation, or conflict interaction. | +| `Reconciliation` | Storage and database enumeration, full scans, offline reconciliation, rebuild direction and ordering, processed-state adoption, initialisation sequencing, and the scoped rebuild interceptor. | Individual transfer implementation, conflict interaction, path-pattern state, periodic scheduling, or host lifecycle. | +| `HiddenFileSyncConflictResolution` | Conflict admission and deduplication, pending paths, the parallel classification and serial interaction queues, automatic merge, newer-revision policy, interactive JSON resolution, and conflict settlement. | Obsidian dialogue instances, general Hidden File Sync scans, processed-state caches, or Service handler registration. | +| `HiddenFileSyncPathAdmission` | The ownership-first eligibility sequence, hidden-path and pattern policy, asynchronous ignore-file check, and the per-context parsed-pattern cache. | Composition-level owner selection, scans, transfer, or persistence. | +| `HiddenFileSyncChangeNotifier` | Changed-folder deduplication, delayed delivery, live suppression and configuration-directory checks, scheduled-task cancellation, and the host Notice show/hide effects. | The Obsidian Notice instance, file extraction, or scan policy. | +| Customisation Sync Obsidian adapter | Obsidian conflict selection, Notice presentation, plug-in reload, restart requests, Vault enumeration, progress telemetry, and platform-derived fallback device names. | Catalogue state, routing, or persisted document operations. | +| Hidden File Sync Obsidian adapter | JSON conflict dialogue lifetime, progress presentation, grouped change Notices, plug-in reload actions, restart scheduling, Vault enumeration, and compatibility activity publication. | Transfer, reconciliation, processed-state, or conflict decisions. | +| `useCustomisationSyncUI` | Command, ribbon, dialogue, open-request subscription, and their unload teardown. | Synchronisation state or Hidden File Sync initialisation behaviour. | +| `useHiddenFileSyncCommands` | Hidden File Sync command registration, setting-change subscription, and their unload teardown. | Synchronisation state or command implementation. | The two domain contexts coordinate one cohesive synchronisation workflow each. Their private operations and focused owners are not additional serviceFeatures: they do not independently register host integration or have separate -application lifetimes. `CustomisationSyncPathOperations` is a stateless -capability which binds live inputs to pure path functions. The Hidden File Sync -path-admission owner holds the parsed-pattern cache, and its change notifier -holds the pending folder set and scheduled-task lifetime. +application lifetimes. The newly extracted catalogue, snapshot, application, +scan, and reconciliation modules omit the domain prefix because their feature +directories already supply that scope; public context and view names retain it +for compatibility. `CustomisationSyncPathOperations` is a stateless capability +which binds live inputs to pure path functions. +`SnapshotPersistence` is a host-neutral operation boundary: it returns +structured mutation and refresh outcomes without reaching into the catalogue +or presentation. `SnapshotOperations` consumes those outcomes and preserves +their refresh timing. `ApplicationOperations` and `ScanOperations` depend on +those narrow ports instead of calling back through the context. + +`CatalogueOperations` owns one catalogue queue, its progress subscription, and +the shared state projected by the format-specific modules. The live V2 setting +selects V1 loading or V1-to-V2 migration when each queued item starts. Persisted +V1 and V2 documents can coexist during migration, so V2 documents remain +recognisable regardless of the currently selected write format. `CatalogueV1` +and `CatalogueV2` contain only their format-specific catalogue behaviour, while +`CatalogueMigration` remains the explicit bridge between them. The Hidden File +Sync path-admission owner holds the parsed-pattern cache, and its change +notifier holds the pending folder set and scheduled-task lifetime. `HiddenFileSyncChangeProcessor` is a focused resource owner because its semaphore, per-path serialisation, activity counters, and event settlement form one independently testable lifecycle. `HiddenFileSyncConflictResolution` is another focused resource owner because conflict admission, pending-path identity, two serialisation stages, and disposal form a separate lifecycle. +`Reconciliation` keeps storage and database enumeration together because +offline comparison, rebuilds, and each initialisation direction depend on both +sides and their ordered processed-state adoption. The two state owners intentionally do not implement a common generic state contract. Customisation Sync projects transient catalogue and presentation @@ -214,12 +256,16 @@ than being re-exported through a broad context testing view. The composition is created after the Service Hub and required ServiceModules exist, and before lifecycle-driven feature work begins. Each context creates its own periodic processor and focused resource owners. -`CustomisationSyncContext` creates one path capability, one catalogue-state -owner, and one recent-event deduplicator. `HiddenFileSyncContext` creates one +`CustomisationSyncContext` creates one path capability, one snapshot-persistence +boundary, one snapshot coordinator, one application owner, one scan owner, one +catalogue owner, and one recent-event deduplicator. The catalogue owner creates +its shared state, one queue and progress subscription, and the V1, V2, and +migration modules. `HiddenFileSyncContext` creates one path-admission owner, one change notifier, and one processed-state owner before composing database write and extraction operations around their narrow ports. -It then creates one change processor and one conflict-resolution owner. The -change processor owns its semaphore and activity state. +It then creates one change processor, one conflict-resolution owner, and one +reconciliation owner. The change processor owns its semaphore and activity +state; the reconciliation owner owns its scoped testing interceptor. Full conflict scans admit discovered paths into the same queue without suspending it, so ordinary database conflict notifications continue during a slow scan. After enumeration, the operation waits for both classification and @@ -255,10 +301,14 @@ appropriate, a migration decision. Focused unit tests cover routing, path-option binding, Hidden File Sync admission ordering and cache invalidation, semantic handler views, context owner isolation, teardown, initial cache selection, exact-revision repair, -change-event serialisation and settlement, notification batching, conflict -queue admission, revision selection, automatic and interactive merge effect -ordering, conflict dialogue adaptation, grouped Notices, and compatibility -activity publication. +Customisation Sync scan reconciliation and context delegation, single-queue +catalogue disposal and publication, live V1/V2 dispatch, V1 and V2 snapshot +persistence outcomes, logical-deletion idempotence, refresh ordering, +application operations, Hidden File Sync change-event serialisation and +settlement, reconciliation direction and scan ordering, notification batching, +conflict queue admission, revision selection, automatic and interactive merge +effect ordering, conflict dialogue adaptation, grouped Notices, and +compatibility activity publication. The boundary test prevents either domain context from regaining core or Obsidian dependencies. diff --git a/src/features/ConfigSync/applicationOperations.ts b/src/features/ConfigSync/applicationOperations.ts new file mode 100644 index 00000000..bd4f96c2 --- /dev/null +++ b/src/features/ConfigSync/applicationOperations.ts @@ -0,0 +1,346 @@ +import { diff_match_patch, parseYaml } from "@/deps.ts"; +import type { + diff_result, + FilePath, + FilePathWithPrefix, + LOG_LEVEL, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { delay, getDocData, getDocDataAsArray, isDocContentSame } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { decodeBinary } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/convert"; +import { digestHash } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/hash"; +import { serialized } from "octagonal-wheels/concurrency/lock"; +import { base64ToArrayBuffer, base64ToString } from "octagonal-wheels/binary/base64"; + +import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError"; +import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; +import { createCustomisationSyncCodec, type PluginDataEx } from "./customisationSyncCodec.ts"; +import type { CatalogueOperations } from "./catalogueOperations.ts"; +import type { CustomisationSyncPathOperations } from "./customisationSyncPathOperations.ts"; +import type { SnapshotOperations } from "./snapshotOperations.ts"; +import { PluginDataExDisplayV2 } from "./customisationSyncModel.ts"; +import type { IPluginDataExDisplay, LoadedEntryPluginDataExFile } from "./customisationSyncView.ts"; + +const { deserialize } = createCustomisationSyncCodec({ digestHash, parseYaml }); + +type ApplicationDatabase = Pick; +type ApplicationStorage = Pick< + StorageAccess, + "ensureDir" | "readHiddenFileBinary" | "readHiddenFileText" | "writeHiddenFileAuto" +>; +type ApplicationPath = Pick; +type ApplicationSnapshotOperations = Pick< + SnapshotOperations, + "isV2Enabled" | "storeCustomisationFileV2" | "storeCustomizationFiles" | "deleteConfigOnDatabase" +>; +type ApplicationCatalogue = Pick< + CatalogueOperations, + "findPlugins" | "manifestLookup" | "updatePluginList" | "updatePluginListV2" +>; + +export type ApplicationOperationsDependencies = { + getLocalDatabase(): ApplicationDatabase; + storageAccess: ApplicationStorage; + path: ApplicationPath; + log: LogFunction; + getConfigDir(): string; + getDeviceAndVaultName(): string; + resolveJsonConflict( + path: FilePath, + files: [LoadedEntryPluginDataExFile, LoadedEntryPluginDataExFile], + remoteName: string, + apply: (content: string) => Promise + ): Promise; + selectTextFile(path: FilePath, diffResult: diff_result, remoteName: string): Promise<"A" | "B" | false>; + reloadPlugin(configDir: string, pluginName: string): Promise; + askRestart(): void; + snapshotOperations: ApplicationSnapshotOperations; + catalogueOperations: ApplicationCatalogue; +}; + +/** + * Owns the Customisation Sync dialogue's compare, apply, duplicate, and + * delete workflows. It deliberately consumes the shared snapshot capability + * and catalogue owner, leaving lifecycle, event admission, and scanning in + * the context. + */ +export class ApplicationOperations { + constructor(private readonly dependencies: ApplicationOperationsDependencies) {} + + private get configDir() { + return this.dependencies.getConfigDir(); + } + + private get localDatabase() { + return this.dependencies.getLocalDatabase(); + } + + private get storageAccess() { + return this.dependencies.storageAccess; + } + + private _log(message: unknown, level?: LOG_LEVEL, key?: string) { + this.dependencies.log(message, level, key); + } + + async compareFileUsingDisplayData( + dataA: IPluginDataExDisplay, + dataB: IPluginDataExDisplay, + filename: string + ): Promise { + const dataACopy = + dataA instanceof PluginDataExDisplayV2 + ? new PluginDataExDisplayV2(dataA, this.dependencies.catalogueOperations.manifestLookup) + : { ...dataA }; + const dataBCopy = + dataB instanceof PluginDataExDisplayV2 + ? new PluginDataExDisplayV2(dataB, this.dependencies.catalogueOperations.manifestLookup) + : { ...dataB }; + dataACopy.files = dataACopy.files.filter((file) => file.filename == filename); + dataBCopy.files = dataBCopy.files.filter((file) => file.filename == filename); + return await this.compareUsingDisplayData(dataACopy, dataBCopy, true); + } + + async compareUsingDisplayData(dataA: IPluginDataExDisplay, dataB: IPluginDataExDisplay, compareEach = false) { + const loadFile = async (data: IPluginDataExDisplay) => { + if (data instanceof PluginDataExDisplayV2 || compareEach) { + return data.files[0] as LoadedEntryPluginDataExFile; + } + const loadDoc = await this.localDatabase.getDBEntry(data.documentPath); + if (!loadDoc) return false; + const pluginData = deserialize(getDocDataAsArray(loadDoc.data), {}) as PluginDataEx; + pluginData.documentPath = data.documentPath; + const file = pluginData.files[0]; + const doc = { ...loadDoc, ...file, datatype: "newnote" } as LoadedEntryPluginDataExFile; + return doc; + }; + const fileA = await loadFile(dataA); + const fileB = await loadFile(dataB); + this._log(`Comparing: ${dataA.documentPath} <-> ${dataB.documentPath}`, LOG_LEVEL_VERBOSE); + if (!fileA || !fileB) { + this._log( + `Could not load ${dataA.name} for comparison: ${!fileA ? dataA.term : ""}${!fileB ? dataB.term : ""}`, + LOG_LEVEL_NOTICE + ); + return false; + } + const path = fileA.filename.split("/").pop() as FilePath; + if (path.endsWith(".json")) { + return serialized("config:merge-data", async () => { + this._log("Opening data-merging dialog", LOG_LEVEL_VERBOSE); + return await this.dependencies.resolveJsonConflict(path, [fileA, fileB], dataB.term, async (result) => { + try { + return await this.applyData(dataA, result); + } catch (ex) { + this._log("Could not apply merged file"); + this._log(ex, LOG_LEVEL_VERBOSE); + return false; + } + }); + }); + } else { + const dmp = new diff_match_patch(); + let docAData = getDocData(fileA.data); + let docBData = getDocData(fileB.data); + if (fileA?.datatype != "plain") { + docAData = base64ToString(docAData); + } + if (fileB?.datatype != "plain") { + docBData = base64ToString(docBData); + } + const diffMap = dmp.diff_linesToChars_(docAData, docBData); + + const diff = dmp.diff_main(diffMap.chars1, diffMap.chars2, false); + dmp.diff_charsToLines_(diff, diffMap.lineArray); + dmp.diff_cleanupSemantic(diff); + const diffResult: diff_result = { + left: { rev: "A", ...fileA, data: docAData }, + right: { rev: "B", ...fileB, data: docBData }, + diff: diff, + }; + const ret = await this.dependencies.selectTextFile(path, diffResult, dataB.term); + if (ret === false) return false; + const resultContent = ret == "A" ? docAData : ret == "B" ? docBData : undefined; + if (resultContent) { + return await this.applyData(dataA, resultContent); + } + return false; + } + } + + async duplicateData(data: IPluginDataExDisplay, deviceName: string): Promise { + const path = `${this.configDir}/${data.files[0].filename}` as FilePath; + await this.dependencies.snapshotOperations.storeCustomizationFiles(path, deviceName); + await this.dependencies.catalogueOperations.updatePluginList( + false, + this.dependencies.path.filenameToUnifiedKey(path, deviceName) + ); + } + + async applyDataV2(data: PluginDataExDisplayV2, content?: string): Promise { + const baseDir = this.configDir; + try { + if (content) { + // Preserve the inherited truthiness check: an explicitly empty + // replacement is treated as the no-content path. + const filename = data.files[0].filename; + this._log(`Applying ${filename} of ${data.displayName || data.name}..`); + const path = `${baseDir}/${filename}` as FilePath; + await this.storageAccess.ensureDir(path); + // If the content has applied, modified time will be updated to the current time. + await this.storageAccess.writeHiddenFileAuto(path, content); + await this.dependencies.snapshotOperations.storeCustomisationFileV2( + path, + this.dependencies.getDeviceAndVaultName() + ); + } else { + const files = data.files; + for (const f of files) { + // If files have applied, modified time will be updated to the current time. + const stat = { mtime: f.mtime, ctime: f.ctime }; + const path = `${baseDir}/${f.filename}` as FilePath; + this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`); + // const contentEach = createBlob(f.data); + await this.storageAccess.ensureDir(path); + + if (f.datatype == "newnote") { + let oldData; + try { + oldData = await this.storageAccess.readHiddenFileBinary(path); + } catch (ex) { + this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); + oldData = new ArrayBuffer(0); + } + const content = base64ToArrayBuffer(f.data); + if (await isDocContentSame(oldData, content)) { + this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); + continue; + } + await this.storageAccess.writeHiddenFileAuto(path, content, stat); + } else { + let oldData; + try { + oldData = await this.storageAccess.readHiddenFileText(path); + } catch (ex) { + this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); + oldData = ""; + } + const content = getDocData(f.data); + if (await isDocContentSame(oldData, content)) { + this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); + continue; + } + await this.storageAccess.writeHiddenFileAuto(path, content, stat); + } + this._log(`Applied ${f.filename} of ${data.displayName || data.name}..`); + await this.dependencies.snapshotOperations.storeCustomisationFileV2( + path, + this.dependencies.getDeviceAndVaultName() + ); + } + } + } catch (ex) { + this._log(`Applying ${data.displayName || data.name}.. Failed`, LOG_LEVEL_NOTICE); + this._log(ex, LOG_LEVEL_VERBOSE); + return false; + } + return true; + } + + async applyData(data: IPluginDataExDisplay, content?: string): Promise { + this._log(`Applying ${data.displayName || data.name}..`); + + if (data instanceof PluginDataExDisplayV2) { + return this.applyDataV2(data, content); + } + return this.applyDataV1(data, content); + } + + private async applyDataV1(data: IPluginDataExDisplay, content?: string): Promise { + const baseDir = this.configDir; + try { + if (!data.documentPath) throw new LiveSyncError("InternalError: Document path not exist"); + const dx = await this.localDatabase.getDBEntry(data.documentPath); + if (dx == false) { + throw new LiveSyncError("Not found on database"); + } + const loadedData = deserialize(getDocDataAsArray(dx.data), {}) as PluginDataEx; + for (const f of loadedData.files) { + this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`); + try { + // console.dir(f); + const path = `${baseDir}/${f.filename}`; + await this.storageAccess.ensureDir(path); + if (!content) { + const dt = decodeBinary(f.data); + await this.storageAccess.writeHiddenFileAuto(path, dt); + } else { + await this.storageAccess.writeHiddenFileAuto(path, content); + } + this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Done`); + } catch (ex) { + this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); + } + } + const uPath = `${baseDir}/${loadedData.files[0].filename}` as FilePath; + await this.dependencies.snapshotOperations.storeCustomizationFiles(uPath); + // The inherited workflow refreshes once through persistence, then + // explicitly refreshes again with the dialogue's notice flag. + await this.dependencies.catalogueOperations.updatePluginList(true, uPath); + await delay(100); + this._log(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL_NOTICE); + if (data.category == "PLUGIN_DATA" || data.category == "PLUGIN_MAIN") { + await this.dependencies.reloadPlugin(baseDir, data.name); + } else if (data.category == "CONFIG") { + this.dependencies.askRestart(); + } + return true; + } catch (ex) { + this._log(`Applying ${data.displayName || data.name}.. Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); + return false; + } + } + + async deleteData(data: PluginDataEx): Promise { + try { + if (data.documentPath) { + const delList: FilePathWithPrefix[] = []; + if (this.dependencies.snapshotOperations.isV2Enabled()) { + const deleteList = this.dependencies.catalogueOperations + .findPlugins(data.documentPath) + .filter((entry) => entry instanceof PluginDataExDisplayV2) + .map((entry) => entry.files) + .flat(); + for (const e of deleteList) { + delList.push(e.path); + } + } + delList.push(data.documentPath); + const p = delList.map(async (e) => { + await this.dependencies.snapshotOperations.deleteConfigOnDatabase(e); + // Preserve the inherited unconditional refresh after the + // persistence wrapper, including when it emitted no refresh. + await this.dependencies.catalogueOperations.updatePluginList(false, e); + }); + await Promise.allSettled(p); + // Preserve the inherited success result even when individual + // deletion/refresh promises settle unsuccessfully. + this._log( + `Deleted: ${data.category}/${data.name} of ${data.category} (${delList.length} items)`, + LOG_LEVEL_NOTICE + ); + } + return true; + } catch (ex) { + this._log(`Failed to delete: ${data.documentPath}`, LOG_LEVEL_NOTICE); + this._log(ex, LOG_LEVEL_VERBOSE); + return false; + } + } +} diff --git a/src/features/ConfigSync/applicationOperations.unit.spec.ts b/src/features/ConfigSync/applicationOperations.unit.spec.ts new file mode 100644 index 00000000..53871b3b --- /dev/null +++ b/src/features/ConfigSync/applicationOperations.unit.spec.ts @@ -0,0 +1,306 @@ +import { describe, expect, it, vi } from "vitest"; + +const asyncHarness = vi.hoisted(() => ({ + delay: vi.fn(async () => undefined), + fireAndForget: vi.fn((operation: () => unknown) => { + void operation(); + }), +})); + +vi.mock("@/deps.ts", () => ({ + diff_match_patch: class DiffMatchPatch {}, + parseYaml: vi.fn(), +})); +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/utils", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + delay: asyncHarness.delay, + fireAndForget: asyncHarness.fireAndForget, + }; +}); + +import type { FilePath, FilePathWithPrefix, LoadedEntry } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { PluginManifest } from "@/deps.ts"; +import { digestHash } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/hash"; +import { createCustomisationSyncCodec, type PluginDataEx } from "./customisationSyncCodec.ts"; +import { ApplicationOperations, type ApplicationOperationsDependencies } from "./applicationOperations.ts"; +import { PluginDataExDisplayV2 } from "./customisationSyncModel.ts"; +import type { SnapshotPersistenceResult } from "./snapshotPersistence.ts"; +import { SnapshotOperations } from "./snapshotOperations.ts"; +import type { IPluginDataExDisplay } from "./customisationSyncView.ts"; + +const codec = createCustomisationSyncCodec({ digestHash, parseYaml: () => undefined }); + +function createOperations() { + const events: string[] = []; + let usePluginSyncV2 = false; + type PersistenceResult = SnapshotPersistenceResult; + const getDBEntry = vi.fn(async (_path: FilePathWithPrefix) => false as false | LoadedEntry); + const ensureDir = vi.fn(async (_path: string) => { + events.push("ensure-dir"); + return true; + }); + const readHiddenFileBinary = vi.fn(async (_path: string) => new ArrayBuffer(0)); + const readHiddenFileText = vi.fn(async (_path: string) => ""); + const writeHiddenFileAuto = vi.fn(async (_path: string, _data: string | ArrayBuffer) => { + events.push("write-file"); + return true; + }); + const storeCustomisationFileV2 = vi.fn( + async (): Promise => ({ + value: true, + status: "saved" as const, + refreshes: [] as const, + }) + ); + const storeCustomizationFiles = vi.fn( + async (): Promise => ({ + value: true, + status: "saved" as const, + refreshes: [] as const, + }) + ); + const deleteConfigOnDatabase = vi.fn( + async (): Promise => ({ + value: true, + status: "deleted" as const, + refreshes: [] as const, + }) + ); + const updatePluginList = vi.fn(async (showMessage: boolean, _path?: FilePathWithPrefix | FilePath) => { + events.push(`refresh-v1:${showMessage}`); + }); + const updatePluginListV2 = vi.fn(async (_showMessage: boolean, _path: FilePathWithPrefix) => { + events.push("refresh-v2"); + }); + const findPlugins = vi.fn(() => [] as readonly IPluginDataExDisplay[]); + const reloadPlugin = vi.fn(async (_configDir: string, _pluginName: string) => { + events.push("reload-plugin"); + }); + const askRestart = vi.fn(() => { + events.push("ask-restart"); + }); + const catalogueOperations = { + findPlugins, + manifestLookup: new Map(), + updatePluginList, + updatePluginListV2, + }; + const snapshotOperations = new SnapshotOperations({ + getSettings: () => ({ usePluginSyncV2 }), + getDeviceAndVaultName: () => "device-a", + log: vi.fn(), + snapshotPersistence: { + storeCustomisationFileV2, + storeCustomizationFiles, + deleteConfigOnDatabase, + }, + catalogueOperations, + }); + const dependencies: ApplicationOperationsDependencies = { + getLocalDatabase: () => ({ getDBEntry }), + storageAccess: { + ensureDir, + readHiddenFileBinary, + readHiddenFileText, + writeHiddenFileAuto, + }, + path: { + filenameToUnifiedKey: (path, term) => `ix:${term}/CONFIG/${path.split("/").pop()}.md` as FilePathWithPrefix, + }, + log: vi.fn(), + getConfigDir: () => ".obsidian", + getDeviceAndVaultName: () => "device-a", + resolveJsonConflict: vi.fn(async () => false), + selectTextFile: vi.fn(async (): Promise<"A" | "B" | false> => false), + reloadPlugin, + askRestart, + snapshotOperations, + catalogueOperations, + }; + return { + application: new ApplicationOperations(dependencies), + dependencies, + events, + setUseV2: (value: boolean) => { + usePluginSyncV2 = value; + }, + getDBEntry, + persistence: { deleteConfigOnDatabase, storeCustomisationFileV2, storeCustomizationFiles }, + catalogue: { findPlugins, updatePluginList, updatePluginListV2 }, + storage: { ensureDir, readHiddenFileBinary, readHiddenFileText, writeHiddenFileAuto }, + reloadPlugin, + askRestart, + }; +} + +const display = { + documentPath: "ix:device-a/PLUGIN_DATA/example.md" as FilePathWithPrefix, + category: "PLUGIN_DATA", + name: "example", + term: "device-a", + files: [ + { filename: "plugins/example/data.json", data: ["a"], mtime: 1, size: 1 }, + { filename: "plugins/example/other.json", data: ["b"], mtime: 2, size: 1 }, + ], + mtime: 2, +} satisfies IPluginDataExDisplay; + +describe("Customisation Sync application operations", () => { + it("keeps file-level comparison clones and duplication behaviour inside the owner", async () => { + const fixture = createOperations(); + const compareUsingDisplayData = vi + .spyOn(fixture.application, "compareUsingDisplayData") + .mockResolvedValue(true); + + await expect( + fixture.application.compareFileUsingDisplayData(display, display, "plugins/example/data.json") + ).resolves.toBe(true); + const [left, right, compareEach] = compareUsingDisplayData.mock.calls[0]; + expect(left.files.map((file) => file.filename)).toEqual(["plugins/example/data.json"]); + expect(right.files.map((file) => file.filename)).toEqual(["plugins/example/data.json"]); + expect(compareEach).toBe(true); + expect(display.files).toHaveLength(2); + + await fixture.application.duplicateData(display, "device-b"); + expect(fixture.persistence.storeCustomizationFiles).toHaveBeenCalledWith( + ".obsidian/plugins/example/data.json", + "device-b" + ); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledWith(false, "ix:device-b/CONFIG/data.json.md"); + }); + + it("uses the compared filename for legacy file comparisons", async () => { + const fixture = createOperations(); + + await expect( + fixture.application.compareFileUsingDisplayData(display, display, "plugins/example/data.json") + ).resolves.toBe(false); + expect(fixture.dependencies.resolveJsonConflict).toHaveBeenCalledWith( + "data.json", + expect.any(Array), + "device-a", + expect.any(Function) + ); + }); + + it("awaits V1 refreshes before the explicit effect and preserves reload ordering", async () => { + const fixture = createOperations(); + fixture.setUseV2(false); + fixture.persistence.storeCustomizationFiles.mockImplementation(async () => { + fixture.events.push("persist"); + return { + value: true, + status: "saved" as const, + refreshes: [ + { + mode: "v1" as const, + timing: "await" as const, + path: "ix:device-a/PLUGIN_MAIN/example.md" as FilePathWithPrefix, + }, + ], + }; + }); + fixture.getDBEntry.mockResolvedValue({ + data: codec.serialize({ + category: "PLUGIN_MAIN", + name: "example", + term: "device-a", + files: [{ filename: "plugins/example/main.js", data: ["source"], mtime: 1, size: 6 }], + mtime: 1, + } satisfies PluginDataEx), + } as LoadedEntry); + + const data = { ...display, category: "PLUGIN_MAIN", name: "example" } satisfies IPluginDataExDisplay; + await expect(fixture.application.applyData(data, "replacement")).resolves.toBe(true); + + expect(fixture.events).toEqual([ + "ensure-dir", + "write-file", + "persist", + "refresh-v1:false", + "refresh-v1:true", + "reload-plugin", + ]); + expect(fixture.reloadPlugin).toHaveBeenCalledWith(".obsidian", "example"); + expect(asyncHarness.delay).toHaveBeenCalledWith(100); + }); + + it("starts V2 catalogue refreshes without awaiting them", async () => { + const fixture = createOperations(); + let releaseRefresh!: () => void; + const refresh = new Promise((resolve) => { + releaseRefresh = resolve; + }); + fixture.setUseV2(true); + fixture.persistence.storeCustomisationFileV2.mockResolvedValue({ + value: true, + status: "saved", + refreshes: [ + { + mode: "v2", + timing: "fire-and-forget", + path: "ix:device-a/CONFIG/app.json%app.json" as FilePathWithPrefix, + }, + ], + }); + fixture.catalogue.updatePluginListV2.mockImplementation(async () => await refresh); + const data = new PluginDataExDisplayV2( + { + ...display, + files: [{ filename: "app.json", data: ["source"], mtime: 1, size: 6 }], + }, + new Map() + ); + + await expect(fixture.application.applyData(data, "replacement")).resolves.toBe(true); + expect(fixture.catalogue.updatePluginListV2).toHaveBeenCalledWith( + false, + "ix:device-a/CONFIG/app.json%app.json" + ); + releaseRefresh(); + await refresh; + }); + + it("deletes the V2 files and binder through the direct owners", async () => { + const fixture = createOperations(); + fixture.setUseV2(true); + const v2Path = "ix:device-a/PLUGIN_DATA/example%data.json" as FilePathWithPrefix; + const binderPath = "ix:device-a/PLUGIN_DATA/example.md" as FilePathWithPrefix; + const v2Entry = new PluginDataExDisplayV2( + { + ...display, + documentPath: binderPath, + files: [ + { + filename: "data.json", + path: v2Path, + data: ["source"], + mtime: 1, + ctime: 1, + size: 6, + datatype: "plain", + } as never, + ], + }, + new Map() + ); + fixture.catalogue.findPlugins.mockReturnValue([v2Entry]); + + await expect( + fixture.application.deleteData({ + ...display, + documentPath: binderPath, + }) + ).resolves.toBe(true); + + expect(fixture.persistence.deleteConfigOnDatabase).toHaveBeenNthCalledWith(1, v2Path, false); + expect(fixture.persistence.deleteConfigOnDatabase).toHaveBeenNthCalledWith(2, binderPath, false); + expect(fixture.catalogue.updatePluginList).toHaveBeenNthCalledWith(1, false, v2Path); + expect(fixture.catalogue.updatePluginList).toHaveBeenNthCalledWith(2, false, binderPath); + }); +}); diff --git a/src/features/ConfigSync/catalogueMigration.ts b/src/features/ConfigSync/catalogueMigration.ts new file mode 100644 index 00000000..8c7d63c8 --- /dev/null +++ b/src/features/ConfigSync/catalogueMigration.ts @@ -0,0 +1,126 @@ +import { createBlob, getDocDataAsArray } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import type { + AnyEntry, + FilePathWithPrefix, + LOG_LEVEL, + SavingEntry, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { IPathService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; + +import { ICXHeader } from "@/common/types.ts"; +import type { SnapshotPersistence } from "./snapshotPersistence.ts"; +import type { CustomisationSyncReadCodec } from "./customisationSyncReadOperations.ts"; + +type CatalogueMigrationDatabase = Pick; + +type CatalogueMigrationCodec = Pick & { + dummyHead: string; + dummyEnd: string; +}; + +export type CatalogueMigrationDependencies = { + getLocalDatabase(): CatalogueMigrationDatabase; + path: Pick; + log: LogFunction; + snapshotPersistence: Pick; + refreshV1(showMessage: boolean, path: FilePathWithPrefix): Promise; + codec: CatalogueMigrationCodec; +}; + +/** Bridges persisted V1 binders into the V2 per-file document format. */ +export class CatalogueMigration { + constructor(private readonly dependencies: CatalogueMigrationDependencies) {} + + private _log(message: unknown, level?: LOG_LEVEL, key?: string): void { + this.dependencies.log(message, level, key); + } + + async migrateV1ToV2(showMessage: boolean, entry: AnyEntry): Promise { + const v1Path = entry.path; + this._log(`Migrating ${entry.path} to V2`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO); + if (entry.deleted) { + this._log(`The entry ${v1Path} is already deleted`, LOG_LEVEL_VERBOSE); + return; + } + // Compatibility question: the inherited conjunction admits any `ix:` + // path or any `.md` path, although the log describes a stricter binder + // check. Preserve it until malformed migration candidates are covered. + if (!v1Path.endsWith(".md") && !v1Path.startsWith(ICXHeader)) { + this._log(`The entry ${v1Path} is not a customisation sync binder`, LOG_LEVEL_VERBOSE); + return; + } + if (v1Path.indexOf("%") !== -1) { + this._log(`The entry ${v1Path} is already migrated`, LOG_LEVEL_VERBOSE); + return; + } + const loadedEntry = await this.dependencies.getLocalDatabase().getDBEntry(v1Path); + if (!loadedEntry) { + this._log(`The entry ${v1Path} is not found`, LOG_LEVEL_VERBOSE); + return; + } + + const pluginData = this.dependencies.codec.deserialize(getDocDataAsArray(loadedEntry.data), {}) as { + category: string; + files: Array<{ filename: string; data: string[] }>; + }; + const prefixPath = v1Path.slice(0, -".md".length) + "%"; + const category = pluginData.category; + + for (const f of pluginData.files) { + const stripTable: Record = { + CONFIG: 0, + THEME: 2, + SNIPPET: 1, + PLUGIN_MAIN: 2, + PLUGIN_DATA: 2, + PLUGIN_ETC: 2, + }; + const deletePrefixCount = stripTable?.[category] ?? 1; + const relativeFilename = f.filename.split("/").slice(deletePrefixCount).join("/"); + const v2Path = (prefixPath + relativeFilename) as FilePathWithPrefix; + this._log(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`, LOG_LEVEL_VERBOSE); + const newId = await this.dependencies.path.path2id(v2Path); + + const data = createBlob([ + this.dependencies.codec.dummyHead, + this.dependencies.codec.dummyEnd, + ...getDocDataAsArray(f.data), + ]); + const saving: SavingEntry = { + ...loadedEntry, + _rev: undefined, + _id: newId, + path: v2Path, + data, + datatype: "plain", + type: "plain", + children: [], + eden: {}, + }; + const result = await this.dependencies.getLocalDatabase().putDBEntry(saving); + if (result && result.ok) { + this._log(`Migrated ${v1Path} / ${f.filename} to ${v2Path}`, LOG_LEVEL_INFO); + const deletion = await this.dependencies.snapshotPersistence.deleteConfigOnDatabase(v1Path); + const deleted = deletion.value; + if (deleted) { + this._log(`Deleted ${v1Path} successfully`, LOG_LEVEL_INFO); + } else { + this._log(`Failed to delete ${v1Path}`, LOG_LEVEL_NOTICE); + } + // Compatibility: the inherited migration called the context + // deletion wrapper, which awaited its V1 catalogue refresh. + // Apply that refresh explicitly now that deletion is a host- + // neutral persistence operation, and only when deletion emitted + // the same mutation outcome. + for (const refresh of deletion.refreshes) { + if (refresh.mode == "v1" && refresh.timing == "await") { + await this.dependencies.refreshV1(false, refresh.path); + } + } + } + } + } +} diff --git a/src/features/ConfigSync/catalogueOperations.ts b/src/features/ConfigSync/catalogueOperations.ts new file mode 100644 index 00000000..daad570a --- /dev/null +++ b/src/features/ConfigSync/catalogueOperations.ts @@ -0,0 +1,216 @@ +import { parseYaml } from "@/deps.ts"; +import { writable } from "svelte/store"; +import type { + AnyEntry, + FilePathWithPrefix, + LoadedEntry, + LOG_LEVEL, + ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { ICXHeader } from "@/common/types.ts"; +import { fireAndForget } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { digestHash } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/hash"; +import { QueueProcessor } from "octagonal-wheels/concurrency/processor"; +import { reactiveSource, type ReactiveSource } from "octagonal-wheels/dataobject/reactive"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { IPathService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; + +import { CatalogueMigration } from "./catalogueMigration.ts"; +import { CatalogueState } from "./catalogueState.ts"; +import { CatalogueV1 } from "./catalogueV1.ts"; +import { CatalogueV2 } from "./catalogueV2.ts"; +import { createCustomisationSyncCodec } from "./customisationSyncCodec.ts"; +import type { SnapshotPersistence } from "./snapshotPersistence.ts"; +import type { IPluginDataExDisplay, LoadedEntryPluginDataExFile } from "./customisationSyncView.ts"; + +const { + serialize, + deserialize, + dummyHead: DUMMY_HEAD, + dummyEnd: DUMMY_END, +} = createCustomisationSyncCodec({ digestHash, parseYaml }); +const READ_CODEC = { deserialize, serialize }; +const MIGRATION_CODEC = { deserialize, dummyHead: DUMMY_HEAD, dummyEnd: DUMMY_END }; +const V2_CODEC = { dummyEnd: DUMMY_END }; + +type CatalogueSettings = Pick; + +type CatalogueDatabase = Pick; + +export type CatalogueOperationsDependencies = { + getSettings(): CatalogueSettings; + getLocalDatabase(): CatalogueDatabase; + path: Pick; + log: LogFunction; + snapshotPersistence: Pick; + publishScanCount(count: number): void; +}; + +/** Coordinates the shared catalogue state, scan queue, and format modules. */ +export class CatalogueOperations { + private readonly dependencies: CatalogueOperationsDependencies; + private readonly catalogueState = new CatalogueState(); + private readonly scanProgress = reactiveSource(0); + private readonly pluginScanningChanged: Parameters["onChanged"]>[0] = (event) => { + this.enumerationActive.set(event.value != 0); + this.dependencies.publishScanCount(event.value); + }; + private readonly pluginScanProcessor: QueueProcessor; + private readonly catalogueV1: CatalogueV1; + private readonly catalogueV2: CatalogueV2; + private readonly catalogueMigration: CatalogueMigration; + + readonly enumerationActive = writable(false); + readonly catalogue = this.catalogueState.catalogue; + readonly migrationProgress = this.catalogueState.migrationProgress; + readonly manifests = this.catalogueState.manifests; + + constructor(dependencies: CatalogueOperationsDependencies) { + this.dependencies = dependencies; + + this.catalogueV1 = new CatalogueV1({ + getLocalDatabase: () => this.dependencies.getLocalDatabase(), + path: { + getPath: (entry) => this.getPath(entry), + }, + log: (message, level, key) => this._log(message, level, key), + state: this.catalogueState, + }); + this.catalogueV2 = new CatalogueV2({ + getLocalDatabase: () => this.dependencies.getLocalDatabase(), + log: (message, level, key) => this._log(message, level, key), + state: this.catalogueState, + codec: V2_CODEC, + }); + this.catalogueMigration = new CatalogueMigration({ + getLocalDatabase: () => this.dependencies.getLocalDatabase(), + path: { + path2id: (path) => this.path2id(path), + }, + log: (message, level, key) => this._log(message, level, key), + snapshotPersistence: this.dependencies.snapshotPersistence, + refreshV1: async (showMessage, path) => await this.updatePluginList(showMessage, path), + codec: MIGRATION_CODEC, + }); + + this.scanProgress.onChanged(this.pluginScanningChanged); + // The single queue deliberately chooses V1 loading or migration when + // each item starts. Settings can change after enqueueing an item. + this.pluginScanProcessor = new QueueProcessor( + async (v: AnyEntry[]) => { + const plugin = v[0]; + if (this.dependencies.getSettings().usePluginSyncV2) { + await this.migrateV1ToV2(false, plugin); + return []; + } + await this.catalogueV1.load(plugin, READ_CODEC); + return []; + }, + { + suspended: false, + batchSize: 1, + concurrentLimit: 10, + delay: 100, + yieldThreshold: 10, + maintainDelay: false, + totalRemainingReactiveSource: this.scanProgress, + } + ).startPipeline(); + } + + private get settings() { + return this.dependencies.getSettings(); + } + + private get localDatabase() { + return this.dependencies.getLocalDatabase(); + } + + private getPath(entry: AnyEntry): FilePathWithPrefix { + return this.dependencies.path.getPath(entry); + } + + private async path2id(filename: FilePathWithPrefix) { + return await this.dependencies.path.path2id(filename); + } + + private _log(message: unknown, level?: LOG_LEVEL, key?: string): void { + this.dependencies.log(message, level, key); + } + + /** The current manifest lookup passed to V2 display rows. */ + get manifestLookup() { + return this.catalogueState.manifestLookup; + } + + /** Returns every row matching a document path, preserving legacy duplicates. */ + findPlugins(documentPath: FilePathWithPrefix | string): readonly IPluginDataExDisplay[] { + return this.catalogueState.findPlugins(documentPath); + } + + dispose(): void { + this.pluginScanProcessor.terminate(); + this.scanProgress.offChanged(this.pluginScanningChanged); + this.enumerationActive.set(false); + this.dependencies.publishScanCount(0); + } + + async reloadPluginList(showMessage: boolean): Promise { + this.catalogueState.clearForReload(); + await this.updatePluginList(showMessage); + } + + async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise { + if (!this.settings.usePluginSync) { + this.pluginScanProcessor.clearQueue(); + this.catalogueState.clearForDisabledRefresh(); + return; + } + try { + this.catalogueState.beginUpdate(); + const updatedDocumentId = updatedDocumentPath ? await this.path2id(updatedDocumentPath) : ""; + const plugins = updatedDocumentPath + ? this.localDatabase.findEntries(updatedDocumentId, updatedDocumentId + "\u{10ffff}", { + include_docs: true, + key: updatedDocumentId, + limit: 1, + }) + : this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true }); + for await (const v of plugins) { + if (v.deleted || v._deleted) continue; + if (v.path.indexOf("%") !== -1) { + fireAndForget(() => this.updatePluginListV2(showMessage, v.path)); + continue; + } + + const path = v.path || this.getPath(v); + if (updatedDocumentPath && updatedDocumentPath != path) continue; + this.pluginScanProcessor.enqueue(v); + } + } finally { + this.enumerationActive.set(false); + this.catalogueState.endUpdate(); + } + this.enumerationActive.set(false); + } + + async createPluginDataExFileV2( + unifiedPathV2: FilePathWithPrefix, + loaded?: LoadedEntry + ): Promise { + return await this.catalogueV2.createPluginDataExFileV2(unifiedPathV2, loaded); + } + + createPluginDataFromV2(unifiedPathV2: FilePathWithPrefix) { + return this.catalogueV2.createPluginDataFromV2(unifiedPathV2); + } + + async updatePluginListV2(showMessage: boolean, unifiedFilenameWithKey: FilePathWithPrefix): Promise { + await this.catalogueV2.updatePluginListV2(showMessage, unifiedFilenameWithKey); + } + + private async migrateV1ToV2(showMessage: boolean, entry: AnyEntry): Promise { + await this.catalogueMigration.migrateV1ToV2(showMessage, entry); + } +} diff --git a/src/features/ConfigSync/catalogueOperations.unit.spec.ts b/src/features/ConfigSync/catalogueOperations.unit.spec.ts new file mode 100644 index 00000000..43577c9c --- /dev/null +++ b/src/features/ConfigSync/catalogueOperations.unit.spec.ts @@ -0,0 +1,222 @@ +import { get } from "svelte/store"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const testState = vi.hoisted(() => ({ + processors: [] as Array<{ + clearQueue: ReturnType; + enqueue: ReturnType; + terminate: ReturnType; + startPipeline: ReturnType; + process: (entries: AnyEntry[]) => Promise; + }>, + reactiveSources: [] as Array<{ + value: number; + onChanged: ReturnType; + offChanged: ReturnType; + }>, +})); + +vi.mock("@/deps.ts", () => ({ + parseYaml: vi.fn(), +})); +vi.mock("@/common/types.ts", () => ({ + ICXHeader: "ix:", +})); +vi.mock("@/common/utils.ts", () => ({ + fireAndForget: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("octagonal-wheels/concurrency/processor", () => ({ + QueueProcessor: class QueueProcessor { + clearQueue = vi.fn(); + enqueue = vi.fn(); + terminate = vi.fn(); + startPipeline = vi.fn(() => this); + + process: (entries: AnyEntry[]) => Promise; + + constructor(process: (entries: AnyEntry[]) => Promise) { + this.process = process; + testState.processors.push(this); + } + }, +})); +vi.mock("octagonal-wheels/dataobject/reactive", () => ({ + reactiveSource: vi.fn((value: number) => { + const source = { + value, + onChanged: vi.fn(), + offChanged: vi.fn(), + }; + testState.reactiveSources.push(source); + return source; + }), +})); + +import { scheduleTask } from "@/common/utils.ts"; +import type { + AnyEntry, + DocumentID, + FilePathWithPrefix, + LoadedEntry, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { digestHash } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/hash"; +import { createCustomisationSyncCodec } from "./customisationSyncCodec.ts"; +import { CatalogueOperations, type CatalogueOperationsDependencies } from "./catalogueOperations.ts"; +import type { SnapshotPersistenceResult } from "./snapshotPersistence.ts"; + +const codec = createCustomisationSyncCodec({ + digestHash, + parseYaml: () => undefined, +}); +const v2Path = "ix:device-a/PLUGIN_DATA/example%data.json" as FilePathWithPrefix; +const v1Path = "ix:device-a/CONFIG/app.json.md" as FilePathWithPrefix; + +function loadedEntry(): LoadedEntry { + const data = `${codec.dummyHead}${codec.dummyEnd}${btoa("example data")}`; + return { + _id: "entry-id", + _rev: "1-a", + path: v2Path, + type: "plain", + datatype: "plain", + data, + ctime: 10, + mtime: 20, + size: data.length, + children: [], + eden: {}, + } as unknown as LoadedEntry; +} + +function createOperations() { + const settings = { usePluginSync: true, usePluginSyncV2: true }; + const database = { + findEntries: vi.fn(async function* () { + // No refresh entries are needed by the focused V2 test. + }), + getDBEntry: vi.fn(async () => loadedEntry()), + putDBEntry: vi.fn(async () => ({ ok: true, id: "entry-id", rev: "2-b" })), + }; + const deleteConfigOnDatabase = vi.fn( + async (): Promise> => ({ + value: true, + status: "missing", + refreshes: [], + }) + ); + const snapshotPersistence = { + deleteConfigOnDatabase, + }; + const dependencies: CatalogueOperationsDependencies = { + getSettings: () => settings, + getLocalDatabase: () => database, + path: { + getPath: (entry) => entry.path, + path2id: async (path) => path as unknown as DocumentID, + }, + log: vi.fn(), + snapshotPersistence, + publishScanCount: vi.fn(), + }; + return { + database, + dependencies, + settings, + operations: new CatalogueOperations(dependencies), + snapshotPersistence, + }; +} + +describe("Customisation Sync catalogue operations", () => { + beforeEach(() => { + testState.processors.length = 0; + testState.reactiveSources.length = 0; + vi.clearAllMocks(); + }); + + it("owns and releases the active shared scan processor and its progress subscription", () => { + const { dependencies, operations } = createOperations(); + operations.enumerationActive.set(true); + + operations.dispose(); + + expect(testState.processors).toHaveLength(1); + expect(testState.processors[0].terminate).toHaveBeenCalledOnce(); + expect(testState.reactiveSources[0].offChanged).toHaveBeenCalledOnce(); + expect(get(operations.enumerationActive)).toBe(false); + expect(dependencies.publishScanCount).toHaveBeenCalledWith(0); + }); + + it("chooses loading or migration when queued work starts", async () => { + const { operations, settings } = createOperations(); + const migrate = vi + .spyOn( + operations as unknown as { migrateV1ToV2: (showMessage: boolean, entry: AnyEntry) => Promise }, + "migrateV1ToV2" + ) + .mockResolvedValue(undefined); + const entry = { path: v1Path, deleted: false } as AnyEntry; + + settings.usePluginSyncV2 = false; + await testState.processors[0].process([entry]); + expect(migrate).not.toHaveBeenCalled(); + + settings.usePluginSyncV2 = true; + await testState.processors[0].process([entry]); + expect(migrate).toHaveBeenCalledOnce(); + operations.dispose(); + }); + + it("keeps V2 row publication delayed behind the process-global refresh task", async () => { + const { operations } = createOperations(); + + await operations.updatePluginListV2(false, v2Path); + + expect(get(operations.catalogue)).toEqual([]); + expect(scheduleTask).toHaveBeenCalledWith("updatePluginListV2", 100, expect.any(Function)); + + const publish = vi.mocked(scheduleTask).mock.calls[0]?.[2] as (() => void) | undefined; + publish?.(); + + expect(get(operations.catalogue)).toHaveLength(1); + expect(get(operations.catalogue)[0]).toMatchObject({ + documentPath: "ix:device-a/PLUGIN_DATA/example.md", + files: [{ filename: "plugins/example/data.json" }], + }); + operations.dispose(); + }); + + it("uses the persistence deletion outcome and explicitly awaits migration refresh", async () => { + const { database, operations, snapshotPersistence } = createOperations(); + const loadedV1 = { + ...loadedEntry(), + path: v1Path, + data: codec.serialize({ + category: "CONFIG", + name: "app.json", + term: "device-a", + files: [{ filename: "app.json", data: [btoa("config")], mtime: 10, size: 6 }], + mtime: 10, + }), + } as LoadedEntry; + database.getDBEntry.mockResolvedValue(loadedV1); + snapshotPersistence.deleteConfigOnDatabase.mockResolvedValue({ + value: true, + status: "deleted", + refreshes: [{ mode: "v1", timing: "await", path: v1Path }], + }); + const refresh = vi.spyOn(operations, "updatePluginList").mockResolvedValue(undefined); + + await ( + operations as unknown as { + migrateV1ToV2(showMessage: boolean, entry: LoadedEntry): Promise; + } + ).migrateV1ToV2(false, { path: v1Path, deleted: false } as LoadedEntry); + + expect(database.putDBEntry).toHaveBeenCalledOnce(); + expect(snapshotPersistence.deleteConfigOnDatabase).toHaveBeenCalledWith(v1Path); + expect(refresh).toHaveBeenCalledWith(false, v1Path); + operations.dispose(); + }); +}); diff --git a/src/features/ConfigSync/customisationSyncCatalogueState.ts b/src/features/ConfigSync/catalogueState.ts similarity index 94% rename from src/features/ConfigSync/customisationSyncCatalogueState.ts rename to src/features/ConfigSync/catalogueState.ts index 3fd4aed8..b64b607c 100644 --- a/src/features/ConfigSync/customisationSyncCatalogueState.ts +++ b/src/features/ConfigSync/catalogueState.ts @@ -9,11 +9,12 @@ import type { IPluginDataExDisplay } from "./customisationSyncView.ts"; /** * Owns the transient catalogue projection used by Customisation Sync. * - * The database and storage operations remain in the context. This owner only - * coordinates the in-memory rows, their reactive publications, manifest - * lookup, and the update counter which is derived from those operations. + * The database and storage operations remain in the catalogue modules. This + * owner only coordinates the in-memory rows, their reactive publications, + * manifest lookup, and the update counter which is derived from those + * operations. */ -export class CustomisationSyncCatalogueState { +export class CatalogueState { private catalogueRows: IPluginDataExDisplay[] = []; private readonly manifestByKey = new Map(); private readonly loadedManifestMTimeByKey = new Map(); @@ -33,7 +34,7 @@ export class CustomisationSyncCatalogueState { return this.loadedManifestMTimeByKey; } - /** Returns the authoritative row for a V1 document path, when present. */ + /** Returns the authoritative row for a document path, when present. */ findPlugin(documentPath: FilePathWithPrefix | string): IPluginDataExDisplay | undefined { return this.catalogueRows.find((entry) => entry.documentPath == documentPath); } diff --git a/src/features/ConfigSync/customisationSyncCatalogueState.unit.spec.ts b/src/features/ConfigSync/catalogueState.unit.spec.ts similarity index 90% rename from src/features/ConfigSync/customisationSyncCatalogueState.unit.spec.ts rename to src/features/ConfigSync/catalogueState.unit.spec.ts index f57e34db..9f34744b 100644 --- a/src/features/ConfigSync/customisationSyncCatalogueState.unit.spec.ts +++ b/src/features/ConfigSync/catalogueState.unit.spec.ts @@ -3,7 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import type { PluginManifest } from "@/deps.ts"; import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import { CustomisationSyncCatalogueState } from "./customisationSyncCatalogueState.ts"; +import { CatalogueState } from "./catalogueState.ts"; import { PluginDataExDisplayV2 } from "./customisationSyncModel.ts"; import type { IPluginDataExDisplay, LoadedEntryPluginDataExFile } from "./customisationSyncView.ts"; @@ -30,7 +30,7 @@ function file(filename: string, mtime: number): LoadedEntryPluginDataExFile { describe("Customisation Sync catalogue state", () => { it("publishes V1 replacement and keeps V2 replacement delayed", async () => { - const state = new CustomisationSyncCatalogueState(); + const state = new CatalogueState(); const setCatalogue = vi.spyOn(state.catalogue, "set"); const row = display(); @@ -54,7 +54,7 @@ describe("Customisation Sync catalogue state", () => { }); it("retains the first parsed manifest and records failed mtimes", () => { - const state = new CustomisationSyncCatalogueState(); + const state = new CatalogueState(); const first = { name: "First", version: "1.0.0" } as PluginManifest; const parseManifest = vi.fn(() => first); @@ -69,7 +69,7 @@ describe("Customisation Sync catalogue state", () => { expect(state.loadedManifestMTime.get("device-a/plugins/example")).toBe(20); expect(parseManifest).toHaveBeenCalledOnce(); - const failedState = new CustomisationSyncCatalogueState(); + const failedState = new CatalogueState(); const onParseError = vi.fn(); const failure = new SyntaxError("invalid"); failedState.processManifest( @@ -88,7 +88,7 @@ describe("Customisation Sync catalogue state", () => { }); it("clears rows and loaded mtimes on reload while retaining manifest lookup", () => { - const state = new CustomisationSyncCatalogueState(); + const state = new CatalogueState(); const key = "device-a/plugins/example"; state.processManifest(key, 20, () => ({ name: "Example" }) as PluginManifest); state.replacePlugin(display()); @@ -102,7 +102,7 @@ describe("Customisation Sync catalogue state", () => { }); it("keeps manifest caches through the narrower disabled refresh", () => { - const state = new CustomisationSyncCatalogueState(); + const state = new CatalogueState(); const key = "device-a/plugins/example"; state.processManifest(key, 20, () => ({ name: "Example" }) as PluginManifest); state.replacePlugin(display()); @@ -115,7 +115,7 @@ describe("Customisation Sync catalogue state", () => { }); it("tracks V2 updates through migration progress", () => { - const state = new CustomisationSyncCatalogueState(); + const state = new CatalogueState(); state.beginUpdate(); state.beginUpdate(); diff --git a/src/features/ConfigSync/catalogueV1.ts b/src/features/ConfigSync/catalogueV1.ts new file mode 100644 index 00000000..2af4c5cb --- /dev/null +++ b/src/features/ConfigSync/catalogueV1.ts @@ -0,0 +1,50 @@ +import type { AnyEntry, LOG_LEVEL } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { IPathService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; + +import { CatalogueState } from "./catalogueState.ts"; +import { loadCustomisationDisplayData, type CustomisationSyncReadCodec } from "./customisationSyncReadOperations.ts"; + +type CatalogueV1Database = Pick; + +export type CatalogueV1Dependencies = { + getLocalDatabase(): CatalogueV1Database; + path: Pick; + log: LogFunction; + state: CatalogueState; +}; + +/** Loads and publishes legacy V1 catalogue rows. */ +export class CatalogueV1 { + constructor(private readonly dependencies: CatalogueV1Dependencies) {} + + private _log(message: unknown, level?: LOG_LEVEL, key?: string): void { + this.dependencies.log(message, level, key); + } + + async load(entry: AnyEntry, codec: CustomisationSyncReadCodec): Promise { + const path = entry.path || this.dependencies.path.getPath(entry); + const oldEntry = this.dependencies.state.findPlugin(path); + if (oldEntry && oldEntry.mtime == entry.mtime) return; + try { + const pluginData = await loadCustomisationDisplayData( + { + getLocalDatabase: () => this.dependencies.getLocalDatabase(), + path: this.dependencies.path, + log: this.dependencies.log, + }, + path, + codec + ); + if (pluginData) { + this.dependencies.state.replacePlugin(pluginData); + } + // Failed to load + } catch (ex) { + this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); + this._log(ex, LOG_LEVEL_VERBOSE); + } + } +} diff --git a/src/features/ConfigSync/catalogueV2.ts b/src/features/ConfigSync/catalogueV2.ts new file mode 100644 index 00000000..711a3468 --- /dev/null +++ b/src/features/ConfigSync/catalogueV2.ts @@ -0,0 +1,123 @@ +import type { PluginManifest } from "@/deps.ts"; +import type { FilePathWithPrefix, LoadedEntry, LOG_LEVEL } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; + +import { scheduleTask } from "@/common/utils.ts"; +import { CatalogueState } from "./catalogueState.ts"; +import { PluginDataExDisplayV2 } from "./customisationSyncModel.ts"; +import { parseCustomisationSyncV2DocumentPath } from "./customisationSyncPaths.ts"; +import { decodeCustomisationSyncV2File, loadCustomisationV2Entry } from "./customisationSyncReadOperations.ts"; +import type { LoadedEntryPluginDataExFile } from "./customisationSyncView.ts"; + +type CatalogueV2Database = Pick; + +export type CatalogueV2Dependencies = { + getLocalDatabase(): CatalogueV2Database; + log: LogFunction; + state: CatalogueState; + codec: { dummyEnd: string }; +}; + +/** Builds, updates, and publishes V2 catalogue rows and manifests. */ +export class CatalogueV2 { + constructor(private readonly dependencies: CatalogueV2Dependencies) {} + + private _log(message: unknown, level?: LOG_LEVEL, key?: string): void { + this.dependencies.log(message, level, key); + } + + get manifestLookup() { + return this.dependencies.state.manifestLookup; + } + + async createPluginDataExFileV2( + unifiedPathV2: FilePathWithPrefix, + loaded?: LoadedEntry + ): Promise { + // Compatibility: a caller-supplied entry bypasses the database lookup + // and the isLoadedEntry check performed by loadCustomisationV2Entry. + const loadedEntry = + loaded ?? + (await loadCustomisationV2Entry( + { + getLocalDatabase: () => this.dependencies.getLocalDatabase(), + log: this.dependencies.log, + }, + unifiedPathV2 + )); + if (!loadedEntry) return false; + const { confKey, file, isManifest } = decodeCustomisationSyncV2File( + unifiedPathV2, + loadedEntry, + this.dependencies.codec.dummyEnd + ); + if (isManifest) { + this.dependencies.state.processManifest( + confKey, + file.mtime, + () => JSON.parse(file.data[0]) as PluginManifest, + (error) => { + this._log( + `The file ${loadedEntry.path} seems to manifest, but could not be decoded as JSON`, + LOG_LEVEL_VERBOSE + ); + this._log(error, LOG_LEVEL_VERBOSE); + } + ); + } + return file; + } + + createPluginDataFromV2(unifiedPathV2: FilePathWithPrefix): PluginDataExDisplayV2 | undefined { + const { category, device, key, pathV1 } = parseCustomisationSyncV2DocumentPath(unifiedPathV2); + if (category == "") return; + + return new PluginDataExDisplayV2( + { + documentPath: pathV1, + category, + name: key, + term: `${device}`, + files: [], + mtime: 0, + }, + this.dependencies.state.manifestLookup + ); + } + + async updatePluginListV2(showMessage: boolean, unifiedFilenameWithKey: FilePathWithPrefix): Promise { + // The public parameter is retained for the established catalogue + // signature; V2 publication has never used it. + void showMessage; + try { + this.dependencies.state.beginUpdate(); + const { pathV1 } = parseCustomisationSyncV2DocumentPath(unifiedFilenameWithKey); + + const oldEntry = this.dependencies.state.findPlugin(pathV1); + let entry: PluginDataExDisplayV2 | undefined; + // Compatibility question: when a V1 row is found first for this + // logical path, the inherited implementation constructs a fresh + // V2 row rather than looking for another existing V2 row. Preserve + // that selection until mixed-format catalogue races are covered. + if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) { + entry = this.createPluginDataFromV2(unifiedFilenameWithKey); + } else { + entry = oldEntry; + } + if (!entry) return; + + const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey); + // Compatibility: the inherited update always re-adds an empty V2 + // row after deleting its final file. + await this.dependencies.state.updateV2Plugin(entry, file, unifiedFilenameWithKey); + + scheduleTask("updatePluginListV2", 100, () => { + this.dependencies.state.publishCatalogue(); + }); + } finally { + this.dependencies.state.endUpdate(); + } + } +} diff --git a/src/features/ConfigSync/customisationSyncContext.command.unit.spec.ts b/src/features/ConfigSync/customisationSyncContext.command.unit.spec.ts index 4774fd66..be6a07cf 100644 --- a/src/features/ConfigSync/customisationSyncContext.command.unit.spec.ts +++ b/src/features/ConfigSync/customisationSyncContext.command.unit.spec.ts @@ -64,27 +64,17 @@ describe("CustomisationSyncContext commands", () => { expect(updatePluginList).toHaveBeenCalledWith(false, "ix:example"); }); - it("releases every owned processor and reactive subscription", () => { + it("delegates catalogue resource release during disposal", () => { const hideConfigurationNotice = vi.fn(); - const publishScanCount = vi.fn(); const periodicPluginSweepProcessor = { disable: vi.fn() }; - const pluginScanProcessor = { terminate: vi.fn() }; - const pluginScanProcessorV2 = { terminate: vi.fn() }; - const pluginScanningChanged = vi.fn(); - const offChanged = vi.fn(); - const setEnumerationActive = vi.fn(); + const catalogueOperations = { dispose: vi.fn() }; const configSync = Object.create(CustomisationSyncContext.prototype) as CustomisationSyncContext; Object.assign(configSync, { dependencies: createCustomisationSyncTestDependencies({ hideConfigurationNotice, - publishScanCount, }), periodicPluginSweepProcessor, - pluginScanProcessor, - pluginScanProcessorV2, - pluginScanningChanged, - scanProgress: { offChanged }, - enumerationActive: { set: setEnumerationActive }, + catalogueOperations, }); configSync.dispose(); @@ -92,11 +82,7 @@ describe("CustomisationSyncContext commands", () => { expect(cancelTask).toHaveBeenCalledWith("config-sync:updated-configuration"); expect(hideConfigurationNotice).toHaveBeenCalledOnce(); expect(periodicPluginSweepProcessor.disable).toHaveBeenCalledOnce(); - expect(pluginScanProcessor.terminate).toHaveBeenCalledOnce(); - expect(pluginScanProcessorV2.terminate).toHaveBeenCalledOnce(); - expect(offChanged).toHaveBeenCalledWith(pluginScanningChanged); - expect(setEnumerationActive).toHaveBeenCalledWith(false); - expect(publishScanCount).toHaveBeenCalledWith(0); + expect(catalogueOperations.dispose).toHaveBeenCalledOnce(); }); it("characterises the inherited setting-realisation gates pending separate review", async () => { diff --git a/src/features/ConfigSync/customisationSyncContext.routing.unit.spec.ts b/src/features/ConfigSync/customisationSyncContext.routing.unit.spec.ts index bbf79b50..9c8d8a88 100644 --- a/src/features/ConfigSync/customisationSyncContext.routing.unit.spec.ts +++ b/src/features/ConfigSync/customisationSyncContext.routing.unit.spec.ts @@ -90,9 +90,9 @@ describe("Customisation Sync raw-event admission", () => { it("rejects a path outside the recognised Customisation Sync categories", async () => { const { configSync, ownsLocalFile } = createConfigSync(); - await expect(configSync.serviceHandlers.processOptionalFileEvent(".obsidian/workspace" as FilePath)).resolves.toBe( - false - ); + await expect( + configSync.serviceHandlers.processOptionalFileEvent(".obsidian/workspace" as FilePath) + ).resolves.toBe(false); expect(ownsLocalFile).not.toHaveBeenCalled(); expect(scheduleTask).not.toHaveBeenCalled(); }); diff --git a/src/features/ConfigSync/customisationSyncContext.scan-routing.unit.spec.ts b/src/features/ConfigSync/customisationSyncContext.scan-routing.unit.spec.ts index d48b1a2f..9b644b33 100644 --- a/src/features/ConfigSync/customisationSyncContext.scan-routing.unit.spec.ts +++ b/src/features/ConfigSync/customisationSyncContext.scan-routing.unit.spec.ts @@ -1,5 +1,4 @@ import { describe, expect, it, vi } from "vitest"; -import type { FilePath, FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; vi.mock("@/deps.ts", () => ({ diff_match_patch: class DiffMatchPatch {}, @@ -18,116 +17,21 @@ vi.mock("@/common/utils.ts", () => ({ isPluginMetadata: vi.fn(), scheduleTask: vi.fn(), })); -vi.mock("@/common/PeriodicProcessor.ts", () => ({ - PeriodicProcessor: class PeriodicProcessor {}, -})); vi.mock("@/common/translation", () => ({ $msg: vi.fn((message: string) => message), })); -vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ - getObsidianCommunityPluginManager: vi.fn(), -})); import { CustomisationSyncContext } from "./customisationSyncContext.ts"; -import { createCustomisationSyncTestDependencies } from "./customisationSyncContext.unit.fixture.ts"; -const PATH = ".obsidian/plugins/example/data.json" as FilePath; -const V1_PATH = "ix:device-a/PLUGIN_DATA/example.md" as FilePathWithPrefix; -const V2_PATH = "ix:device-a/PLUGIN_DATA/example%data.json" as FilePathWithPrefix; +describe("Customisation Sync scan delegation", () => { + it("preserves the public scan argument and result through the focused owner", async () => { + const scanAllConfigFiles = vi.fn(async (_showMessage: boolean) => undefined); + const context = Object.create(CustomisationSyncContext.prototype) as CustomisationSyncContext; + Object.assign(context, { scanOperations: { scanAllConfigFiles } }); -function asyncEntries(entries: object[]) { - return { - async *[Symbol.asyncIterator]() { - yield* entries; - }, - }; -} + await expect(context.scanAllConfigFiles(true)).resolves.toBeUndefined(); -function createConfigSync(options: { - useV2: boolean; - localFiles?: FilePath[]; - databasePaths?: FilePathWithPrefix[]; - ownsLocalFile?: boolean; - ownsLocalDocument?: boolean; -}) { - const storeCustomizationFiles = vi.fn(async () => true); - const storeCustomisationFileV2 = vi.fn(async () => true); - const deleteConfigOnDatabase = vi.fn(async () => true); - const ownsLocalFile = vi.fn(() => options.ownsLocalFile ?? true); - const ownsLocalDocument = vi.fn(() => options.ownsLocalDocument ?? true); - const databaseEntries = (options.databasePaths ?? []).map((path) => ({ _id: path, path })); - const localDatabase = { - findEntries: vi.fn(() => asyncEntries(databaseEntries)), - allDocsRaw: vi.fn(async () => ({ - rows: databaseEntries.map((doc) => ({ doc: { ...doc, deleted: false } })), - })), - }; - const configSync = Object.create(CustomisationSyncContext.prototype) as CustomisationSyncContext; - Object.assign(configSync, { - dependencies: createCustomisationSyncTestDependencies({ - getConfigDir: () => ".obsidian", - getSettings: () => ({ - usePluginSync: true, - usePluginSyncV2: options.useV2, - usePluginEtc: true, - pluginSyncExtendedSetting: {}, - autoSweepPlugins: false, - autoSweepPluginsPeriodic: false, - watchInternalFileChanges: false, - notifyPluginOrSettingUpdated: false, - }), - getLocalDatabase: () => localDatabase as never, - ownsLocalFile, - ownsLocalDocument, - }), - scanInternalFiles: vi.fn(async () => options.localFiles ?? []), - pathOperations: { - isTargetPath: vi.fn(() => true), - filenameToUnifiedKey: vi.fn(() => V1_PATH), - filenameWithUnifiedKey: vi.fn(() => V2_PATH), - unifiedKeyPrefixOfTerminal: vi.fn(() => "ix:device-a/"), - }, - getPath: vi.fn((entry: { path: FilePathWithPrefix }) => entry.path), - storeCustomizationFiles, - storeCustomisationFileV2, - deleteConfigOnDatabase, - updatePluginList: vi.fn(async () => undefined), - _log: vi.fn(), - }); - return { - configSync, - deleteConfigOnDatabase, - ownsLocalDocument, - ownsLocalFile, - storeCustomisationFileV2, - storeCustomizationFiles, - }; -} - -describe("Customisation Sync scan ownership", () => { - it.each([ - ["V1", false], - ["V2", true], - ] as const)("does not store a %s local file assigned to another owner", async (_label, useV2) => { - const fixture = createConfigSync({ useV2, localFiles: [PATH], ownsLocalFile: false }); - - await fixture.configSync.scanAllConfigFiles(false); - - expect(fixture.ownsLocalFile).toHaveBeenCalledWith(PATH); - expect(fixture.storeCustomizationFiles).not.toHaveBeenCalled(); - expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); - }); - - it("does not create a deletion for an existing V1 document which is no longer locally owned", async () => { - const fixture = createConfigSync({ - useV2: false, - databasePaths: [V1_PATH], - ownsLocalDocument: false, - }); - - await fixture.configSync.scanAllConfigFiles(false); - - expect(fixture.ownsLocalDocument).toHaveBeenCalledWith(V1_PATH); - expect(fixture.deleteConfigOnDatabase).not.toHaveBeenCalled(); + expect(scanAllConfigFiles).toHaveBeenCalledOnce(); + expect(scanAllConfigFiles).toHaveBeenCalledWith(true); }); }); diff --git a/src/features/ConfigSync/customisationSyncContext.ts b/src/features/ConfigSync/customisationSyncContext.ts index 7171b18a..fc51d52d 100644 --- a/src/features/ConfigSync/customisationSyncContext.ts +++ b/src/features/ConfigSync/customisationSyncContext.ts @@ -1,51 +1,22 @@ -import { writable } from "svelte/store"; import type PouchDB from "pouchdb-core"; -import { type PluginManifest, parseYaml, normalizePath, diff_match_patch } from "@/deps.ts"; +import { normalizePath } from "@/deps.ts"; import type { EntryDoc, LoadedEntry, - InternalFileEntry, FilePathWithPrefix, FilePath, AnyEntry, - SavingEntry, diff_result, SYNC_MODE, ObsidianLiveSyncSettings, LOG_LEVEL, } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import { - LOG_LEVEL_DEBUG, - LOG_LEVEL_INFO, - LOG_LEVEL_NOTICE, - LOG_LEVEL_VERBOSE, - MODE_SELECTIVE, -} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { ICXHeader, PERIODIC_PLUGIN_SWEEP } from "@/common/types.ts"; -import { - createBlob, - createTextBlob, - delay, - fireAndForget, - getDocData, - getDocDataAsArray, - isDocContentSame, -} from "@vrtmrz/livesync-commonlib/compat/common/utils"; -import { digestHash } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/hash"; -import { arrayBufferToBase64, decodeBinary } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/convert"; -import { serialized, shareRunningResult } from "octagonal-wheels/concurrency/lock"; -import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; -import { cancelTask, EVEN, scheduleTask } from "@/common/utils.ts"; -import { QueueProcessor } from "octagonal-wheels/concurrency/processor"; -import { reactiveSource, type ReactiveSource } from "octagonal-wheels/dataobject/reactive"; -import { base64ToArrayBuffer, base64ToString } from "octagonal-wheels/binary/base64"; -import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; +import { cancelTask, scheduleTask } from "@/common/utils.ts"; import { $msg } from "@/common/translation"; -import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError"; import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; -import { parseCustomisationSyncV2DocumentPath } from "./customisationSyncPaths.ts"; -import { createCustomisationSyncCodec, type PluginDataEx } from "./customisationSyncCodec.ts"; import type { CustomisationSyncDialogView, CustomisationSyncUIControl, @@ -62,19 +33,14 @@ import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/ import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; import type { IPathService, IReplicationService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; -import { - collectOptionalFileSyncFiles, - type OptionalFileSyncFileTreeDependencies, -} from "@/features/optionalFileSyncFileTree.ts"; +import type { OptionalFileSyncFileTreeDependencies } from "@/features/optionalFileSyncFileTree.ts"; import { PluginDataExDisplayV2 } from "./customisationSyncModel.ts"; -import { - decodeCustomisationSyncV2File, - loadCustomisationDisplayData, - loadCustomisationV2Entry, - readCustomisationFile, -} from "./customisationSyncReadOperations.ts"; -import { CustomisationSyncCatalogueState } from "./customisationSyncCatalogueState.ts"; +import { ApplicationOperations, type ApplicationOperationsDependencies } from "./applicationOperations.ts"; import { CustomisationSyncRecentEventDeduplicator } from "./customisationSyncRecentEventDeduplicator.ts"; +import { CatalogueOperations, type CatalogueOperationsDependencies } from "./catalogueOperations.ts"; +import { SnapshotPersistence, type SnapshotPersistenceDependencies } from "./snapshotPersistence.ts"; +import { SnapshotOperations } from "./snapshotOperations.ts"; +import { ScanOperations, type ScanOperationsDependencies } from "./scanOperations.ts"; import { createCustomisationSyncPathOperations, type CustomisationSyncPathOperations, @@ -92,14 +58,6 @@ export { PluginDataExDisplayV2 } from "./customisationSyncModel.ts"; const UPDATED_CONFIGURATION_NOTICE_KEY = "config-sync:updated-configuration"; -const { - serialize, - deserialize, - dummyHead: DUMMY_HEAD, - dummyEnd: DUMMY_END, -} = createCustomisationSyncCodec({ digestHash, parseYaml }); -const CUSTOMISATION_SYNC_READ_CODEC = { deserialize, serialize }; - type CustomisationSyncSettings = Pick< ObsidianLiveSyncSettings, | "usePluginSync" @@ -164,20 +122,14 @@ export type CustomisationSyncContextDependencies = OptionalFileSyncFileTreeDepen export class CustomisationSyncContext implements CustomisationSyncDialogView { private readonly dependencies: CustomisationSyncContextDependencies; private readonly pathOperations: CustomisationSyncPathOperations; - private readonly catalogueState = new CustomisationSyncCatalogueState(); + private readonly snapshotPersistence: SnapshotPersistence; + private readonly snapshotOperations: SnapshotOperations; + private readonly catalogueOperations: CatalogueOperations; + private readonly applicationOperations: ApplicationOperations; + private readonly scanOperations: ScanOperations; private readonly recentProcessedInternalFiles = new CustomisationSyncRecentEventDeduplicator(); private serviceHandlersView: CustomisationSyncServiceHandlers | undefined; private testingView: CustomisationSyncTestingView | undefined; - private readonly scanProgress = reactiveSource(0); - private readonly pluginScanningChanged: Parameters["onChanged"]>[0] = (event) => { - this.enumerationActive.set(event.value != 0); - this.dependencies.publishScanCount(event.value); - }; - - readonly enumerationActive = writable(false); - readonly catalogue = this.catalogueState.catalogue; - readonly migrationProgress = this.catalogueState.migrationProgress; - readonly manifests = this.catalogueState.manifests; private readonly periodicPluginSweepProcessor: CustomisationSyncPeriodicProcessor; @@ -189,10 +141,104 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { getUsePluginEtc: () => dependencies.getSettings().usePluginEtc, getDeviceAndVaultName: () => dependencies.getDeviceAndVaultName(), }); + const snapshotPersistenceDependencies: SnapshotPersistenceDependencies = { + getLocalDatabase: () => dependencies.getLocalDatabase(), + storageAccess: dependencies.storageAccess, + path: { + ...this.pathOperations, + path2id: (filename, prefix) => dependencies.path.path2id(filename, prefix), + isMarkedAsSameChanges: (file, mtimes) => dependencies.path.isMarkedAsSameChanges(file, mtimes), + markChangesAreSame: (file, newMtime, oldMtime) => + dependencies.path.markChangesAreSame(file, newMtime, oldMtime), + }, + log: (message, level, key) => dependencies.log(message, level, key), + getConfigDir: () => dependencies.getConfigDir(), + }; + this.snapshotPersistence = new SnapshotPersistence(snapshotPersistenceDependencies); + this.catalogueOperations = new CatalogueOperations({ + getSettings: () => { + const settings = dependencies.getSettings(); + return { + usePluginSync: settings.usePluginSync, + usePluginSyncV2: settings.usePluginSyncV2, + }; + }, + getLocalDatabase: () => dependencies.getLocalDatabase(), + path: { + getPath: (entry) => dependencies.path.getPath(entry), + path2id: (filename, prefix) => dependencies.path.path2id(filename, prefix), + }, + log: (message, level, key) => dependencies.log(message, level, key), + snapshotPersistence: this.snapshotPersistence, + publishScanCount: (count) => dependencies.publishScanCount(count), + } satisfies CatalogueOperationsDependencies); + this.snapshotOperations = new SnapshotOperations({ + getSettings: () => ({ usePluginSyncV2: dependencies.getSettings().usePluginSyncV2 }), + getDeviceAndVaultName: () => dependencies.getDeviceAndVaultName(), + log: (message, level, key) => dependencies.log(message, level, key), + snapshotPersistence: this.snapshotPersistence, + catalogueOperations: this.catalogueOperations, + }); + const applicationOperationsDependencies: ApplicationOperationsDependencies = { + getLocalDatabase: () => ({ getDBEntry: (path) => dependencies.getLocalDatabase().getDBEntry(path) }), + storageAccess: dependencies.storageAccess, + path: { + filenameToUnifiedKey: (path, termOverride) => + this.pathOperations.filenameToUnifiedKey(path, termOverride), + }, + log: (message, level, key) => dependencies.log(message, level, key), + getConfigDir: () => dependencies.getConfigDir(), + getDeviceAndVaultName: () => dependencies.getDeviceAndVaultName(), + resolveJsonConflict: (path, files, remoteName, apply) => + dependencies.resolveJsonConflict(path, files, remoteName, apply), + selectTextFile: (path, diffResult, remoteName) => dependencies.selectTextFile(path, diffResult, remoteName), + reloadPlugin: (configDir, pluginName) => dependencies.reloadPlugin(configDir, pluginName), + askRestart: () => dependencies.askRestart(), + snapshotOperations: this.snapshotOperations, + catalogueOperations: this.catalogueOperations, + }; + this.applicationOperations = new ApplicationOperations(applicationOperationsDependencies); + this.scanOperations = new ScanOperations({ + listFiles: async (path) => await dependencies.listFiles(path), + getSettings: () => ({ usePluginSyncV2: dependencies.getSettings().usePluginSyncV2 }), + getLocalDatabase: () => dependencies.getLocalDatabase(), + path: { + getPath: (entry) => dependencies.path.getPath(entry), + isTargetPath: (path) => this.pathOperations.isTargetPath(path), + filenameToUnifiedKey: (path, termOverride) => + this.pathOperations.filenameToUnifiedKey(path, termOverride), + filenameWithUnifiedKey: (path, termOverride) => + this.pathOperations.filenameWithUnifiedKey(path, termOverride), + unifiedKeyPrefixOfTerminal: (termOverride) => + this.pathOperations.unifiedKeyPrefixOfTerminal(termOverride), + }, + log: (message, level, key) => dependencies.log(message, level, key), + getConfigDir: () => dependencies.getConfigDir(), + getDeviceAndVaultName: () => dependencies.getDeviceAndVaultName(), + ownsLocalFile: (path) => dependencies.ownsLocalFile(path), + ownsLocalDocument: (path) => dependencies.ownsLocalDocument(path), + snapshotOperations: this.snapshotOperations, + catalogueOperations: this.catalogueOperations, + } satisfies ScanOperationsDependencies); this.periodicPluginSweepProcessor = dependencies.createPeriodicProcessor( async () => await this.scanAllConfigFiles(false) ); - this.scanProgress.onChanged(this.pluginScanningChanged); + } + + get catalogue() { + return this.catalogueOperations.catalogue; + } + + get enumerationActive() { + return this.catalogueOperations.enumerationActive; + } + + get migrationProgress() { + return this.catalogueOperations.migrationProgress; + } + + get manifests() { + return this.catalogueOperations.manifests; } /** @@ -225,17 +271,18 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { if (!this.testingView) { this.testingView = Object.freeze({ configDir: this.configDir, - scanInternalFiles: async () => await this.scanInternalFiles(), + scanInternalFiles: async () => await this.scanOperations.scanInternalFiles(), scanAllConfigFiles: async (showMessage: boolean) => await this.scanAllConfigFiles(showMessage), storeCustomizationFiles: async (path: FilePath, termOverride?: string) => - await this.storeCustomizationFiles(path, termOverride), + await this.snapshotOperations.storeCustomizationFiles(path, termOverride), deleteConfigOnDatabase: async (path: FilePathWithPrefix, forceWrite?: boolean) => - await this.deleteConfigOnDatabase(path, forceWrite), - createPluginDataFromV2: (path: FilePathWithPrefix) => this.createPluginDataFromV2(path), + await this.snapshotOperations.deleteConfigOnDatabase(path, forceWrite), + createPluginDataFromV2: (path: FilePathWithPrefix) => + this.catalogueOperations.createPluginDataFromV2(path), createPluginDataExFileV2: async (path: FilePathWithPrefix, loaded?: LoadedEntry) => - await this.createPluginDataExFileV2(path, loaded), + await this.catalogueOperations.createPluginDataExFileV2(path, loaded), applyDataV2: async (data: PluginDataExDisplayV2, content?: string) => - await this.applyDataV2(data, content), + await this.applicationOperations.applyDataV2(data, content), }); } return this.testingView; @@ -249,18 +296,10 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { return this.dependencies.getSettings(); } - private get localDatabase() { - return this.dependencies.getLocalDatabase(); - } - private get storageAccess() { return this.dependencies.storageAccess; } - private async path2id(filename: FilePathWithPrefix | FilePath, prefix?: string) { - return await this.dependencies.path.path2id(filename, prefix); - } - private getPath(entry: AnyEntry): FilePathWithPrefix { return this.dependencies.path.getPath(entry); } @@ -277,9 +316,6 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { this.dependencies.log(message, level, key); } - private get useV2() { - return this.settings.usePluginSyncV2; - } private get useSyncPluginEtc() { return this.settings.usePluginEtc; } @@ -341,33 +377,17 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { dataB: IPluginDataExDisplay, filename: string ): Promise { - const dataACopy = - dataA instanceof PluginDataExDisplayV2 - ? new PluginDataExDisplayV2(dataA, this.catalogueState.manifestLookup) - : { ...dataA }; - const dataBCopy = - dataB instanceof PluginDataExDisplayV2 - ? new PluginDataExDisplayV2(dataB, this.catalogueState.manifestLookup) - : { ...dataB }; - dataACopy.files = dataACopy.files.filter((file) => file.filename == filename); - dataBCopy.files = dataBCopy.files.filter((file) => file.filename == filename); - return await this.compareUsingDisplayData(dataACopy, dataBCopy, true); + return await this.applicationOperations.compareFileUsingDisplayData(dataA, dataB, filename); } async duplicateData(data: IPluginDataExDisplay, deviceName: string): Promise { - const path = `${this.configDir}/${data.files[0].filename}` as FilePath; - await this.storeCustomizationFiles(path, deviceName); - await this.updatePluginList(false, this.pathOperations.filenameToUnifiedKey(path, deviceName)); + await this.applicationOperations.duplicateData(data, deviceName); } dispose() { cancelTask(UPDATED_CONFIGURATION_NOTICE_KEY); this.periodicPluginSweepProcessor?.disable(); - this.pluginScanProcessor?.terminate(); - this.pluginScanProcessorV2?.terminate(); - this.scanProgress.offChanged(this.pluginScanningChanged); - this.enumerationActive.set(false); - this.dependencies.publishScanCount(0); + this.catalogueOperations.dispose(); this.dependencies.hideConfigurationNotice(); } @@ -407,474 +427,19 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { return true; } async reloadPluginList(showMessage: boolean) { - this.catalogueState.clearForReload(); - await this.updatePluginList(showMessage); + await this.catalogueOperations.reloadPluginList(showMessage); } - private pluginScanProcessor = new QueueProcessor( - async (v: AnyEntry[]) => { - const plugin = v[0]; - if (this.useV2) { - await this.migrateV1ToV2(false, plugin); - return []; - } - const path = plugin.path || this.getPath(plugin); - const oldEntry = this.catalogueState.findPlugin(path); - if (oldEntry && oldEntry.mtime == plugin.mtime) return []; - try { - const pluginData = await loadCustomisationDisplayData( - this.dependencies, - path, - CUSTOMISATION_SYNC_READ_CODEC - ); - if (pluginData) { - this.catalogueState.replacePlugin(pluginData); - } - // Failed to load - return []; - } catch (ex) { - this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); - this._log(ex, LOG_LEVEL_VERBOSE); - } - return []; - }, - { - suspended: false, - batchSize: 1, - concurrentLimit: 10, - delay: 100, - yieldThreshold: 10, - maintainDelay: false, - totalRemainingReactiveSource: this.scanProgress, - } - ).startPipeline(); - - // Compatibility question: no production path currently enqueues work into - // this second processor. Preserve its construction and disposal until the - // intended V2 scan path, or its safe removal, has focused coverage. - private pluginScanProcessorV2 = new QueueProcessor( - async (v: AnyEntry[]) => { - const plugin = v[0]; - const path = plugin.path || this.getPath(plugin); - const oldEntry = this.catalogueState.findPlugin(path); - if (oldEntry && oldEntry.mtime == plugin.mtime) return []; - try { - const pluginData = await loadCustomisationDisplayData( - this.dependencies, - path, - CUSTOMISATION_SYNC_READ_CODEC - ); - if (pluginData) { - this.catalogueState.replacePlugin(pluginData); - } - // Failed to load - return []; - } catch (ex) { - this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); - this._log(ex, LOG_LEVEL_VERBOSE); - } - return []; - }, - { - suspended: false, - batchSize: 1, - concurrentLimit: 10, - delay: 100, - yieldThreshold: 10, - maintainDelay: false, - totalRemainingReactiveSource: this.scanProgress, - } - ).startPipeline(); - - private async createPluginDataExFileV2( - unifiedPathV2: FilePathWithPrefix, - loaded?: LoadedEntry - ): Promise { - // Compatibility: a caller-supplied entry bypasses the database lookup - // and the isLoadedEntry check performed by loadCustomisationV2Entry. - const loadedEntry = loaded ?? (await loadCustomisationV2Entry(this.dependencies, unifiedPathV2)); - if (!loadedEntry) return false; - const { confKey, file, isManifest } = decodeCustomisationSyncV2File(unifiedPathV2, loadedEntry, DUMMY_END); - if (isManifest) { - this.catalogueState.processManifest( - confKey, - file.mtime, - () => JSON.parse(file.data[0]) as PluginManifest, - (error) => { - this._log( - `The file ${loadedEntry.path} seems to manifest, but could not be decoded as JSON`, - LOG_LEVEL_VERBOSE - ); - this._log(error, LOG_LEVEL_VERBOSE); - } - ); - } - return file; - } - private createPluginDataFromV2(unifiedPathV2: FilePathWithPrefix) { - const { category, device, key, pathV1 } = parseCustomisationSyncV2DocumentPath(unifiedPathV2); - if (category == "") return; - - const ret: PluginDataExDisplayV2 = new PluginDataExDisplayV2( - { - documentPath: pathV1, - category: category, - name: key, - term: `${device}`, - files: [], - mtime: 0, - }, - this.catalogueState.manifestLookup - ); - return ret; - } - - private async updatePluginListV2(showMessage: boolean, unifiedFilenameWithKey: FilePathWithPrefix): Promise { - try { - this.catalogueState.beginUpdate(); - const { pathV1 } = parseCustomisationSyncV2DocumentPath(unifiedFilenameWithKey); - - const oldEntry = this.catalogueState.findPlugin(pathV1); - let entry: PluginDataExDisplayV2 | undefined = undefined; - - if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) { - const newEntry = this.createPluginDataFromV2(unifiedFilenameWithKey); - if (newEntry) { - entry = newEntry; - } - } else if (oldEntry instanceof PluginDataExDisplayV2) { - entry = oldEntry; - } - if (!entry) return; - const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey); - // Compatibility: the inherited update always re-adds an empty V2 - // row after deleting its final file. - await this.catalogueState.updateV2Plugin(entry, file, unifiedFilenameWithKey); - - scheduleTask("updatePluginListV2", 100, () => { - this.catalogueState.publishCatalogue(); - }); - } finally { - this.catalogueState.endUpdate(); - } - } - - private async migrateV1ToV2(showMessage: boolean, entry: AnyEntry): Promise { - const v1Path = entry.path; - this._log(`Migrating ${entry.path} to V2`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO); - if (entry.deleted) { - this._log(`The entry ${v1Path} is already deleted`, LOG_LEVEL_VERBOSE); - return; - } - if (!v1Path.endsWith(".md") && !v1Path.startsWith(ICXHeader)) { - this._log(`The entry ${v1Path} is not a customisation sync binder`, LOG_LEVEL_VERBOSE); - return; - } - if (v1Path.indexOf("%") !== -1) { - this._log(`The entry ${v1Path} is already migrated`, LOG_LEVEL_VERBOSE); - return; - } - const loadedEntry = await this.localDatabase.getDBEntry(v1Path); - if (!loadedEntry) { - this._log(`The entry ${v1Path} is not found`, LOG_LEVEL_VERBOSE); - return; - } - - const pluginData = deserialize(getDocDataAsArray(loadedEntry.data), {}) as PluginDataEx; - const prefixPath = v1Path.slice(0, -".md".length) + "%"; - const category = pluginData.category; - - for (const f of pluginData.files) { - const stripTable: Record = { - CONFIG: 0, - THEME: 2, - SNIPPET: 1, - PLUGIN_MAIN: 2, - PLUGIN_DATA: 2, - PLUGIN_ETC: 2, - }; - const deletePrefixCount = stripTable?.[category] ?? 1; - const relativeFilename = f.filename.split("/").slice(deletePrefixCount).join("/"); - const v2Path = (prefixPath + relativeFilename) as FilePathWithPrefix; - // console.warn(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`); - this._log(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`, LOG_LEVEL_VERBOSE); - const newId = await this.path2id(v2Path); - // const buf = - - const data = createBlob([DUMMY_HEAD, DUMMY_END, ...getDocDataAsArray(f.data)]); - - const saving: SavingEntry = { - ...loadedEntry, - _rev: undefined, - _id: newId, - path: v2Path, - data: data, - datatype: "plain", - type: "plain", - children: [], - eden: {}, - }; - const r = await this.localDatabase.putDBEntry(saving); - if (r && r.ok) { - this._log(`Migrated ${v1Path} / ${f.filename} to ${v2Path}`, LOG_LEVEL_INFO); - const delR = await this.deleteConfigOnDatabase(v1Path); - if (delR) { - this._log(`Deleted ${v1Path} successfully`, LOG_LEVEL_INFO); - } else { - this._log(`Failed to delete ${v1Path}`, LOG_LEVEL_NOTICE); - } - } - } - } - async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise { - if (!this.isThisModuleEnabled()) { - this.pluginScanProcessor.clearQueue(); - this.catalogueState.clearForDisabledRefresh(); - return; - } - try { - this.catalogueState.beginUpdate(); - const updatedDocumentId = updatedDocumentPath ? await this.path2id(updatedDocumentPath) : ""; - const plugins = updatedDocumentPath - ? this.localDatabase.findEntries(updatedDocumentId, updatedDocumentId + "\u{10ffff}", { - include_docs: true, - key: updatedDocumentId, - limit: 1, - }) - : this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true }); - for await (const v of plugins) { - if (v.deleted || v._deleted) continue; - if (v.path.indexOf("%") !== -1) { - fireAndForget(() => this.updatePluginListV2(showMessage, v.path)); - continue; - } - - const path = v.path || this.getPath(v); - if (updatedDocumentPath && updatedDocumentPath != path) continue; - this.pluginScanProcessor.enqueue(v); - } - } finally { - this.enumerationActive.set(false); - this.catalogueState.endUpdate(); - } - this.enumerationActive.set(false); - // return entries; + await this.catalogueOperations.updatePluginList(showMessage, updatedDocumentPath); } async compareUsingDisplayData(dataA: IPluginDataExDisplay, dataB: IPluginDataExDisplay, compareEach = false) { - const loadFile = async (data: IPluginDataExDisplay) => { - if (data instanceof PluginDataExDisplayV2 || compareEach) { - return data.files[0] as LoadedEntryPluginDataExFile; - } - const loadDoc = await this.localDatabase.getDBEntry(data.documentPath); - if (!loadDoc) return false; - const pluginData = deserialize(getDocDataAsArray(loadDoc.data), {}) as PluginDataEx; - pluginData.documentPath = data.documentPath; - const file = pluginData.files[0]; - const doc = { ...loadDoc, ...file, datatype: "newnote" } as LoadedEntryPluginDataExFile; - return doc; - }; - const fileA = await loadFile(dataA); - const fileB = await loadFile(dataB); - this._log(`Comparing: ${dataA.documentPath} <-> ${dataB.documentPath}`, LOG_LEVEL_VERBOSE); - if (!fileA || !fileB) { - this._log( - `Could not load ${dataA.name} for comparison: ${!fileA ? dataA.term : ""}${!fileB ? dataB.term : ""}`, - LOG_LEVEL_NOTICE - ); - return false; - } - let path = stripAllPrefixes(fileA.path.split("/").slice(-1).join("/") as FilePath); // TODO:adjust - if (path.indexOf("%") !== -1) { - path = path.split("%")[1] as FilePath; - } - if (fileA.path.endsWith(".json")) { - return serialized("config:merge-data", async () => { - this._log("Opening data-merging dialog", LOG_LEVEL_VERBOSE); - return await this.dependencies.resolveJsonConflict(path, [fileA, fileB], dataB.term, async (result) => { - try { - return await this.applyData(dataA, result); - } catch (ex) { - this._log("Could not apply merged file"); - this._log(ex, LOG_LEVEL_VERBOSE); - return false; - } - }); - }); - } else { - const dmp = new diff_match_patch(); - let docAData = getDocData(fileA.data); - let docBData = getDocData(fileB.data); - if (fileA?.datatype != "plain") { - docAData = base64ToString(docAData); - } - if (fileB?.datatype != "plain") { - docBData = base64ToString(docBData); - } - const diffMap = dmp.diff_linesToChars_(docAData, docBData); - - const diff = dmp.diff_main(diffMap.chars1, diffMap.chars2, false); - dmp.diff_charsToLines_(diff, diffMap.lineArray); - dmp.diff_cleanupSemantic(diff); - const diffResult: diff_result = { - left: { rev: "A", ...fileA, data: docAData }, - right: { rev: "B", ...fileB, data: docBData }, - diff: diff, - }; - const ret = await this.dependencies.selectTextFile(path, diffResult, dataB.term); - if (ret === false) return false; - const resultContent = ret == "A" ? docAData : ret == "B" ? docBData : undefined; - if (resultContent) { - return await this.applyData(dataA, resultContent); - } - return false; - } - } - private async applyDataV2(data: PluginDataExDisplayV2, content?: string): Promise { - const baseDir = this.configDir; - try { - if (content) { - // const dt = createBlob(content); - const filename = data.files[0].filename; - this._log(`Applying ${filename} of ${data.displayName || data.name}..`); - const path = `${baseDir}/${filename}` as FilePath; - await this.storageAccess.ensureDir(path); - // If the content has applied, modified time will be updated to the current time. - await this.storageAccess.writeHiddenFileAuto(path, content); - await this.storeCustomisationFileV2(path, this.dependencies.getDeviceAndVaultName()); - } else { - const files = data.files; - for (const f of files) { - // If files have applied, modified time will be updated to the current time. - const stat = { mtime: f.mtime, ctime: f.ctime }; - const path = `${baseDir}/${f.filename}` as FilePath; - this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`); - // const contentEach = createBlob(f.data); - await this.storageAccess.ensureDir(path); - - if (f.datatype == "newnote") { - let oldData; - try { - oldData = await this.storageAccess.readHiddenFileBinary(path); - } catch (ex) { - this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); - this._log(ex, LOG_LEVEL_VERBOSE); - oldData = new ArrayBuffer(0); - } - const content = base64ToArrayBuffer(f.data); - if (await isDocContentSame(oldData, content)) { - this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); - continue; - } - await this.storageAccess.writeHiddenFileAuto(path, content, stat); - } else { - let oldData; - try { - oldData = await this.storageAccess.readHiddenFileText(path); - } catch (ex) { - this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); - this._log(ex, LOG_LEVEL_VERBOSE); - oldData = ""; - } - const content = getDocData(f.data); - if (await isDocContentSame(oldData, content)) { - this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); - continue; - } - await this.storageAccess.writeHiddenFileAuto(path, content, stat); - } - this._log(`Applied ${f.filename} of ${data.displayName || data.name}..`); - await this.storeCustomisationFileV2(path, this.dependencies.getDeviceAndVaultName()); - } - } - } catch (ex) { - this._log(`Applying ${data.displayName || data.name}.. Failed`, LOG_LEVEL_NOTICE); - this._log(ex, LOG_LEVEL_VERBOSE); - return false; - } - return true; + return await this.applicationOperations.compareUsingDisplayData(dataA, dataB, compareEach); } async applyData(data: IPluginDataExDisplay, content?: string): Promise { - this._log(`Applying ${data.displayName || data.name}..`); - - if (data instanceof PluginDataExDisplayV2) { - return this.applyDataV2(data, content); - } - const baseDir = this.configDir; - try { - if (!data.documentPath) throw new LiveSyncError("InternalError: Document path not exist"); - const dx = await this.localDatabase.getDBEntry(data.documentPath); - if (dx == false) { - throw new LiveSyncError("Not found on database"); - } - const loadedData = deserialize(getDocDataAsArray(dx.data), {}) as PluginDataEx; - for (const f of loadedData.files) { - this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`); - try { - // console.dir(f); - const path = `${baseDir}/${f.filename}`; - await this.storageAccess.ensureDir(path); - if (!content) { - const dt = decodeBinary(f.data); - await this.storageAccess.writeHiddenFileAuto(path, dt); - } else { - await this.storageAccess.writeHiddenFileAuto(path, content); - } - this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Done`); - } catch (ex) { - this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Failed`); - this._log(ex, LOG_LEVEL_VERBOSE); - } - } - const uPath = `${baseDir}/${loadedData.files[0].filename}` as FilePath; - await this.storeCustomizationFiles(uPath); - await this.updatePluginList(true, uPath); - await delay(100); - this._log(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL_NOTICE); - if (data.category == "PLUGIN_DATA" || data.category == "PLUGIN_MAIN") { - await this.dependencies.reloadPlugin(baseDir, data.name); - } else if (data.category == "CONFIG") { - this.dependencies.askRestart(); - } - return true; - } catch (ex) { - this._log(`Applying ${data.displayName || data.name}.. Failed`); - this._log(ex, LOG_LEVEL_VERBOSE); - return false; - } + return await this.applicationOperations.applyData(data, content); } - async deleteData(data: PluginDataEx): Promise { - try { - if (data.documentPath) { - const delList = []; - if (this.useV2) { - const deleteList = this.catalogueState - .findPlugins(data.documentPath) - .filter((entry) => entry instanceof PluginDataExDisplayV2) - .map((entry) => entry.files) - .flat(); - for (const e of deleteList) { - delList.push(e.path); - } - } - delList.push(data.documentPath); - const p = delList.map(async (e) => { - await this.deleteConfigOnDatabase(e); - await this.updatePluginList(false, e); - }); - await Promise.allSettled(p); - // await this.deleteConfigOnDatabase(data.documentPath); - // await this.updatePluginList(false, data.documentPath); - this._log( - `Deleted: ${data.category}/${data.name} of ${data.category} (${delList.length} items)`, - LOG_LEVEL_NOTICE - ); - } - return true; - } catch (ex) { - this._log(`Failed to delete: ${data.documentPath}`, LOG_LEVEL_NOTICE); - this._log(ex, LOG_LEVEL_VERBOSE); - return false; - } + async deleteData(data: IPluginDataExDisplay): Promise { + return await this.applicationOperations.deleteData(data); } private async processVirtualDocument(docs: PouchDB.Core.ExistingDocument) { if (!docs._id.startsWith(ICXHeader)) return false; @@ -913,229 +478,6 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { return true; } - private async storeCustomisationFileV2(path: FilePath, term: string, force = false) { - const vf = this.pathOperations.filenameWithUnifiedKey(path, term); - return await serialized(`plugin-${vf}`, async () => { - const prefixedFileName = vf; - - const id = await this.path2id(prefixedFileName); - const stat = await this.storageAccess.statHidden(path); - if (!stat) { - return false; - } - const mtime = stat.mtime; - const content = await this.storageAccess.readHiddenFileBinary(path); - const contentBlob = createBlob([DUMMY_HEAD, DUMMY_END, ...(await arrayBufferToBase64(content))]); - // const contentBlob = createBlob(content); - try { - const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, false); - let saveData: SavingEntry; - if (old === false) { - saveData = { - _id: id, - path: prefixedFileName, - data: contentBlob, - mtime, - ctime: mtime, - datatype: "plain", - size: contentBlob.size, - children: [], - deleted: false, - type: "plain", - eden: {}, - }; - } else { - if ( - this.dependencies.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN - ) { - this._log( - `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, - LOG_LEVEL_DEBUG - ); - return; - } - const docXDoc = await this.localDatabase.getDBEntryFromMeta(old, false, false); - if (docXDoc == false) { - throw new LiveSyncError("Could not load the document"); - } - const dataSrc = getDocData(docXDoc.data); - const dataStart = dataSrc.indexOf(DUMMY_END); - const oldContent = dataSrc.substring(dataStart + DUMMY_END.length); - const oldContentArray = base64ToArrayBuffer(oldContent); - if (await isDocContentSame(oldContentArray, content)) { - this._log( - `STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, - LOG_LEVEL_VERBOSE - ); - this.dependencies.path.markChangesAreSame(prefixedFileName, old.mtime, mtime + 1); - return true; - } - saveData = { - ...old, - data: contentBlob, - mtime, - size: contentBlob.size, - datatype: "plain", - children: [], - deleted: false, - type: "plain", - }; - } - const ret = await this.localDatabase.putDBEntry(saveData); - this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); - fireAndForget(() => this.updatePluginListV2(false, this.pathOperations.filenameWithUnifiedKey(path))); - return ret; - } catch (ex) { - this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); - this._log(ex, LOG_LEVEL_VERBOSE); - return false; - } - }); - } - private async storeCustomizationFiles(path: FilePath, termOverRide?: string) { - const term = termOverRide || this.dependencies.getDeviceAndVaultName(); - if (term == "") { - this._log($msg("We have to configure the device name"), LOG_LEVEL_NOTICE); - return; - } - if (this.useV2) { - return await this.storeCustomisationFileV2(path, term); - } - const vf = this.pathOperations.filenameToUnifiedKey(path, term); - // console.warn(`Storing ${path} to ${bareVF} :--> ${keyedVF}`); - - return await serialized(`plugin-${vf}`, async () => { - const category = this.pathOperations.getFileCategory(path); - let mtime = 0; - let fileTargets = [] as FilePath[]; - // let savePath = ""; - const name = - category == "CONFIG" || category == "SNIPPET" - ? path.split("/").reverse()[0] - : path.split("/").reverse()[1]; - const parentPath = path.split("/").slice(0, -1).join("/"); - const prefixedFileName = this.pathOperations.filenameToUnifiedKey(path, term); - const id = await this.path2id(prefixedFileName); - const dt: PluginDataEx = { - category: category, - files: [], - name: name, - mtime: 0, - term: term, - }; - // let scheduleKey = ""; - if ( - category == "CONFIG" || - category == "SNIPPET" || - category == "PLUGIN_ETC" || - category == "PLUGIN_DATA" - ) { - fileTargets = [path]; - if (category == "PLUGIN_ETC") { - dt.displayName = path.split("/").slice(-1).join("/"); - } - } else if (category == "PLUGIN_MAIN") { - fileTargets = ["manifest.json", "main.js", "styles.css"].map((e) => `${parentPath}/${e}` as FilePath); - } else if (category == "THEME") { - fileTargets = ["manifest.json", "theme.css"].map((e) => `${parentPath}/${e}` as FilePath); - } - for (const target of fileTargets) { - const data = await readCustomisationFile(this.dependencies, target, this.configDir); - if (data == false) { - this._log(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE); - continue; - } - if (data.version) { - dt.version = data.version; - } - if (data.displayName) { - dt.displayName = data.displayName; - } - // Use average for total modified time. - mtime = mtime == 0 ? data.mtime : (data.mtime + mtime) / 2; - dt.files.push(data); - } - dt.mtime = mtime; - - // this._log(`Configuration saving: ${prefixedFileName}`); - if (dt.files.length == 0) { - this._log(`Nothing left: deleting.. ${path}`); - await this.deleteConfigOnDatabase(prefixedFileName); - await this.updatePluginList(false, prefixedFileName); - return; - } - - const content = createTextBlob(serialize(dt)); - try { - const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, false); - let saveData: SavingEntry; - if (old === false) { - saveData = { - _id: id, - path: prefixedFileName, - data: content, - mtime, - ctime: mtime, - datatype: "newnote", - size: content.size, - children: [], - deleted: false, - type: "newnote", - eden: {}, - }; - } else { - if (old.mtime == mtime) { - // this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE); - return true; - } - const oldC = await this.localDatabase.getDBEntryFromMeta(old, false, false); - if (oldC) { - const d = deserialize(getDocDataAsArray(oldC.data), {}) as PluginDataEx; - if (d.files.length == dt.files.length) { - const diffs = d.files - .map((previous) => ({ - prev: previous, - curr: dt.files.find((e) => e.filename == previous.filename), - })) - .map(async (e) => { - try { - return await isDocContentSame(e.curr?.data ?? [], e.prev.data); - } catch { - return false; - } - }); - const isSame = (await Promise.all(diffs)).every((e) => e == true); - if (isSame) { - this._log( - `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, - LOG_LEVEL_VERBOSE - ); - return true; - } - } - } - saveData = { - ...old, - data: content, - mtime, - size: content.size, - datatype: "newnote", - children: [], - deleted: false, - type: "newnote", - }; - } - const ret = await this.localDatabase.putDBEntry(saveData); - await this.updatePluginList(false, saveData.path); - this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); - return ret; - } catch (ex) { - this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); - this._log(ex, LOG_LEVEL_VERBOSE); - return false; - } - }); - } private async processOptionalFileEvent(path: FilePath): Promise { return await this.watchVaultRawEventsAsync(path); } @@ -1161,174 +503,15 @@ export class CustomisationSyncContext implements CustomisationSyncDialogView { // To prevent saving half-collected file sets. const keySchedule = this.pathOperations.filenameToUnifiedKey(path); scheduleTask(keySchedule, 100, async () => { - await this.storeCustomizationFiles(path); + await this.snapshotOperations.storeCustomizationFiles(path); }); // Okay, it may handled after 100ms. // This was my own job. return true; } - async scanAllConfigFiles(showMessage: boolean) { - await shareRunningResult("scanAllConfigFiles", async () => { - const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; - this._log("Scanning customizing files.", logLevel, "scan-all-config"); - const term = this.dependencies.getDeviceAndVaultName(); - if (term == "") { - this._log($msg("We have to configure the device name"), LOG_LEVEL_NOTICE); - return; - } - const filesAll = await this.scanInternalFiles(); - if (this.useV2) { - const filesAllUnified = filesAll - .filter((e) => this.pathOperations.isTargetPath(e)) - .map( - (e) => - [this.pathOperations.filenameWithUnifiedKey(e, term), e] as [ - FilePathWithPrefix, - FilePath, - ] - ); - const localFileMap = new Map(filesAllUnified.map((e) => [e[0], e[1]])); - const prefix = this.pathOperations.unifiedKeyPrefixOfTerminal(term); - const entries = this.localDatabase.findEntries(prefix + "", `${prefix}\u{10ffff}`, { - include_docs: true, - }); - const tasks = [] as (() => Promise)[]; - const concurrency = 10; - const semaphore = Semaphore(concurrency); - for await (const item of entries) { - if (item.path.indexOf("%") !== -1) { - continue; - } - tasks.push(async () => { - const releaser = await semaphore.acquire(); - try { - const unifiedFilenameWithKey = `${item._id}` as FilePathWithPrefix; - const localPath = localFileMap.get(unifiedFilenameWithKey); - if (localPath) { - if (this.dependencies.ownsLocalFile(localPath)) { - await this.storeCustomisationFileV2(localPath, term); - } - localFileMap.delete(unifiedFilenameWithKey); - } else if (this.dependencies.ownsLocalDocument(this.getPath(item))) { - await this.deleteConfigOnDatabase(unifiedFilenameWithKey); - } - } catch (ex) { - this._log(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE); - this._log(ex, LOG_LEVEL_VERBOSE); - } finally { - releaser(); - } - }); - } - await Promise.all(tasks.map((e) => e())); - // Extra files - const taskExtra = [] as (() => Promise)[]; - for (const [, filePath] of localFileMap) { - if (!this.dependencies.ownsLocalFile(filePath)) continue; - taskExtra.push(async () => { - const releaser = await semaphore.acquire(); - try { - await this.storeCustomisationFileV2(filePath, term); - } catch (ex) { - this._log(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE); - this._log(ex, LOG_LEVEL_VERBOSE); - } finally { - releaser(); - } - }); - } - await Promise.all(taskExtra.map((e) => e())); - fireAndForget(() => this.updatePluginList(false)); - } else { - const files = filesAll - .filter((e) => this.pathOperations.isTargetPath(e)) - .map((e) => ({ key: this.pathOperations.filenameToUnifiedKey(e), file: e })); - const virtualPathsOfLocalFiles = [...new Set(files.map((e) => e.key))]; - const filesOnDB = ( - ( - await this.localDatabase.allDocsRaw({ - startkey: ICXHeader + "", - endkey: `${ICXHeader}\u{10ffff}`, - include_docs: true, - }) - ).rows.map((e) => e.doc) as InternalFileEntry[] - ).filter((e) => !e.deleted); - let deleteCandidate = filesOnDB - .map((e) => this.getPath(e)) - .filter((e) => e.startsWith(`${ICXHeader}${term}/`)); - for (const vp of virtualPathsOfLocalFiles) { - const p = files.find((e) => e.key == vp)?.file; - if (!p) { - this._log(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE); - continue; - } - if (this.dependencies.ownsLocalFile(p)) { - await this.storeCustomizationFiles(p); - } - deleteCandidate = deleteCandidate.filter((e) => e != vp); - } - for (const vp of deleteCandidate) { - if (this.dependencies.ownsLocalDocument(vp)) { - await this.deleteConfigOnDatabase(vp); - } - } - fireAndForget(() => this.updatePluginList(false)); - } - }); - } - - private async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false): Promise { - // const id = await this.path2id(prefixedFileName); - const mtime = new Date().getTime(); - return await serialized("file-x-" + prefixedFileName, async () => { - try { - const old = (await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, false)) as - | InternalFileEntry - | false; - let saveData: InternalFileEntry; - if (old === false) { - this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted (Not found on database)`); - return true; - } else { - if (old.deleted) { - this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted`); - return true; - } - saveData = { - ...old, - mtime, - size: 0, - children: [], - deleted: true, - type: "newnote", - }; - } - await this.localDatabase.putRaw(saveData); - await this.updatePluginList(false, prefixedFileName); - this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Done`); - return true; - } catch (ex) { - this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`); - this._log(ex, LOG_LEVEL_VERBOSE); - return false; - } - }); - } - - private async scanInternalFiles(): Promise { - const filenames = ( - await collectOptionalFileSyncFiles(this.dependencies, this.configDir, { - maxDepth: 2, - onError: (path, error) => { - this._log(`Could not traverse(CustomisationSync):${path}`, LOG_LEVEL_INFO); - this._log(error, LOG_LEVEL_VERBOSE); - }, - }) - ) - .filter((e) => e.startsWith(".")) - .filter((e) => !e.startsWith(".trash")); - return filenames as FilePath[]; + async scanAllConfigFiles(showMessage: boolean): Promise { + await this.scanOperations.scanAllConfigFiles(showMessage); } private suspendExtraSync(): Promise { diff --git a/src/features/ConfigSync/customisationSyncContext.v2Manifest.unit.spec.ts b/src/features/ConfigSync/customisationSyncContext.v2Manifest.unit.spec.ts index 4e835735..2d9b8bbd 100644 --- a/src/features/ConfigSync/customisationSyncContext.v2Manifest.unit.spec.ts +++ b/src/features/ConfigSync/customisationSyncContext.v2Manifest.unit.spec.ts @@ -26,6 +26,16 @@ vi.mock("@/common/translation", () => ({ vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ getObsidianCommunityPluginManager: vi.fn(), })); +vi.mock("octagonal-wheels/concurrency/processor", () => ({ + QueueProcessor: class QueueProcessor { + clearQueue = vi.fn(); + enqueue = vi.fn(); + terminate = vi.fn(); + startPipeline() { + return this; + } + }, +})); import { LOG_LEVEL_VERBOSE, @@ -34,7 +44,7 @@ import { } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { digestHash } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/hash"; import { createCustomisationSyncCodec } from "./customisationSyncCodec.ts"; -import { CustomisationSyncCatalogueState } from "./customisationSyncCatalogueState.ts"; +import { CatalogueOperations } from "./catalogueOperations.ts"; import { CustomisationSyncContext } from "./customisationSyncContext.ts"; import { createCustomisationSyncTestDependencies } from "./customisationSyncContext.unit.fixture.ts"; @@ -64,22 +74,31 @@ function loadedManifest(manifestSource: string, mtime: number): LoadedEntry { function createContext() { const log = vi.fn(); - const catalogueState = new CustomisationSyncCatalogueState(); - const pluginManifests = catalogueState.manifestLookup; - const loadedManifest_mTime = catalogueState.loadedManifestMTime; - const setManifests = vi.spyOn(catalogueState.manifests, "set"); - const setCatalogue = vi.spyOn(catalogueState.catalogue, "set"); + const snapshotPersistence = { + deleteConfigOnDatabase: vi.fn(async () => ({ value: true, status: "missing" as const, refreshes: [] })), + }; + const catalogueOperations = new CatalogueOperations({ + ...createCustomisationSyncTestDependencies({ + log, + getLocalDatabase: () => ({ getDBEntry: async () => false }) as never, + }), + snapshotPersistence, + publishScanCount: vi.fn(), + }); + const pluginManifests = catalogueOperations.manifestLookup; + const setManifests = vi.spyOn(catalogueOperations.manifests, "set"); + const setCatalogue = vi.spyOn(catalogueOperations.catalogue, "set"); const context = Object.create(CustomisationSyncContext.prototype) as CustomisationSyncContext; Object.assign(context, { dependencies: createCustomisationSyncTestDependencies({ log, getLocalDatabase: () => ({ getDBEntry: async () => false }) as never, }), - catalogueState, + catalogueOperations, }); return { + catalogueOperations, context, - loadedManifest_mTime, log, pluginManifests, setCatalogue, @@ -89,7 +108,7 @@ function createContext() { describe("compatibility: Customisation Sync V2 manifest state", () => { it("keeps the first parsed manifest when a later file has a different mtime", async () => { - const { context, loadedManifest_mTime, pluginManifests, setManifests } = createContext(); + const { catalogueOperations, context, pluginManifests, setManifests } = createContext(); await context.testing.createPluginDataExFileV2( path, @@ -101,12 +120,12 @@ describe("compatibility: Customisation Sync V2 manifest state", () => { ); expect(pluginManifests.get(confKey)).toMatchObject({ name: "First", version: "1.0.0" }); - expect(loadedManifest_mTime.get(confKey)).toBe(20); expect(setManifests).toHaveBeenCalledOnce(); + catalogueOperations.dispose(); }); it("records a failed manifest mtime and does not retry the same revision", async () => { - const { context, loadedManifest_mTime, log, pluginManifests, setCatalogue } = createContext(); + const { catalogueOperations, context, log, pluginManifests, setCatalogue } = createContext(); const invalid = loadedManifest("{invalid", 20); await expect(context.testing.createPluginDataExFileV2(path, invalid)).resolves.toMatchObject({ @@ -115,7 +134,6 @@ describe("compatibility: Customisation Sync V2 manifest state", () => { await context.testing.createPluginDataExFileV2(path, invalid); expect(pluginManifests.has(confKey)).toBe(false); - expect(loadedManifest_mTime.get(confKey)).toBe(20); expect(log).toHaveBeenCalledTimes(2); expect(log).toHaveBeenNthCalledWith( 1, @@ -125,5 +143,6 @@ describe("compatibility: Customisation Sync V2 manifest state", () => { ); expect(log).toHaveBeenNthCalledWith(2, expect.any(SyntaxError), LOG_LEVEL_VERBOSE, undefined); expect(setCatalogue).toHaveBeenCalledOnce(); + catalogueOperations.dispose(); }); }); diff --git a/src/features/ConfigSync/customisationSyncContext.view.unit.spec.ts b/src/features/ConfigSync/customisationSyncContext.view.unit.spec.ts index 861763bd..a139db1f 100644 --- a/src/features/ConfigSync/customisationSyncContext.view.unit.spec.ts +++ b/src/features/ConfigSync/customisationSyncContext.view.unit.spec.ts @@ -121,31 +121,18 @@ describe("CustomisationSyncContext dialogue view", () => { expect(askString).toHaveBeenCalledWith("Duplicate", "device name", ""); }); - it("keeps file-level comparison clones and duplication inside the view boundary", async () => { + it("delegates file-level comparison and duplication to the application owner", async () => { const { configSync } = createConfigSync(); - const compareUsingDisplayData = vi.fn< - (dataA: IPluginDataExDisplay, dataB: IPluginDataExDisplay, compareEach?: boolean) => Promise - >(async () => true); - const storeCustomizationFiles = vi.fn(async () => true); - const updatePluginList = vi.fn(async () => undefined); + const compareFileUsingDisplayData = vi.fn(async () => true); + const duplicateData = vi.fn(async () => undefined); Object.assign(configSync, { - compareUsingDisplayData, - pathOperations: { - filenameToUnifiedKey: vi.fn(() => "ix:device-b/PLUGIN_DATA/example.md"), - }, - storeCustomizationFiles, - updatePluginList, + applicationOperations: { compareFileUsingDisplayData, duplicateData }, }); await expect(configSync.compareFileUsingDisplayData(display, display, "data.json")).resolves.toBe(true); - const [left, right, compareEach] = compareUsingDisplayData.mock.calls[0]; - expect(left.files.map((file) => file.filename)).toEqual(["data.json"]); - expect(right.files.map((file) => file.filename)).toEqual(["data.json"]); - expect(compareEach).toBe(true); - expect(display.files).toHaveLength(2); + expect(compareFileUsingDisplayData).toHaveBeenCalledWith(display, display, "data.json"); await configSync.duplicateData(display, "device-b"); - expect(storeCustomizationFiles).toHaveBeenCalledWith(".obsidian/data.json", "device-b"); - expect(updatePluginList).toHaveBeenCalledWith(false, "ix:device-b/PLUGIN_DATA/example.md"); + expect(duplicateData).toHaveBeenCalledWith(display, "device-b"); }); }); diff --git a/src/features/ConfigSync/scanOperations.ts b/src/features/ConfigSync/scanOperations.ts new file mode 100644 index 00000000..7124e843 --- /dev/null +++ b/src/features/ConfigSync/scanOperations.ts @@ -0,0 +1,197 @@ +import type { + AnyEntry, + FilePath, + FilePathWithPrefix, + InternalFileEntry, + LOG_LEVEL, + ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { fireAndForget } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; +import { shareRunningResult } from "octagonal-wheels/concurrency/lock"; +import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; + +import { $msg } from "@/common/translation"; +import { ICXHeader } from "@/common/types.ts"; +import { + collectOptionalFileSyncFiles, + type OptionalFileSyncFileTreeDependencies, +} from "@/features/optionalFileSyncFileTree.ts"; +import type { CatalogueOperations } from "./catalogueOperations.ts"; +import type { CustomisationSyncPathOperations } from "./customisationSyncPathOperations.ts"; +import type { SnapshotOperations } from "./snapshotOperations.ts"; + +type ScanSettings = Pick; +type ScanDatabase = Pick; +type ScanPathOperations = Pick< + CustomisationSyncPathOperations, + "isTargetPath" | "filenameToUnifiedKey" | "filenameWithUnifiedKey" | "unifiedKeyPrefixOfTerminal" +> & { + getPath(entry: AnyEntry): FilePathWithPrefix; +}; +type ScanSnapshotOperations = Pick< + SnapshotOperations, + "storeCustomisationFileV2" | "storeCustomizationFiles" | "deleteConfigOnDatabase" +>; +type ScanCatalogueOperations = Pick; + +export type ScanOperationsDependencies = OptionalFileSyncFileTreeDependencies & { + getSettings(): ScanSettings; + getLocalDatabase(): ScanDatabase; + path: ScanPathOperations; + log: LogFunction; + getConfigDir(): string; + getDeviceAndVaultName(): string; + ownsLocalFile(path: FilePath): boolean; + ownsLocalDocument(path: FilePathWithPrefix): boolean; + snapshotOperations: ScanSnapshotOperations; + catalogueOperations: ScanCatalogueOperations; +}; + +/** + * Owns Customisation Sync file enumeration and reconciliation with the local + * database. Snapshot writes and catalogue publication remain explicit ports so + * scans do not depend on the context or its lifecycle. + */ +export class ScanOperations { + constructor(private readonly dependencies: ScanOperationsDependencies) {} + + private get localDatabase() { + return this.dependencies.getLocalDatabase(); + } + + private getPath(entry: AnyEntry): FilePathWithPrefix { + return this.dependencies.path.getPath(entry); + } + + private _log(message: unknown, level?: LOG_LEVEL, key?: string) { + this.dependencies.log(message, level, key); + } + + async scanInternalFiles(): Promise { + const filenames = ( + await collectOptionalFileSyncFiles(this.dependencies, this.dependencies.getConfigDir(), { + maxDepth: 2, + onError: (path, error) => { + this._log(`Could not traverse(CustomisationSync):${path}`, LOG_LEVEL_INFO); + this._log(error, LOG_LEVEL_VERBOSE); + }, + }) + ) + .filter((e) => e.startsWith(".")) + .filter((e) => !e.startsWith(".trash")); + return filenames as FilePath[]; + } + + async scanAllConfigFiles(showMessage: boolean): Promise { + await shareRunningResult("scanAllConfigFiles", async () => { + const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; + this._log("Scanning customizing files.", logLevel, "scan-all-config"); + const term = this.dependencies.getDeviceAndVaultName(); + if (term == "") { + this._log($msg("We have to configure the device name"), LOG_LEVEL_NOTICE); + return; + } + const filesAll = await this.scanInternalFiles(); + if (this.dependencies.getSettings().usePluginSyncV2) { + await this.scanV2ConfigFiles(filesAll, term); + } else { + await this.scanV1ConfigFiles(filesAll, term); + } + }); + } + + private async scanV2ConfigFiles(filesAll: readonly FilePath[], term: string): Promise { + const filesAllUnified = filesAll + .filter((e) => this.dependencies.path.isTargetPath(e)) + .map((e) => [this.dependencies.path.filenameWithUnifiedKey(e, term), e] as [FilePathWithPrefix, FilePath]); + const localFileMap = new Map(filesAllUnified.map((e) => [e[0], e[1]])); + const prefix = this.dependencies.path.unifiedKeyPrefixOfTerminal(term); + const entries = this.localDatabase.findEntries(prefix + "", `${prefix}\u{10ffff}`, { + include_docs: true, + }); + const tasks = [] as (() => Promise)[]; + const concurrency = 10; + const semaphore = Semaphore(concurrency); + for await (const item of entries) { + if (item.path.indexOf("%") !== -1) { + continue; + } + tasks.push(async () => { + const releaser = await semaphore.acquire(); + try { + const unifiedFilenameWithKey = `${item._id}` as FilePathWithPrefix; + const localPath = localFileMap.get(unifiedFilenameWithKey); + if (localPath) { + if (this.dependencies.ownsLocalFile(localPath)) { + await this.dependencies.snapshotOperations.storeCustomisationFileV2(localPath, term); + } + localFileMap.delete(unifiedFilenameWithKey); + } else if (this.dependencies.ownsLocalDocument(this.getPath(item))) { + await this.dependencies.snapshotOperations.deleteConfigOnDatabase(unifiedFilenameWithKey); + } + } catch (ex) { + this._log(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); + } finally { + releaser(); + } + }); + } + await Promise.all(tasks.map((e) => e())); + // Extra files + const taskExtra = [] as (() => Promise)[]; + for (const [, filePath] of localFileMap) { + if (!this.dependencies.ownsLocalFile(filePath)) continue; + taskExtra.push(async () => { + const releaser = await semaphore.acquire(); + try { + await this.dependencies.snapshotOperations.storeCustomisationFileV2(filePath, term); + } catch (ex) { + this._log(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); + } finally { + releaser(); + } + }); + } + await Promise.all(taskExtra.map((e) => e())); + fireAndForget(() => this.dependencies.catalogueOperations.updatePluginList(false)); + } + + private async scanV1ConfigFiles(filesAll: readonly FilePath[], term: string): Promise { + const files = filesAll + .filter((e) => this.dependencies.path.isTargetPath(e)) + .map((e) => ({ key: this.dependencies.path.filenameToUnifiedKey(e), file: e })); + const virtualPathsOfLocalFiles = [...new Set(files.map((e) => e.key))]; + const filesOnDB = ( + ( + await this.localDatabase.allDocsRaw({ + startkey: ICXHeader + "", + endkey: `${ICXHeader}\u{10ffff}`, + include_docs: true, + }) + ).rows.map((e) => e.doc) as InternalFileEntry[] + ).filter((e) => !e.deleted); + let deleteCandidate = filesOnDB.map((e) => this.getPath(e)).filter((e) => e.startsWith(`${ICXHeader}${term}/`)); + for (const vp of virtualPathsOfLocalFiles) { + const p = files.find((e) => e.key == vp)?.file; + if (!p) { + this._log(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE); + continue; + } + if (this.dependencies.ownsLocalFile(p)) { + await this.dependencies.snapshotOperations.storeCustomizationFiles(p); + } + deleteCandidate = deleteCandidate.filter((e) => e != vp); + } + for (const vp of deleteCandidate) { + if (this.dependencies.ownsLocalDocument(vp)) { + await this.dependencies.snapshotOperations.deleteConfigOnDatabase(vp); + } + } + fireAndForget(() => this.dependencies.catalogueOperations.updatePluginList(false)); + } +} diff --git a/src/features/ConfigSync/scanOperations.unit.spec.ts b/src/features/ConfigSync/scanOperations.unit.spec.ts new file mode 100644 index 00000000..fc5fef19 --- /dev/null +++ b/src/features/ConfigSync/scanOperations.unit.spec.ts @@ -0,0 +1,327 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + LOG_LEVEL_INFO, + LOG_LEVEL_NOTICE, + LOG_LEVEL_VERBOSE, + type FilePath, + type FilePathWithPrefix, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; + +const asyncHarness = vi.hoisted(() => ({ + fireAndForget: vi.fn((operation: () => unknown) => { + void operation(); + }), +})); + +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/utils", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fireAndForget: asyncHarness.fireAndForget, + }; +}); + +import { ScanOperations, type ScanOperationsDependencies } from "./scanOperations.ts"; + +type ScanEntry = { + _id: FilePathWithPrefix; + path: FilePathWithPrefix; + deleted?: boolean; +}; + +type FixtureOptions = { + useV2?: boolean; + usePluginSync?: boolean; + term?: string; + files?: FilePath[]; + targetFiles?: FilePath[]; + databaseEntries?: ScanEntry[]; + v1Paths?: Record; + v2Paths?: Record; + ownsLocalFile?: (path: FilePath) => boolean; + ownsLocalDocument?: (path: FilePathWithPrefix) => boolean; + listFiles?: (path: string) => Promise<{ files: readonly string[]; folders: readonly string[] }>; +}; + +function asyncEntries(entries: readonly T[]) { + return { + async *[Symbol.asyncIterator]() { + yield* entries; + }, + }; +} + +function createFixture(options: FixtureOptions = {}) { + const files = options.files ?? []; + const term = options.term ?? "device-a"; + const databaseEntries = options.databaseEntries ?? []; + const v1Paths = options.v1Paths ?? {}; + const v2Paths = options.v2Paths ?? {}; + const log = vi.fn(); + const allDocsRaw = vi.fn(async () => ({ + rows: databaseEntries.map((doc) => ({ id: doc._id, doc })), + })); + const findEntries = vi.fn(() => asyncEntries(databaseEntries)); + const storeCustomisationFileV2 = vi.fn(async (_path: FilePath, _term: string) => true); + const storeCustomizationFiles = vi.fn(async (_path: FilePath) => true); + const deleteConfigOnDatabase = vi.fn(async (_path: FilePathWithPrefix) => true); + const updatePluginList = vi.fn(async (_showMessage: boolean) => undefined); + const listFiles = vi.fn( + options.listFiles ?? + (async () => ({ files, folders: [] }) as { files: readonly string[]; folders: readonly string[] }) + ); + const dependencies = { + listFiles, + getSettings: () => ({ usePluginSyncV2: options.useV2 ?? false, usePluginSync: options.usePluginSync ?? true }), + getLocalDatabase: () => ({ allDocsRaw, findEntries }), + path: { + isTargetPath: (path: string) => options.targetFiles?.includes(path as FilePath) ?? true, + filenameToUnifiedKey: (path: string) => + v1Paths[path] ?? (`ix:${term}/CONFIG/${path.split("/").pop()}.md` as FilePathWithPrefix), + filenameWithUnifiedKey: (path: string) => + v2Paths[path] ?? + (`ix:${term}/CONFIG/${path.split("/").pop()}%${path.split("/").pop()}` as FilePathWithPrefix), + unifiedKeyPrefixOfTerminal: (termOverride?: string) => `ix:${termOverride ?? term}/` as FilePathWithPrefix, + getPath: (entry: ScanEntry) => entry.path, + }, + log, + getConfigDir: () => ".obsidian", + getDeviceAndVaultName: () => term, + ownsLocalFile: options.ownsLocalFile ?? (() => true), + ownsLocalDocument: options.ownsLocalDocument ?? (() => true), + snapshotOperations: { + storeCustomisationFileV2, + storeCustomizationFiles, + deleteConfigOnDatabase, + }, + catalogueOperations: { updatePluginList }, + } as unknown as ScanOperationsDependencies; + return { + operations: new ScanOperations(dependencies), + dependencies, + listFiles, + log, + allDocsRaw, + findEntries, + snapshot: { storeCustomisationFileV2, storeCustomizationFiles, deleteConfigOnDatabase }, + catalogue: { updatePluginList }, + }; +} + +describe("ScanOperations", () => { + beforeEach(() => { + asyncHarness.fireAndForget.mockClear(); + }); + + it("filters the bounded file tree and logs traversal errors", async () => { + const traversalError = new Error("cannot read folder"); + const listFiles = vi.fn(async (path: string) => { + switch (path) { + case ".obsidian": + return { + files: [".obsidian/app.json", "settings.json", ".trash/root.json"], + folders: [".obsidian/plugins", ".trash", ".obsidian/unreadable"], + }; + case ".obsidian/plugins": + return { + files: [".obsidian/plugins/example/data.json"], + folders: [".obsidian/plugins/example"], + }; + case ".obsidian/plugins/example": + return { + files: [".obsidian/plugins/example/manifest.json"], + folders: [".obsidian/plugins/example/deeper"], + }; + case ".trash": + return { files: [".trash/ignored.json"], folders: [] }; + case ".obsidian/unreadable": + throw traversalError; + default: + throw new Error(`unexpected traversal: ${path}`); + } + }); + const fixture = createFixture({ listFiles }); + + await expect(fixture.operations.scanInternalFiles()).resolves.toEqual([ + ".obsidian/app.json", + ".obsidian/plugins/example/data.json", + ".obsidian/plugins/example/manifest.json", + ]); + + expect(listFiles).not.toHaveBeenCalledWith(".obsidian/plugins/example/deeper"); + expect(fixture.log).toHaveBeenCalledWith( + "Could not traverse(CustomisationSync):.obsidian/unreadable", + LOG_LEVEL_INFO, + undefined + ); + expect(fixture.log).toHaveBeenCalledWith(traversalError, LOG_LEVEL_VERBOSE, undefined); + }); + + it("stops before traversal when the device term is empty", async () => { + const fixture = createFixture({ term: "", usePluginSync: false, files: [".obsidian/app.json"] as FilePath[] }); + + await fixture.operations.scanAllConfigFiles(true); + + expect(fixture.listFiles).not.toHaveBeenCalled(); + expect(fixture.allDocsRaw).not.toHaveBeenCalled(); + expect(fixture.findEntries).not.toHaveBeenCalled(); + expect(fixture.snapshot.storeCustomizationFiles).not.toHaveBeenCalled(); + expect(fixture.snapshot.storeCustomisationFileV2).not.toHaveBeenCalled(); + expect(fixture.snapshot.deleteConfigOnDatabase).not.toHaveBeenCalled(); + expect(fixture.catalogue.updatePluginList).not.toHaveBeenCalled(); + expect(fixture.log).toHaveBeenCalledWith("Scanning customizing files.", LOG_LEVEL_NOTICE, "scan-all-config"); + expect(fixture.log).toHaveBeenCalledWith("We have to configure the device name", LOG_LEVEL_NOTICE, undefined); + }); + + it("dispatches according to the current V1/V2 setting on each scan", async () => { + const path = ".obsidian/app.json" as FilePath; + const fixture = createFixture({ files: [path], useV2: false }); + let useV2 = false; + fixture.dependencies.getSettings = () => ({ usePluginSyncV2: useV2 }); + + await fixture.operations.scanAllConfigFiles(false); + useV2 = true; + await fixture.operations.scanAllConfigFiles(false); + + expect(fixture.snapshot.storeCustomizationFiles).toHaveBeenCalledWith(path); + expect(fixture.snapshot.storeCustomizationFiles).toHaveBeenCalledTimes(1); + expect(fixture.snapshot.storeCustomisationFileV2).toHaveBeenCalledWith(path, "device-a"); + expect(fixture.snapshot.storeCustomisationFileV2).toHaveBeenCalledTimes(1); + }); + + it("routes V1 ownership and deletes only stale owned documents", async () => { + const ownedPath = ".obsidian/app.json" as FilePath; + const unownedPath = ".obsidian/appearance.json" as FilePath; + const ownedDocument = "ix:device-a/CONFIG/app.json.md" as FilePathWithPrefix; + const unownedDocument = "ix:device-a/CONFIG/appearance.json.md" as FilePathWithPrefix; + const staleDocument = "ix:device-a/CONFIG/stale.json.md" as FilePathWithPrefix; + const fixture = createFixture({ + useV2: false, + usePluginSync: false, + files: [ownedPath, unownedPath], + v1Paths: { + [ownedPath]: ownedDocument, + [unownedPath]: unownedDocument, + }, + databaseEntries: [ + { _id: ownedDocument, path: ownedDocument }, + { _id: unownedDocument, path: unownedDocument }, + { _id: staleDocument, path: staleDocument }, + ], + ownsLocalFile: (path) => path == ownedPath, + }); + + await fixture.operations.scanAllConfigFiles(false); + + expect(fixture.snapshot.storeCustomizationFiles).toHaveBeenCalledWith(ownedPath); + expect(fixture.snapshot.storeCustomizationFiles).toHaveBeenCalledTimes(1); + expect(fixture.snapshot.deleteConfigOnDatabase).toHaveBeenCalledWith(staleDocument); + expect(fixture.snapshot.deleteConfigOnDatabase).toHaveBeenCalledTimes(1); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledWith(false); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledTimes(1); + expect(fixture.allDocsRaw).toHaveBeenCalledWith({ + startkey: "ix:", + endkey: "ix:\u{10ffff}", + include_docs: true, + }); + }); + + it("propagates V1 snapshot failures without publishing a final refresh", async () => { + const path = ".obsidian/app.json" as FilePath; + const failure = new Error("V1 write failed"); + const fixture = createFixture({ files: [path] }); + fixture.snapshot.storeCustomizationFiles.mockRejectedValueOnce(failure); + + await expect(fixture.operations.scanAllConfigFiles(false)).rejects.toBe(failure); + + expect(fixture.catalogue.updatePluginList).not.toHaveBeenCalled(); + }); + + it("routes V2 ownership, removes matched keys, and deletes stale documents", async () => { + const ownedPath = ".obsidian/app.json" as FilePath; + const unownedPath = ".obsidian/appearance.json" as FilePath; + const extraPath = ".obsidian/plugins/example/main.js" as FilePath; + const ownedDocument = "ix:device-a/CONFIG/app.json%app.json" as FilePathWithPrefix; + const unownedDocument = "ix:device-a/CONFIG/appearance.json%appearance.json" as FilePathWithPrefix; + const staleDocument = "ix:device-a/CONFIG/stale.json" as FilePathWithPrefix; + const skippedDocument = "ix:device-a/CONFIG/skipped%app.json" as FilePathWithPrefix; + const owned = new Set([ownedPath, extraPath]); + const fixture = createFixture({ + useV2: true, + files: [ownedPath, unownedPath, extraPath], + v2Paths: { + [ownedPath]: ownedDocument, + [unownedPath]: unownedDocument, + [extraPath]: "ix:device-a/PLUGIN_MAIN/example%main.js" as FilePathWithPrefix, + }, + databaseEntries: [ + { _id: ownedDocument, path: "ix:device-a/CONFIG/app.json.md" as FilePathWithPrefix }, + { _id: unownedDocument, path: "ix:device-a/CONFIG/appearance.json.md" as FilePathWithPrefix }, + { _id: staleDocument, path: staleDocument }, + { _id: skippedDocument, path: skippedDocument }, + ], + ownsLocalFile: (path) => owned.has(path), + }); + + await fixture.operations.scanAllConfigFiles(false); + + expect(fixture.snapshot.storeCustomisationFileV2).toHaveBeenCalledWith(ownedPath, "device-a"); + expect(fixture.snapshot.storeCustomisationFileV2).toHaveBeenCalledWith(extraPath, "device-a"); + expect(fixture.snapshot.storeCustomisationFileV2).toHaveBeenCalledTimes(2); + expect(fixture.snapshot.storeCustomisationFileV2).not.toHaveBeenCalledWith(unownedPath, "device-a"); + expect(fixture.snapshot.deleteConfigOnDatabase).toHaveBeenCalledWith(staleDocument); + expect(fixture.snapshot.deleteConfigOnDatabase).toHaveBeenCalledTimes(1); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledWith(false); + expect(fixture.findEntries).toHaveBeenCalledWith("ix:device-a/", "ix:device-a/\u{10ffff}", { + include_docs: true, + }); + }); + + it("catches and logs each V2 entry failure before publishing the refresh", async () => { + const path = ".obsidian/app.json" as FilePath; + const document = "ix:device-a/CONFIG/app.json%app.json" as FilePathWithPrefix; + const failure = new Error("V2 write failed"); + const fixture = createFixture({ + useV2: true, + files: [path], + v2Paths: { [path]: document }, + databaseEntries: [{ _id: document, path: "ix:device-a/CONFIG/app.json.md" as FilePathWithPrefix }], + }); + fixture.snapshot.storeCustomisationFileV2.mockRejectedValueOnce(failure); + + await fixture.operations.scanAllConfigFiles(false); + + expect(fixture.log).toHaveBeenCalledWith( + `scanAllConfigFiles - Error: ${document}`, + LOG_LEVEL_VERBOSE, + undefined + ); + expect(fixture.log).toHaveBeenCalledWith(failure, LOG_LEVEL_VERBOSE, undefined); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledWith(false); + }); + + it("starts the final catalogue refresh without awaiting it", async () => { + const path = ".obsidian/app.json" as FilePath; + const fixture = createFixture({ files: [path] }); + let releaseRefresh!: () => void; + const refresh = new Promise((resolve) => { + releaseRefresh = resolve; + }); + const refreshStarted = vi.fn(); + fixture.catalogue.updatePluginList.mockImplementation(async () => { + refreshStarted(); + await refresh; + }); + + await fixture.operations.scanAllConfigFiles(false); + + expect(asyncHarness.fireAndForget).toHaveBeenCalledOnce(); + expect(refreshStarted).toHaveBeenCalledOnce(); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledWith(false); + releaseRefresh(); + await refresh; + }); +}); diff --git a/src/features/ConfigSync/snapshotOperations.ts b/src/features/ConfigSync/snapshotOperations.ts new file mode 100644 index 00000000..d043c0dd --- /dev/null +++ b/src/features/ConfigSync/snapshotOperations.ts @@ -0,0 +1,82 @@ +import type { + FilePath, + FilePathWithPrefix, + LOG_LEVEL, + ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_NOTICE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { fireAndForget } from "@vrtmrz/livesync-commonlib/compat/common/utils"; + +import { $msg } from "@/common/translation"; +import type { CatalogueOperations } from "./catalogueOperations.ts"; +import type { SnapshotPersistence, SnapshotRefresh } from "./snapshotPersistence.ts"; + +type SnapshotSettings = Pick; +type SnapshotPersistencePort = Pick< + SnapshotPersistence, + "storeCustomisationFileV2" | "storeCustomizationFiles" | "deleteConfigOnDatabase" +>; +type SnapshotCatalogue = Pick; + +export type SnapshotOperationsDependencies = { + getSettings(): SnapshotSettings; + getDeviceAndVaultName(): string; + log(message: unknown, level?: LOG_LEVEL, key?: string): void; + snapshotPersistence: SnapshotPersistencePort; + catalogueOperations: SnapshotCatalogue; +}; + +/** + * Adapts host-neutral Customisation Sync snapshot mutations to catalogue + * refreshes. Current-term selection and the inherited refresh timing live in + * this owner so scan, dialogue, and testing callers share one policy. + */ +export class SnapshotOperations { + constructor(private readonly dependencies: SnapshotOperationsDependencies) {} + + private _log(message: unknown, level?: LOG_LEVEL, key?: string) { + this.dependencies.log(message, level, key); + } + + isV2Enabled(): boolean { + return this.dependencies.getSettings().usePluginSyncV2; + } + + private async applyPersistenceRefreshes(refreshes: readonly SnapshotRefresh[]) { + for (const refresh of refreshes) { + if (refresh.mode == "v2" && refresh.timing == "fire-and-forget") { + fireAndForget(() => this.dependencies.catalogueOperations.updatePluginListV2(false, refresh.path)); + } else if (refresh.mode == "v1" && refresh.timing == "await") { + await this.dependencies.catalogueOperations.updatePluginList(false, refresh.path); + } + } + } + + async storeCustomisationFileV2(path: FilePath, term: string, force = false) { + const persistence = await this.dependencies.snapshotPersistence.storeCustomisationFileV2(path, term, force); + await this.applyPersistenceRefreshes(persistence.refreshes); + return persistence.value; + } + + async storeCustomizationFiles(path: FilePath, termOverride?: string) { + const term = termOverride || this.dependencies.getDeviceAndVaultName(); + if (term == "") { + this._log($msg("We have to configure the device name"), LOG_LEVEL_NOTICE); + return; + } + const persistence = this.isV2Enabled() + ? await this.dependencies.snapshotPersistence.storeCustomisationFileV2(path, term) + : await this.dependencies.snapshotPersistence.storeCustomizationFiles(path, term); + await this.applyPersistenceRefreshes(persistence.refreshes); + return persistence.value; + } + + async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false): Promise { + const persistence = await this.dependencies.snapshotPersistence.deleteConfigOnDatabase( + prefixedFileName, + forceWrite + ); + await this.applyPersistenceRefreshes(persistence.refreshes); + return persistence.value; + } +} diff --git a/src/features/ConfigSync/snapshotOperations.unit.spec.ts b/src/features/ConfigSync/snapshotOperations.unit.spec.ts new file mode 100644 index 00000000..bd6339bc --- /dev/null +++ b/src/features/ConfigSync/snapshotOperations.unit.spec.ts @@ -0,0 +1,137 @@ +import { describe, expect, it, vi } from "vitest"; + +const asyncHarness = vi.hoisted(() => ({ + fireAndForget: vi.fn((operation: () => unknown) => { + void operation(); + }), +})); + +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/utils", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fireAndForget: asyncHarness.fireAndForget, + }; +}); + +import type { FilePath, FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { SnapshotPersistenceResult } from "./snapshotPersistence.ts"; +import { SnapshotOperations, type SnapshotOperationsDependencies } from "./snapshotOperations.ts"; + +const CONFIG_PATH = ".obsidian/app.json" as FilePath; +const V1_PATH = "ix:device-b/CONFIG/app.json.md" as FilePathWithPrefix; +const V2_PATH = "ix:device-a/CONFIG/app.json%app.json" as FilePathWithPrefix; + +function createOperations(usePluginSyncV2: boolean) { + const events: string[] = []; + type PersistenceResult = SnapshotPersistenceResult; + const storeCustomisationFileV2 = vi.fn( + async (_path: FilePath, _term: string, _force?: boolean): Promise => ({ + value: true, + status: "saved", + refreshes: [], + }) + ); + const storeCustomizationFiles = vi.fn( + async (_path: FilePath, _term: string): Promise => ({ + value: true, + status: "saved", + refreshes: [], + }) + ); + const deleteConfigOnDatabase = vi.fn( + async (_path: FilePathWithPrefix, _force?: boolean): Promise => ({ + value: true, + status: "deleted", + refreshes: [], + }) + ); + const updatePluginList = vi.fn(async () => { + events.push("refresh-v1"); + }); + const updatePluginListV2 = vi.fn(async () => { + events.push("refresh-v2"); + }); + const dependencies: SnapshotOperationsDependencies = { + getSettings: () => ({ usePluginSyncV2 }), + getDeviceAndVaultName: () => "device-a", + log: vi.fn(), + snapshotPersistence: { + storeCustomisationFileV2, + storeCustomizationFiles, + deleteConfigOnDatabase, + }, + catalogueOperations: { updatePluginList, updatePluginListV2 }, + }; + return { + operations: new SnapshotOperations(dependencies), + events, + persistence: { storeCustomisationFileV2, storeCustomizationFiles, deleteConfigOnDatabase }, + catalogue: { updatePluginList, updatePluginListV2 }, + }; +} + +describe("Snapshot Operations", () => { + it("selects V1 persistence with an override term and awaits its refresh", async () => { + const fixture = createOperations(false); + fixture.persistence.storeCustomizationFiles.mockImplementation(async (_path: FilePath, term: string) => { + fixture.events.push(`persist:${term}`); + return { + value: true, + status: "saved", + refreshes: [{ mode: "v1", timing: "await", path: V1_PATH }], + }; + }); + + await expect(fixture.operations.storeCustomizationFiles(CONFIG_PATH, "device-b")).resolves.toBe(true); + + expect(fixture.persistence.storeCustomizationFiles).toHaveBeenCalledWith(CONFIG_PATH, "device-b"); + expect(fixture.events).toEqual(["persist:device-b", "refresh-v1"]); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledWith(false, V1_PATH); + }); + + it("selects V2 persistence with the current term and does not await its refresh", async () => { + const fixture = createOperations(true); + let releaseRefresh!: () => void; + const refresh = new Promise((resolve) => { + releaseRefresh = resolve; + }); + fixture.persistence.storeCustomisationFileV2.mockImplementation(async (_path: FilePath, term: string) => { + fixture.events.push(`persist:${term}`); + return { + value: true, + status: "saved", + refreshes: [{ mode: "v2", timing: "fire-and-forget", path: V2_PATH }], + }; + }); + fixture.catalogue.updatePluginListV2.mockImplementation(async () => { + fixture.events.push("refresh-v2-start"); + await refresh; + fixture.events.push("refresh-v2-end"); + }); + + await expect(fixture.operations.storeCustomizationFiles(CONFIG_PATH)).resolves.toBe(true); + + expect(fixture.events).toEqual(["persist:device-a", "refresh-v2-start"]); + releaseRefresh(); + await refresh; + expect(fixture.events).toEqual(["persist:device-a", "refresh-v2-start", "refresh-v2-end"]); + }); + + it("returns the persistence result after applying deletion refreshes", async () => { + const fixture = createOperations(false); + fixture.persistence.deleteConfigOnDatabase.mockImplementation(async () => ({ + value: true, + status: "deleted", + refreshes: [{ mode: "v1", timing: "await", path: V1_PATH }], + })); + + await expect(fixture.operations.deleteConfigOnDatabase(V1_PATH)).resolves.toBe(true); + + expect(fixture.persistence.deleteConfigOnDatabase).toHaveBeenCalledWith(V1_PATH, false); + expect(fixture.catalogue.updatePluginList).toHaveBeenCalledWith(false, V1_PATH); + }); +}); diff --git a/src/features/ConfigSync/snapshotPersistence.ts b/src/features/ConfigSync/snapshotPersistence.ts new file mode 100644 index 00000000..75ac4808 --- /dev/null +++ b/src/features/ConfigSync/snapshotPersistence.ts @@ -0,0 +1,402 @@ +import { parseYaml } from "@/deps.ts"; +import type { + FilePath, + FilePathWithPrefix, + InternalFileEntry, + LOG_LEVEL, + SavingEntry, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { LOG_LEVEL_DEBUG, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + createBlob, + createTextBlob, + getDocData, + getDocDataAsArray, + isDocContentSame, +} from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { EVEN } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const.symbols"; +import { digestHash } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/hash"; +import { arrayBufferToBase64 } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/convert"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; +import type { IPathService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; +import { base64ToArrayBuffer } from "octagonal-wheels/binary/base64"; +import { serialized } from "octagonal-wheels/concurrency/lock"; +import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError"; + +import { createCustomisationSyncCodec, type PluginDataEx } from "./customisationSyncCodec.ts"; +import type { CustomisationSyncPathOperations } from "./customisationSyncPathOperations.ts"; +import { readCustomisationFile } from "./customisationSyncReadOperations.ts"; + +const { + serialize, + deserialize, + dummyHead: DUMMY_HEAD, + dummyEnd: DUMMY_END, +} = createCustomisationSyncCodec({ digestHash, parseYaml }); + +type SnapshotPersistenceDatabase = Pick< + LiveSyncLocalDB, + "getDBEntryFromMeta" | "getDBEntryMeta" | "putDBEntry" | "putRaw" +>; + +type SnapshotPersistenceStorage = Pick; + +type SnapshotPersistencePath = Pick< + CustomisationSyncPathOperations, + "getFileCategory" | "filenameToUnifiedKey" | "filenameWithUnifiedKey" +> & + Pick; + +export type SnapshotPersistenceDependencies = { + getLocalDatabase(): SnapshotPersistenceDatabase; + storageAccess: SnapshotPersistenceStorage; + path: SnapshotPersistencePath; + log: LogFunction; + getConfigDir(): string; +}; + +export type SnapshotRefresh = { + mode: "v1" | "v2"; + timing: "await" | "fire-and-forget"; + path: FilePathWithPrefix; +}; + +export type SnapshotPersistenceStatus = "saved" | "skipped" | "missing" | "deleted" | "already-deleted" | "failed"; + +export type SnapshotPersistenceResult = { + value: Value; + status: SnapshotPersistenceStatus; + refreshes: readonly SnapshotRefresh[]; +}; + +type DatabaseSaveResult = Awaited>; +type StoreResultValue = DatabaseSaveResult | true | undefined; + +function result( + value: Value, + status: SnapshotPersistenceStatus, + refreshes: readonly SnapshotRefresh[] = [] +): SnapshotPersistenceResult { + return { value, status, refreshes }; +} + +/** + * Persists local Customisation Sync snapshots without owning catalogue state, + * lifecycle, replication, or user-interface behaviour. + */ +export class SnapshotPersistence { + private readonly dependencies: SnapshotPersistenceDependencies; + + constructor(dependencies: SnapshotPersistenceDependencies) { + this.dependencies = dependencies; + } + + private _log(message: unknown, level?: LOG_LEVEL, key?: string) { + this.dependencies.log(message, level, key); + } + + private async readFile(path: FilePath) { + return await readCustomisationFile( + { + storageAccess: this.dependencies.storageAccess, + log: this.dependencies.log, + }, + path, + this.dependencies.getConfigDir() + ); + } + + // Compatibility question: the inherited force parameter is not read. + // Preserve it until its intended write-bypass semantics are decided. + async storeCustomisationFileV2( + path: FilePath, + term: string, + force = false + ): Promise> { + void force; + const vf = this.dependencies.path.filenameWithUnifiedKey(path, term); + return await serialized(`plugin-${vf}`, async () => { + const prefixedFileName = vf; + + const id = await this.dependencies.path.path2id(prefixedFileName); + const stat = await this.dependencies.storageAccess.statHidden(path); + if (!stat) { + return result(false, "missing"); + } + const mtime = stat.mtime; + const content = await this.dependencies.storageAccess.readHiddenFileBinary(path); + const contentBlob = createBlob([DUMMY_HEAD, DUMMY_END, ...(await arrayBufferToBase64(content))]); + // const contentBlob = createBlob(content); + try { + const old = await this.dependencies + .getLocalDatabase() + .getDBEntryMeta(prefixedFileName, undefined, false); + let saveData: SavingEntry; + if (old === false) { + saveData = { + _id: id, + path: prefixedFileName, + data: contentBlob, + mtime, + ctime: mtime, + datatype: "plain", + size: contentBlob.size, + children: [], + deleted: false, + type: "plain", + eden: {}, + }; + } else { + // Compatibility question: this inherited marker check + // precedes loading the old document and can suppress a + // content comparison. Preserve that event-suppression + // ordering until its scan contract is reviewed. + if ( + this.dependencies.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN + ) { + this._log( + `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, + LOG_LEVEL_DEBUG + ); + return result(undefined, "skipped"); + } + const docXDoc = await this.dependencies.getLocalDatabase().getDBEntryFromMeta(old, false, false); + if (docXDoc == false) { + throw new LiveSyncError("Could not load the document"); + } + const dataSrc = getDocData(docXDoc.data); + const dataStart = dataSrc.indexOf(DUMMY_END); + const oldContent = dataSrc.substring(dataStart + DUMMY_END.length); + const oldContentArray = base64ToArrayBuffer(oldContent); + if (await isDocContentSame(oldContentArray, content)) { + this._log( + `STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, + LOG_LEVEL_VERBOSE + ); + this.dependencies.path.markChangesAreSame(prefixedFileName, old.mtime, mtime + 1); + return result(true, "skipped"); + } + saveData = { + ...old, + data: contentBlob, + mtime, + size: contentBlob.size, + datatype: "plain", + children: [], + deleted: false, + type: "plain", + }; + } + const ret = await this.dependencies.getLocalDatabase().putDBEntry(saveData); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); + // Compatibility question: the inherited refresh path omits the + // explicit term override and therefore uses the current term. + // Preserve that path until its cross-device semantics are reviewed. + return result(ret, "saved", [ + { + mode: "v2", + timing: "fire-and-forget", + path: this.dependencies.path.filenameWithUnifiedKey(path), + }, + ]); + } catch (ex) { + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); + return result(false, "failed"); + } + }); + } + + async storeCustomizationFiles(path: FilePath, term: string): Promise> { + const vf = this.dependencies.path.filenameToUnifiedKey(path, term); + // console.warn(`Storing ${path} to ${bareVF} :--> ${keyedVF}`); + + return await serialized(`plugin-${vf}`, async () => { + const category = this.dependencies.path.getFileCategory(path); + let mtime = 0; + let fileTargets = [] as FilePath[]; + // let savePath = ""; + const name = + category == "CONFIG" || category == "SNIPPET" + ? path.split("/").reverse()[0] + : path.split("/").reverse()[1]; + const parentPath = path.split("/").slice(0, -1).join("/"); + const prefixedFileName = this.dependencies.path.filenameToUnifiedKey(path, term); + const id = await this.dependencies.path.path2id(prefixedFileName); + const dt: PluginDataEx = { + category: category, + files: [], + name: name, + mtime: 0, + term: term, + }; + // let scheduleKey = ""; + if ( + category == "CONFIG" || + category == "SNIPPET" || + category == "PLUGIN_ETC" || + category == "PLUGIN_DATA" + ) { + fileTargets = [path]; + if (category == "PLUGIN_ETC") { + dt.displayName = path.split("/").slice(-1).join("/"); + } + } else if (category == "PLUGIN_MAIN") { + fileTargets = ["manifest.json", "main.js", "styles.css"].map((e) => `${parentPath}/${e}` as FilePath); + } else if (category == "THEME") { + fileTargets = ["manifest.json", "theme.css"].map((e) => `${parentPath}/${e}` as FilePath); + } + for (const target of fileTargets) { + const data = await this.readFile(target); + if (data == false) { + this._log(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE); + continue; + } + if (data.version) { + dt.version = data.version; + } + if (data.displayName) { + dt.displayName = data.displayName; + } + // Compatibility question: the inherited aggregation uses an + // average rather than the newest member mtime. Preserve that + // scan behaviour until its timestamp policy is reviewed. + mtime = mtime == 0 ? data.mtime : (data.mtime + mtime) / 2; + dt.files.push(data); + } + dt.mtime = mtime; + + // Compatibility question: the inherited empty-file path performs a + // deletion refresh and then an unconditional explicit refresh. Keep + // both outcomes, including the extra refresh when deletion succeeds. + if (dt.files.length == 0) { + this._log(`Nothing left: deleting.. ${path}`); + const deletion = await this.deleteConfigOnDatabase(prefixedFileName); + return result(undefined, deletion.status, [ + ...deletion.refreshes, + { mode: "v1", timing: "await", path: prefixedFileName }, + ]); + } + + const content = createTextBlob(serialize(dt)); + try { + const old = await this.dependencies + .getLocalDatabase() + .getDBEntryMeta(prefixedFileName, undefined, false); + let saveData: SavingEntry; + if (old === false) { + saveData = { + _id: id, + path: prefixedFileName, + data: content, + mtime, + ctime: mtime, + datatype: "newnote", + size: content.size, + children: [], + deleted: false, + type: "newnote", + eden: {}, + }; + } else { + if (old.mtime == mtime) { + // this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE); + return result(true, "skipped"); + } + const oldC = await this.dependencies.getLocalDatabase().getDBEntryFromMeta(old, false, false); + if (oldC) { + const d = deserialize(getDocDataAsArray(oldC.data), {}) as PluginDataEx; + if (d.files.length == dt.files.length) { + // Compatibility question: the inherited comparison + // looks up each current file by the previous filename + // and compares a missing lookup as empty content. + // Preserve this rename/empty-file behaviour for now. + const diffs = d.files + .map((previous) => ({ + prev: previous, + curr: dt.files.find((e) => e.filename == previous.filename), + })) + .map(async (e) => { + try { + return await isDocContentSame(e.curr?.data ?? [], e.prev.data); + } catch { + return false; + } + }); + const isSame = (await Promise.all(diffs)).every((e) => e == true); + if (isSame) { + this._log( + `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, + LOG_LEVEL_VERBOSE + ); + return result(true, "skipped"); + } + } + } + saveData = { + ...old, + data: content, + mtime, + size: content.size, + datatype: "newnote", + children: [], + deleted: false, + type: "newnote", + }; + } + const ret = await this.dependencies.getLocalDatabase().putDBEntry(saveData); + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); + return result(ret, "saved", [{ mode: "v1", timing: "await", path: saveData.path }]); + } catch (ex) { + this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); + return result(false, "failed"); + } + }); + } + + // Compatibility question: the inherited forceWrite parameter is not read. + // Preserve it until callers define whether deletion should bypass a marker. + async deleteConfigOnDatabase( + prefixedFileName: FilePathWithPrefix, + forceWrite = false + ): Promise> { + void forceWrite; + // const id = await this.path2id(prefixedFileName); + const mtime = new Date().getTime(); + return await serialized("file-x-" + prefixedFileName, async () => { + try { + const old = (await this.dependencies + .getLocalDatabase() + .getDBEntryMeta(prefixedFileName, undefined, false)) as InternalFileEntry | false; + let saveData: InternalFileEntry; + if (old === false) { + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted (Not found on database)`); + return result(true, "missing"); + } else { + if (old.deleted) { + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted`); + return result(true, "already-deleted"); + } + saveData = { + ...old, + mtime, + size: 0, + children: [], + deleted: true, + type: "newnote", + }; + } + await this.dependencies.getLocalDatabase().putRaw(saveData); + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Done`); + return result(true, "deleted", [{ mode: "v1", timing: "await", path: prefixedFileName }]); + } catch (ex) { + this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`); + this._log(ex, LOG_LEVEL_VERBOSE); + return result(false, "failed"); + } + }); + } +} diff --git a/src/features/ConfigSync/snapshotPersistence.unit.spec.ts b/src/features/ConfigSync/snapshotPersistence.unit.spec.ts new file mode 100644 index 00000000..8b14cd04 --- /dev/null +++ b/src/features/ConfigSync/snapshotPersistence.unit.spec.ts @@ -0,0 +1,219 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + diff_match_patch: class DiffMatchPatch {}, + normalizePath: vi.fn((path: string) => path), + parseYaml: vi.fn(), +})); +vi.mock("@/common/utils.ts", () => ({ + EVEN: Symbol("even"), + cancelTask: vi.fn(), + fireAndForget: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/types.ts", () => ({ + ICXHeader: "ix:", + PERIODIC_PLUGIN_SWEEP: 60, +})); +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ + getObsidianCommunityPluginManager: vi.fn(), +})); +vi.mock("@/features/optionalFileSyncFileTree.ts", () => ({ + collectOptionalFileSyncFiles: vi.fn(), +})); + +import type { FilePath, FilePathWithPrefix, LoadedEntry, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { EVEN } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const.symbols"; +import { createCustomisationSyncCodec } from "./customisationSyncCodec.ts"; +import { SnapshotPersistence, type SnapshotPersistenceDependencies } from "./snapshotPersistence.ts"; + +const CONFIG_PATH = ".obsidian/app.json" as FilePath; +const V1_PATH = "ix:device-a/CONFIG/app.json.md" as FilePathWithPrefix; +const V2_PATH = "ix:device-a/CONFIG/app.json%app.json" as FilePathWithPrefix; +const codec = createCustomisationSyncCodec({ + digestHash: (source) => source.join(""), + parseYaml: () => undefined, +}); + +function loadedV2Entry(source: string, mtime = 10): LoadedEntry { + const data = `${codec.dummyHead}${codec.dummyEnd}${btoa(source)}`; + return { + _id: "entry-id", + _rev: "1-a", + path: V2_PATH, + type: "plain", + datatype: "plain", + data, + ctime: mtime, + mtime, + size: data.length, + children: [], + eden: {}, + } as unknown as LoadedEntry; +} + +function createPersistence( + options: { + category?: "CONFIG" | "PLUGIN_MAIN"; + old?: false | LoadedEntry; + stat?: UXStat | null; + content?: string; + currentTerm?: string; + } = {} +) { + const currentTerm = options.currentTerm ?? "device-a"; + const statHidden = vi.fn( + async (_path: string): Promise => + options.stat === undefined ? { type: "file", ctime: 10, mtime: 10, size: 5 } : options.stat + ); + const readHiddenFileBinary = vi.fn( + async (_path: string) => new TextEncoder().encode(options.content ?? "hello").buffer + ); + const getDBEntryMeta = vi.fn(async () => options.old ?? false); + const getDBEntryFromMeta = vi.fn(async (entry: LoadedEntry) => entry); + const putDBEntry = vi.fn(async () => ({ ok: true, id: "entry-id", rev: "2-b" })); + const putRaw = vi.fn(async () => ({ ok: true, id: "entry-id", rev: "2-b" })); + const filenameToUnifiedKey = vi.fn( + (_path: string, term?: string) => + `ix:${term ?? currentTerm}/${options.category ?? "CONFIG"}/app.json.md` as FilePathWithPrefix + ); + const filenameWithUnifiedKey = vi.fn( + (_path: string, term?: string) => + `ix:${term ?? currentTerm}/${options.category ?? "CONFIG"}/app.json%app.json` as FilePathWithPrefix + ); + const dependencies: SnapshotPersistenceDependencies = { + getLocalDatabase: () => ({ getDBEntryMeta, getDBEntryFromMeta, putDBEntry, putRaw }), + storageAccess: { statHidden, readHiddenFileBinary }, + path: { + getFileCategory: () => options.category ?? "CONFIG", + filenameToUnifiedKey, + filenameWithUnifiedKey, + path2id: vi.fn(async (path) => path), + isMarkedAsSameChanges: vi.fn(), + markChangesAreSame: vi.fn(), + }, + log: vi.fn(), + getConfigDir: () => ".obsidian", + }; + return { + database: { getDBEntryMeta, getDBEntryFromMeta, putDBEntry, putRaw }, + dependencies, + filenameToUnifiedKey, + filenameWithUnifiedKey, + persistence: new SnapshotPersistence(dependencies), + readHiddenFileBinary, + statHidden, + }; +} + +describe("Customisation Sync snapshot persistence", () => { + it("persists a V2 file and returns a fire-and-forget catalogue refresh", async () => { + const fixture = createPersistence(); + + const mutation = await fixture.persistence.storeCustomisationFileV2(CONFIG_PATH, "device-a"); + + expect(mutation).toMatchObject({ + value: { ok: true, id: "entry-id", rev: "2-b" }, + status: "saved", + refreshes: [{ mode: "v2", timing: "fire-and-forget", path: V2_PATH }], + }); + expect(fixture.database.putDBEntry).toHaveBeenCalledOnce(); + expect(fixture.filenameWithUnifiedKey).toHaveBeenNthCalledWith(1, CONFIG_PATH, "device-a"); + expect(fixture.filenameWithUnifiedKey).toHaveBeenNthCalledWith(2, CONFIG_PATH); + }); + + it("aggregates the V1 plug-in file set and returns an awaited refresh", async () => { + const fixture = createPersistence({ category: "PLUGIN_MAIN" }); + + const mutation = await fixture.persistence.storeCustomizationFiles( + ".obsidian/plugins/example/main.js" as FilePath, + "device-a" + ); + + expect(mutation).toMatchObject({ + value: { ok: true }, + status: "saved", + refreshes: [{ mode: "v1", timing: "await", path: "ix:device-a/PLUGIN_MAIN/app.json.md" }], + }); + expect(fixture.readHiddenFileBinary).toHaveBeenCalledTimes(3); + expect(fixture.database.putDBEntry).toHaveBeenCalledOnce(); + }); + + it("keeps the inherited duplicate V1 refresh on the empty-file deletion path", async () => { + const old = { + ...loadedV2Entry("old"), + path: V1_PATH, + datatype: "newnote", + type: "newnote", + deleted: false, + } as LoadedEntry; + const fixture = createPersistence({ old, stat: null }); + + const mutation = await fixture.persistence.storeCustomizationFiles(CONFIG_PATH, "device-a"); + + expect(mutation.value).toBeUndefined(); + expect(mutation.status).toBe("deleted"); + expect(mutation.refreshes).toEqual([ + { mode: "v1", timing: "await", path: V1_PATH }, + { mode: "v1", timing: "await", path: V1_PATH }, + ]); + expect(fixture.database.putRaw).toHaveBeenCalledOnce(); + }); + + it.each([ + ["missing", false, "missing"], + ["already deleted", { ...loadedV2Entry("old"), deleted: true } as LoadedEntry, "already-deleted"], + ] as const)("treats an absent or %s document as a successful no-op", async (_label, old, status) => { + const fixture = createPersistence({ old, stat: null }); + + const mutation = await fixture.persistence.deleteConfigOnDatabase(V1_PATH); + + expect(mutation).toMatchObject({ value: true, status, refreshes: [] }); + expect(fixture.database.putRaw).not.toHaveBeenCalled(); + }); + + it("returns an awaited refresh only when deletion writes a live document", async () => { + const old = { + ...loadedV2Entry("old"), + path: V1_PATH, + deleted: false, + } as LoadedEntry; + const fixture = createPersistence({ old }); + + const mutation = await fixture.persistence.deleteConfigOnDatabase(V1_PATH); + + expect(mutation).toMatchObject({ + value: true, + status: "deleted", + refreshes: [{ mode: "v1", timing: "await", path: V1_PATH }], + }); + expect(fixture.database.putRaw).toHaveBeenCalledOnce(); + }); + + it("preserves the V2 marker and same-content skips", async () => { + const markerFixture = createPersistence({ old: loadedV2Entry("old") }); + const marker = markerFixture.dependencies.path.isMarkedAsSameChanges as ReturnType; + marker.mockReturnValue(EVEN); + await expect( + markerFixture.persistence.storeCustomisationFileV2(CONFIG_PATH, "device-a") + ).resolves.toMatchObject({ + value: undefined, + status: "skipped", + refreshes: [], + }); + expect(markerFixture.database.putDBEntry).not.toHaveBeenCalled(); + + const sameContentFixture = createPersistence({ old: loadedV2Entry("hello") }); + await expect( + sameContentFixture.persistence.storeCustomisationFileV2(CONFIG_PATH, "device-a") + ).resolves.toMatchObject({ + value: true, + status: "skipped", + refreshes: [], + }); + expect(sameContentFixture.database.putDBEntry).not.toHaveBeenCalled(); + }); +}); diff --git a/src/features/HiddenFileSync/hiddenFileSyncContext.ownership.unit.spec.ts b/src/features/HiddenFileSync/hiddenFileSyncContext.ownership.unit.spec.ts index 7fbdee92..7a8938e6 100644 --- a/src/features/HiddenFileSync/hiddenFileSyncContext.ownership.unit.spec.ts +++ b/src/features/HiddenFileSync/hiddenFileSyncContext.ownership.unit.spec.ts @@ -53,6 +53,9 @@ describe("HiddenFileSyncContext ownership and start-up lifecycle", () => { expect(getPrivate(first.context, "changeProcessor")).not.toBe( getPrivate(second.context, "changeProcessor") ); + expect(getPrivate(first.context, "reconciliation")).not.toBe( + getPrivate(second.context, "reconciliation") + ); expect(getPrivate(first.context, "periodicInternalFileScanProcessor")).toBe(first.periodicProcessor); expect(getPrivate(second.context, "periodicInternalFileScanProcessor")).toBe(second.periodicProcessor); @@ -67,8 +70,10 @@ describe("HiddenFileSyncContext ownership and start-up lifecycle", () => { "preserves start-up scan notice selection for the processed-file cache", async (processedFiles, forcedNotice) => { const { context } = createContext(processedFiles); - const applyOfflineChanges = vi.fn(async () => undefined); - context.applyOfflineChanges = applyOfflineChanges; + const reconciliation = getPrivate<{ + applyOfflineChanges(showNotice: boolean): Promise; + }>(context, "reconciliation"); + const applyOfflineChanges = vi.spyOn(reconciliation, "applyOfflineChanges").mockResolvedValue(undefined); await context.serviceHandlers.onDatabaseInitialised(false); diff --git a/src/features/HiddenFileSync/hiddenFileSyncContext.ts b/src/features/HiddenFileSync/hiddenFileSyncContext.ts index d792155e..6b19c518 100644 --- a/src/features/HiddenFileSync/hiddenFileSyncContext.ts +++ b/src/features/HiddenFileSync/hiddenFileSyncContext.ts @@ -6,28 +6,15 @@ import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, - type DocumentID, type MetaEntry, type ObsidianLiveSyncSettings, type LOG_LEVEL, } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "@/common/types.ts"; +import { ICHeader, ICHeaderEnd } from "@/common/types.ts"; import { type CustomRegExp } from "@vrtmrz/livesync-commonlib/compat/common/utils"; -import { - compareMTime, - isInternalMetadata, - TARGET_IS_NEW, - cancelTask, - scheduleTask, - getLogLevel, - onlyInNTimes, - BASE_IS_NEW, - EVEN, -} from "@/common/utils.ts"; -import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; +import { isInternalMetadata, cancelTask, scheduleTask } from "@/common/utils.ts"; +import { serialized } from "octagonal-wheels/concurrency/lock"; import { addPrefix, stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; -import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; -import { tryGetFilePath } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc"; import { configureHiddenFileSyncMode, type ConfigureHiddenFileSyncResult } from "./configureHiddenFileSyncMode.ts"; import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; import { $msg } from "@/common/translation"; @@ -42,14 +29,9 @@ import { type HiddenFileSyncCommandView, type HiddenFileSyncRepairView, type HiddenFileSyncServiceHandlerView, - type HiddenFileSyncTestingRebuild, type HiddenFileSyncTestingView, } from "./hiddenFileSyncViews.ts"; -import { describeHiddenFileSyncDocument, getHiddenFileSyncComparisonMTime } from "./hiddenFileSyncState.ts"; -import { - collectOptionalFileSyncFiles, - type OptionalFileSyncFileTreeDependencies, -} from "@/features/optionalFileSyncFileTree.ts"; +import type { OptionalFileSyncFileTreeDependencies } from "@/features/optionalFileSyncFileTree.ts"; import { deleteHiddenFileFromStorage, ensureHiddenFileDirectory, @@ -81,21 +63,20 @@ import { createHiddenFileSyncChangeProcessor, type HiddenFileSyncChangeProcessor, } from "./hiddenFileSyncChangeProcessor.ts"; -import { - createHiddenFileSyncPathAdmission, - type HiddenFileSyncPathAdmission, -} from "./hiddenFileSyncPathAdmission.ts"; +import { createHiddenFileSyncPathAdmission, type HiddenFileSyncPathAdmission } from "./hiddenFileSyncPathAdmission.ts"; import { createHiddenFileSyncChangeNotifier, type HiddenFileSyncChangeNotifier, } from "./hiddenFileSyncChangeNotifier.ts"; -type SyncDirection = "push" | "pull" | "safe" | "pullForce" | "pushForce"; +import { + createReconciliation, + type InitialisationDirection, + type ReconciliationProgress, + type Reconciliation, +} from "./reconciliation.ts"; -export type HiddenFileSyncProgress = { - log(message: string): void; - once(message: string): void; - done(message?: string): void; -}; +export type { ReconciliationProgress as HiddenFileSyncProgress } from "./reconciliation.ts"; +type SyncDirection = InitialisationDirection; type HiddenFileSyncSettings = Pick< ObsidianLiveSyncSettings, @@ -145,7 +126,7 @@ export type HiddenFileSyncContextDependencies = OptionalFileSyncFileTreeDependen getKeyValueDatabase(): KeyValueDatabase; databaseFileAccess: HiddenFileSyncDatabaseFileAccess; path: Pick; - createProgress(prefix?: string, level?: LOG_LEVEL): HiddenFileSyncProgress; + createProgress(prefix?: string, level?: LOG_LEVEL): ReconciliationProgress; createPeriodicProcessor(process: () => Promise): HiddenFileSyncPeriodicProcessor; isReady(): boolean; isSuspended(): boolean; @@ -180,13 +161,13 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { private readonly databaseExtractionOperations: HiddenFileSyncDatabaseExtractionOperations; private readonly conflictResolution: HiddenFileSyncConflictResolution; private readonly changeProcessor: HiddenFileSyncChangeProcessor; + private readonly reconciliation: Reconciliation; private readonly pathAdmission: HiddenFileSyncPathAdmission; private readonly changeNotifier: HiddenFileSyncChangeNotifier; readonly serviceHandlers: HiddenFileSyncServiceHandlerView; readonly testing: HiddenFileSyncTestingView; readonly repair: HiddenFileSyncRepairView; private readonly periodicInternalFileScanProcessor: HiddenFileSyncPeriodicProcessor; - private rebuildMergingHook: HiddenFileSyncTestingRebuild | undefined; private disposed = false; constructor(dependencies: HiddenFileSyncContextDependencies) { @@ -303,8 +284,21 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { log: (message, level, key) => dependencies.log(message, level, key), publishActivity: (eventCount, processingCount) => dependencies.publishActivity(eventCount, processingCount), }); + this.reconciliation = createReconciliation({ + listFiles: async (path) => await dependencies.listFiles(path), + getLocalDatabase: () => dependencies.getLocalDatabase(), + storageAccess: dependencies.storageAccess, + getRootPath: () => dependencies.getRootPath(), + getPath: (entry) => dependencies.path.getPath(entry), + isTargetFile: async (path) => await this.pathAdmission.isTargetFile(path), + isIgnoredByIgnoreFile: async (path) => await dependencies.isIgnoredByIgnoreFile(path), + createProgress: (prefix, level) => dependencies.createProgress(prefix, level), + processedState: this.processedState, + changeProcessor: this.changeProcessor, + log: (message, level, key) => dependencies.log(message, level, key), + }); this.repair = createHiddenFileSyncRepairView({ - scanInternalFiles: async () => await this.scanInternalFiles(), + scanInternalFiles: async () => await this.reconciliation.scanInternalFiles(), storeInternalFileToDatabase: async (file, forceWrite) => await this.databaseWriteOperations.store(file, forceWrite), storeInternalFileToDatabaseWithBaseRevision: async (file, baseRevision, createIfDifferent) => @@ -337,19 +331,7 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { readFileWithInfo: async (path) => await readHiddenFileWithInfo(dependencies, path), showConfigurationChangeNotice: (updatedFolders) => this.changeNotifier.showConfigurationChangeNotice(updatedFolders), - interceptRebuildMerging: (interceptor) => { - const previousHook = this.rebuildMergingHook; - const runRebuild = async (showNotice: boolean, targetFiles?: FilePath[] | false) => - await this.rebuildMerging(showNotice, targetFiles); - const hook: HiddenFileSyncTestingRebuild = async (showNotice, targetFiles) => - await interceptor(runRebuild, showNotice, targetFiles); - this.rebuildMergingHook = hook; - return () => { - if (this.rebuildMergingHook === hook) { - this.rebuildMergingHook = previousHook; - } - }; - }, + interceptRebuildMerging: (interceptor) => this.reconciliation.interceptRebuildMerging(interceptor), }); this.periodicInternalFileScanProcessor = dependencies.createPeriodicProcessor( async () => @@ -361,18 +343,6 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { return this.dependencies.getSettings(); } - private get localDatabase() { - return this.dependencies.getLocalDatabase(); - } - - private get storageAccess() { - return this.dependencies.storageAccess; - } - - private get databaseFileAccess() { - return this.dependencies.databaseFileAccess; - } - private getPath(entry: AnyEntry): FilePathWithPrefix { return this.dependencies.path.getPath(entry); } @@ -393,10 +363,6 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { this.dependencies.log(message, level, key); } - private _verbose(message: unknown, key?: string) { - this._log(message, LOG_LEVEL_VERBOSE, key); - } - private _progress(prefix: string = "", level: LOG_LEVEL = LOG_LEVEL_NOTICE) { return this.dependencies.createProgress(prefix, level); } @@ -410,10 +376,10 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { this.disposed = true; this.periodicInternalFileScanProcessor?.disable(); this.changeProcessor.dispose(); + this.reconciliation.dispose(); this.conflictResolution.dispose(); this.pathAdmission.dispose(); this.changeNotifier.dispose(); - this.rebuildMergingHook = undefined; this.dependencies.closeJsonConflictDialogs(); } @@ -492,7 +458,7 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { private async processOptionalFileEvent(path: FilePath): Promise { if (this.isReady()) { - return (await this.trackStorageFileModification(path)) || false; + return (await this.reconciliation.processStorageChange(path)) || false; } return false; } @@ -511,7 +477,7 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { // We should return true, we made sure that document is a internalMetadata. return true; } - if (!(await this.processReplicationResult(doc))) { + if (!(await this.reconciliation.processDatabaseDocument(doc))) { this._log(`Failed to process sync file:${unprefixedPath}`, LOG_LEVEL_NOTICE); // Do not yield false, this file had been processed. } @@ -521,290 +487,8 @@ export class HiddenFileSyncContext implements HiddenFileSyncCommandView { return false; } - private async adoptCurrentStorageFilesAsProcessed(targetFiles: FilePath[] | false) { - const allFiles = await this.scanInternalFileNames(); - const files = targetFiles ? allFiles.filter((e) => targetFiles.some((t) => e.indexOf(t) !== -1)) : allFiles; - for (const file of files) { - await this.processedState.updateLastProcessedAsActualFile(file); - } - } - private async adoptCurrentDatabaseFilesAsProcessed(targetFiles: FilePath[] | false) { - const allFiles = await this.getAllDatabaseFiles(); - const files = targetFiles - ? allFiles.filter((e) => targetFiles.some((t) => e.path.indexOf(t) !== -1)) - : allFiles; - for (const file of files) { - const path = stripAllPrefixes(this.getPath(file)); - await this.processedState.updateLastProcessedAsActualDatabase(path, file); - } - } - - private async trackScannedStorageChanges( - processFiles: FilePath[], - showNotice: boolean = false, - onlyNew = false, - forceWriteAll = false, - includeDeleted = true - ) { - const logLevel = getLogLevel(showNotice); - const p = this._progress(`[⚙ Storage -> DB ]\n`, logLevel); - const notifyProgress = onlyInNTimes(100, (progress) => p.log(`${progress}/${processFiles.length}`)); - const processes = processFiles.map(async (file, i) => { - try { - await this.trackStorageFileModification(file, onlyNew, forceWriteAll, includeDeleted); - notifyProgress(); - } catch (ex) { - p.once(`Failed to process storage change file:${file}`); - this._log(ex, LOG_LEVEL_VERBOSE); - } - }); - await Promise.all(processes); - p.done(); - } - async scanAllStorageChanges( - showNotice: boolean = false, - onlyNew = false, - forceWriteAll = false, - includeDeleted = true - ) { - return await skipIfDuplicated("scanAllStorageChanges", async () => { - const logLevel = getLogLevel(showNotice); - const p = this._progress(`[⚙ Scanning Storage -> DB ]\n`, logLevel); - p.log(`Scanning storage files...`); - const knownNames = [...this.processedState.getLastProcessedFileKeys()] as FilePath[]; - const existNames = await this.scanInternalFileNames(); - const files = new Set([...knownNames, ...existNames]); - - this._log( - `Known/Exist ${knownNames.length}/${existNames.length}, Totally ${files.size} files.`, - LOG_LEVEL_VERBOSE - ); - const taskNameAndMeta = [...files].map(async (e) => [e, await this.storageAccess.statHidden(e)] as const); - const nameAndMeta = await Promise.all(taskNameAndMeta); - const processFiles = nameAndMeta - .filter(([path, stat]) => { - if (forceWriteAll) return true; - const key = this.processedState.getLastProcessedFileKey(path); - const newKey = this.processedState.storageStateKey(stat); - return key != newKey; - }) - .map(([path, stat]) => path); - - const staticsMessage = `[Storage hidden file statics] -Known files: ${knownNames.length} -Actual files: ${existNames.length} -All files: ${files.size} -Offline Changed files: ${processFiles.length}`; - // this._log(staticsMessage, logLevel, "scan-changes"); - p.once(staticsMessage); - await this.trackScannedStorageChanges(processFiles, showNotice, onlyNew, forceWriteAll, includeDeleted); - p.done(); - }); - } - - /** - * check the file is changed or not, and if changed, process it. - */ - private async trackStorageFileModification( - path: FilePath, - onlyNew = false, - forceWrite = false, - includeDeleted = true - ): Promise { - if (!(await this.pathAdmission.isTargetFile(path))) { - this._log( - `Storage file tracking: Hidden file skipped: ${path} is filtered out by the defined patterns.`, - LOG_LEVEL_VERBOSE - ); - return false; - } - return await this.changeProcessor.processStorageChange(path, onlyNew, forceWrite, includeDeleted); - } - - // --> Event Source Handler (Database) - private async processReplicationResult(doc: LoadedEntry): Promise { - const info = describeHiddenFileSyncDocument(doc, this.getPath(doc)); - const path = info.path; - const headerLine = `Tracking DB ${info.path} (${info.revDisplay}) :`; - const ret = await this.trackDatabaseFileModification(path, headerLine); - this._log(`${headerLine} Done: ${info.shortenedId})`, LOG_LEVEL_VERBOSE); - return ret; - } - - // <-- Event Source Handler (Database) - // --> Database Event Functions - private async trackScannedDatabaseChange( - processFiles: MetaEntry[], - showNotice: boolean = false, - onlyNew = false, - forceWriteAll = false, - includeDeletion = true - ) { - const logLevel = getLogLevel(showNotice); - const p = this._progress(`[⚙ DB -> Storage ]\n`, logLevel); - const notifyProgress = onlyInNTimes(100, (progress) => p.log(`${progress}/${processFiles.length}`)); - const processes = processFiles.map(async (file) => { - try { - const path = stripAllPrefixes(this.getPath(file)); - if (!(await this.pathAdmission.isTargetFile(path))) { - this._log( - `Database file tracking: Hidden file skipped: ${path} is filtered out by the defined patterns.`, - LOG_LEVEL_VERBOSE - ); - } else { - await this.trackDatabaseFileModification( - path, - "[Hidden file scan]", - !forceWriteAll, - onlyNew, - file, - includeDeletion - ); - } - notifyProgress(); - } catch (ex) { - this._log(`Failed to process storage change file:${tryGetFilePath(file)}`, logLevel); - this._log(ex, LOG_LEVEL_VERBOSE); - } - }); - await Promise.all(processes); - p.done(); - } - - async applyOfflineChanges(showNotice: boolean) { - const logLevel = getLogLevel(showNotice); - return await serialized("applyOfflineChanges", async () => { - const p = this._progress("[⚙ Apply untracked changes ]\n", logLevel); - this._log(`Track changes.`, logLevel); - p.log("Enumerating local files..."); - const currentStorageFiles = await this.scanInternalFileNames(); - p.log("Enumerating database files..."); - const currentDatabaseFiles = await this.getAllDatabaseFiles(); - const allDatabaseMap = Object.fromEntries( - currentDatabaseFiles.map((e) => [stripAllPrefixes(this.getPath(e)), e]) - ); - const currentDatabaseFileNames = [...Object.keys(allDatabaseMap)] as FilePath[]; - const untrackedLocal = currentStorageFiles.filter((e) => !this.processedState.hasLastProcessedFile(e)); - const untrackedDatabase = currentDatabaseFileNames.filter( - (e) => !this.processedState.hasLastProcessedDatabase(e) - ); - const bothUntracked = untrackedLocal.filter((e) => untrackedDatabase.indexOf(e) !== -1); - p.log("Applying untracked changes..."); - const stat = `Tracking statics: -Local files: ${currentStorageFiles.length} -Database files: ${currentDatabaseFileNames.length} -Untracked local files: ${untrackedLocal.length} -Untracked database files: ${untrackedDatabase.length} -Common untracked files: ${bothUntracked.length}`; - p.once(stat); - const semaphores = Semaphore(10); - const notifyProgress = onlyInNTimes(25, (progress) => p.log(`${progress}/${bothUntracked.length}`)); - const allProcesses = bothUntracked.map(async (file) => { - notifyProgress(); - const rel = await semaphores.acquire(); - try { - const fileStat = await this.storageAccess.statHidden(file); - if (fileStat == null) { - // This should not be happened. But, if it happens, we should skip this. - this._log(`Unexpected error: Failed to stat file during applyOfflineChange :${file}`); - return; - } - const dbInfo = allDatabaseMap[file]; - if (dbInfo.deleted || dbInfo._deleted) { - // Applying deletion can be harmful if the local file is not tracked. - // So, we should skip this. - return; - } - const fileMTime = getHiddenFileSyncComparisonMTime(fileStat); - const dbMTime = getHiddenFileSyncComparisonMTime(dbInfo); - const diff = compareMTime(fileMTime, dbMTime); - if (diff == BASE_IS_NEW) { - // Local file is newer than the database file. - // So, we should apply the local file to the database. - await this.trackStorageFileModification(file, true); - } else if (diff == TARGET_IS_NEW) { - // Database file is newer than the local file. - // So, we should apply the database file to the local file. - await this.trackDatabaseFileModification(file, "[Apply]", true, true, dbInfo); - } else if (diff == EVEN) { - // Both are same, we may skip this but should update the last processed key. - this.processedState.updateLastProcessed(file, dbInfo, fileStat); - } - } finally { - rel(); - } - }); - await Promise.all(allProcesses); - await this.scanAllStorageChanges(showNotice); - await this.scanAllDatabaseChanges(showNotice); - - p.done(); - }); - } - - async scanAllDatabaseChanges( - showNotice: boolean = false, - onlyNew = false, - forceWriteAll = false, - includeDeletion = true - ) { - return await skipIfDuplicated("scanAllDatabaseChanges", async () => { - const databaseFiles = await this.getAllDatabaseFiles(); - const files = databaseFiles.filter((e) => { - const doc = e; - const key = this.processedState.databaseStateKey(doc); - const path = stripAllPrefixes(this.getPath(doc)); - const lastKey = this.processedState.getLastProcessedDatabaseKey(path); - return lastKey != key; - }); - const logLevel = getLogLevel(showNotice); - const staticsMessage = `[Database hidden file statics] -All files: ${databaseFiles.length} -Offline Changed files: ${files.length}`; - this._log(staticsMessage, logLevel, "scan-changes"); - return await this.trackScannedDatabaseChange(files, showNotice, onlyNew, forceWriteAll, includeDeletion); - }); - } - - private async useDatabaseFiles(files: MetaEntry[], showNotice = false, onlyNew = false) { - const logLevel = getLogLevel(showNotice); - const p = this._progress(`[⚙ Scanning DB -> Storage ]\n`, logLevel); - p.log("Scanning database files..."); - const notifyProgress = onlyInNTimes(25, (progress) => p.log(`${progress}/${files.length}`)); - const processFiles = files.map(async (file) => { - try { - const path = stripAllPrefixes(this.getPath(file)); - await this.trackDatabaseFileModification(path, "[Scanning]", true, onlyNew, file); - notifyProgress(); - } catch (ex) { - this._log(`Failed to process database changes:${tryGetFilePath(file)}`); - this._log(ex, LOG_LEVEL_VERBOSE); - } - return; - }); - await Promise.all(processFiles); - p.done(); - return true; - } - - private async trackDatabaseFileModification( - path: FilePath, - headerLine: string, - preventDoubleProcess = false, - onlyNew = false, - meta: MetaEntry | false = false, - includeDeletion = true - ): Promise { - return await this.changeProcessor.processDatabaseChange(path, headerLine, { - preventDoubleProcess, - onlyNew, - metaEntry: meta, - includeDeletion, - }); - } - private queueConflict(path: FilePathWithPrefix): Promise { this.conflictResolution.queue(path); return Promise.resolve(true); @@ -812,188 +496,42 @@ Offline Changed files: ${files.length}`; // <-- Database Event Functions - // --> Initialization functions - - private async rebuildMerging(showNotice: boolean, targetFiles: FilePath[] | false = false) { - const logLevel = getLogLevel(showNotice); - const p = this._progress("[⚙ Rebuild by Merge ]\n", logLevel); - this._log(`Rebuilding hidden files from the storage and the local database.`, logLevel); - p.log("Enumerating local files..."); - const currentStorageFilesAll = await this.scanInternalFileNames(); - const currentStorageFiles = targetFiles - ? currentStorageFilesAll.filter((e) => targetFiles.some((f) => f == e)) - : currentStorageFilesAll; - p.log("Enumerating database files..."); - const allDatabaseFiles = await this.getAllDatabaseFiles(); - const allDatabaseMap = new Map(allDatabaseFiles.map((e) => [stripAllPrefixes(this.getPath(e)), e])); - const currentDatabaseFiles = targetFiles - ? allDatabaseFiles.filter((e) => targetFiles.some((f) => f == stripAllPrefixes(this.getPath(e)))) - : allDatabaseFiles; - - const allFileNames = new Set([ - ...currentStorageFiles, - ...currentDatabaseFiles.map((e) => stripAllPrefixes(this.getPath(e))), - ]); - const storageToDatabase = [] as FilePath[]; - const databaseToStorage = [] as MetaEntry[]; - - const eachProgress = onlyInNTimes(100, (progress) => p.log(`Checking ${progress}/${allFileNames.size}`)); - for (const file of allFileNames) { - eachProgress(); - const storageMTime = await this.storageAccess.statHidden(file); - const mtimeStorage = getHiddenFileSyncComparisonMTime(storageMTime); - const dbEntry = allDatabaseMap.get(file)!; - const mtimeDB = getHiddenFileSyncComparisonMTime(dbEntry); - const diff = compareMTime(mtimeStorage, mtimeDB); - if (diff == BASE_IS_NEW) { - storageToDatabase.push(file); - } else if (diff == TARGET_IS_NEW) { - databaseToStorage.push(dbEntry); - } else if (diff == EVEN) { - // For safety, storage to database. - storageToDatabase.push(file); - } - } - p.once( - `Storage to Database: ${storageToDatabase.length} files\n Database to Storage: ${databaseToStorage.length} files` - ); - this.processedState.resetLastProcessedDatabase(targetFiles); - this.processedState.resetLastProcessedFile(targetFiles); - const processes = [ - this.trackScannedStorageChanges(storageToDatabase, showNotice, false, true), - this.useDatabaseFiles(databaseToStorage, showNotice, false), - ]; - p.log("Start processing..."); - await Promise.all(processes); - p.done(); - return [...allFileNames]; + async scanAllStorageChanges( + showNotice: boolean = false, + onlyNew = false, + forceWriteAll = false, + includeDeleted = true + ): Promise { + return await this.reconciliation.scanAllStorageChanges(showNotice, onlyNew, forceWriteAll, includeDeleted); } - private async runRebuildMerging(showNotice: boolean, targetFiles: FilePath[] | false = false) { - return this.rebuildMergingHook - ? await this.rebuildMergingHook(showNotice, targetFiles) - : await this.rebuildMerging(showNotice, targetFiles); + async scanAllDatabaseChanges( + showNotice: boolean = false, + onlyNew = false, + forceWriteAll = false, + includeDeletion = true + ): Promise { + return await this.reconciliation.scanAllDatabaseChanges(showNotice, onlyNew, forceWriteAll, includeDeletion); } - private async rebuildFromStorage(showNotice: boolean, targetFiles: FilePath[] | false = false, onlyNew = false) { - // reset processed file markers - const logLevel = getLogLevel(showNotice); - this._verbose(`Rebuilding hidden files from the storage.`); - this._log(`Rebuilding hidden files from the storage.`, logLevel); - const p = this._progress("[⚙ Rebuild by Storage ]\n", logLevel); - p.log("Enumerating local files..."); - const currentFilesAll = await this.scanInternalFileNames(); - const currentFiles = targetFiles - ? currentFilesAll.filter((e) => targetFiles.some((f) => f == e)) - : currentFilesAll; - p.once(`Storage to Database: ${currentFiles.length} files.`); - p.log("Start processing..."); - this.processedState.resetLastProcessedFile(targetFiles); - await this.trackScannedStorageChanges(currentFiles, showNotice, onlyNew, true); - p.done(); - return currentFiles; - } - - private async getAllDatabaseFiles() { - const allFiles = ( - await this.localDatabase.allDocsRaw({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true }) - ).rows - .filter((e) => isInternalMetadata(e.id as DocumentID)) - .map((e) => e.doc) as MetaEntry[]; - const files = [] as MetaEntry[]; - for (const file of allFiles) { - if (await this.pathAdmission.isTargetFile(stripAllPrefixes(this.getPath(file)))) { - files.push(file); - } - } - return files; - } - - private async rebuildFromDatabase(showNotice: boolean, targetFiles: FilePath[] | false = false, onlyNew = false) { - const logLevel = getLogLevel(showNotice); - this._verbose(`Rebuilding hidden files from the local database.`); - const p = this._progress("[⚙ Rebuild by Database ]\n", logLevel); - p.log("Enumerating database files..."); - const allFiles = await this.getAllDatabaseFiles(); - - // THINKING: Should we exclude conflicted or deleted files? - // Current implementation is to include all files, and following processes will handle for them. - // However, in perspective of performance and future-proofing, I feel somewhat justified in doing it here. - - const currentFiles = targetFiles - ? allFiles.filter((e) => targetFiles.some((f) => f == stripAllPrefixes(this.getPath(e)))) - : allFiles; - - p.once(`Database to Storage: ${currentFiles.length} files.`); - this.processedState.resetLastProcessedDatabase(targetFiles); - p.log("Start processing..."); - await this.useDatabaseFiles(currentFiles, showNotice, onlyNew); - p.done(); - return currentFiles; + async applyOfflineChanges(showNotice: boolean): Promise { + return await this.reconciliation.applyOfflineChanges(showNotice); } async initialiseInternalFileSync( direction: SyncDirection, showMessage: boolean, - // filesAll: InternalFileInfo[] | false = false, targetFilesSrc: string[] | false = false, - initialisationProgress?: HiddenFileSyncProgress - ) { - const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; - const p = initialisationProgress ?? this._progress("[⚙ Initialise]\n", logLevel); - // Compatibility question: the legacy preflight was already disabled. - // Enabling it would change initialisation timing and could open a - // conflict dialogue while the feature is being configured. - // p.log("Resolving conflicts before starting..."); - // await this.conflictResolution.resolveAll(); - p.log("Initialising hidden files sync..."); - // The initialisation progress owns the user-visible Notice. Its child - // rebuild and scan operations still write ordinary log entries, but - // must not each create another keep-alive Notice. - const showChildNotices = false; - // TODO: Handling ignore files cannot be performed to the hidden files. - - const targetFiles = targetFilesSrc - ? targetFilesSrc.map((e) => stripAllPrefixes(e as FilePathWithPrefix)) - : false; - if (direction == "pushForce" || direction == "push") { - const onlyNew = direction == "push"; - p.log(`Started: Storage --> Database ${onlyNew ? "(Only New)" : ""}`); - const updatedFiles = await this.rebuildFromStorage(showChildNotices, targetFiles, onlyNew); - // making doubly sure, No more losing files. - // I did so many times during the development. - await this.adoptCurrentStorageFilesAsProcessed(updatedFiles); - await this.adoptCurrentDatabaseFilesAsProcessed(updatedFiles); - // And, scan other changes on the database (i.e. files which are on only other devices) - p.log("Checking for remaining storage and database changes..."); - await this.scanAllStorageChanges(showChildNotices, true, false); - await this.scanAllDatabaseChanges(showChildNotices, true, false); - } - if (direction == "pullForce" || direction == "pull") { - const onlyNew = direction == "pull"; - p.log(`Started: Database --> Storage ${onlyNew ? "(Only New)" : ""}`); - const updatedEntries = await this.rebuildFromDatabase(showChildNotices, targetFiles, onlyNew); - const updatedFiles = updatedEntries.map((e) => stripAllPrefixes(this.getPath(e))); - // making doubly sure, No more losing files. - await this.adoptCurrentStorageFilesAsProcessed(updatedFiles); - await this.adoptCurrentDatabaseFilesAsProcessed(updatedFiles); - // And, scan other changes on the database (i.e. files which are on only other devices) - p.log("Checking for remaining database and storage changes..."); - await this.scanAllDatabaseChanges(showChildNotices, true, false); - await this.scanAllStorageChanges(showChildNotices, true, false); - } - if (direction == "safe") { - p.log(`Started: Database <--> Storage (by modified date)`); - const updatedFiles = await this.runRebuildMerging(showChildNotices, targetFiles); - await this.adoptCurrentStorageFilesAsProcessed(updatedFiles); - await this.adoptCurrentDatabaseFilesAsProcessed(updatedFiles); - // And, scan other changes on the database (i.e. files which are on only other devices) - p.log("Checking for remaining storage and database changes..."); - await this.scanAllStorageChanges(showChildNotices, true, false); - await this.scanAllDatabaseChanges(showChildNotices, true, false); - } - p.done(); + initialisationProgress?: ReconciliationProgress + ): Promise { + return await this.reconciliation.initialiseInternalFileSync( + direction, + showMessage, + targetFilesSrc, + initialisationProgress + ); } + // <-- Initialization functions private suspendExtraSync(): Promise { @@ -1016,7 +554,7 @@ Offline Changed files: ${files.length}`; } private async configureHiddenFileSync(mode: OptionalSyncFeatureMode) { - let initialisationProgress: HiddenFileSyncProgress | undefined; + let initialisationProgress: ReconciliationProgress | undefined; let result: ConfigureHiddenFileSyncResult; try { result = await configureHiddenFileSyncMode(mode, { @@ -1058,47 +596,5 @@ Offline Changed files: ${files.length}`; } // <-- Configuration handling - // --> Local Storage SubFunctions - private async scanInternalFileNames() { - const findRoot = this.dependencies.getRootPath(); - - const filenames = await collectOptionalFileSyncFiles(this.dependencies, findRoot, { - shouldInclude: (path) => this.pathAdmission.isTargetFile(path as FilePath), - onError: (path, error) => { - this._log(`Could not traverse(HiddenSync):${path}`, LOG_LEVEL_INFO); - this._log(error, LOG_LEVEL_VERBOSE); - }, - }); - - return filenames as FilePath[]; - } - - private async scanInternalFiles(): Promise { - const fileNames = await this.scanInternalFileNames(); - const files = fileNames.map(async (e) => { - return { - path: e, - stat: await this.storageAccess.statHidden(e), - }; - }); - const result: InternalFileInfo[] = []; - for (const f of files) { - const w = await f; - if (await this.dependencies.isIgnoredByIgnoreFile(w.path)) { - continue; - } - const mtime = w.stat?.mtime ?? 0; - const ctime = w.stat?.ctime ?? mtime; - const size = w.stat?.size ?? 0; - result.push({ - ...w, - mtime, - ctime, - size, - }); - } - return result; - } - // <-- Local Storage SubFunctions } diff --git a/src/features/HiddenFileSync/hiddenFileSyncContext.unit.spec.ts b/src/features/HiddenFileSync/hiddenFileSyncContext.unit.spec.ts index 10975395..3db0a76b 100644 --- a/src/features/HiddenFileSync/hiddenFileSyncContext.unit.spec.ts +++ b/src/features/HiddenFileSync/hiddenFileSyncContext.unit.spec.ts @@ -19,6 +19,7 @@ describe("HiddenFileSyncContext lifecycle", () => { const periodicInternalFileScanProcessor = { disable: vi.fn() }; const conflictResolution = { dispose: vi.fn() }; const changeProcessor = { dispose: vi.fn() }; + const reconciliation = { dispose: vi.fn() }; const pathAdmission = { dispose: vi.fn() }; const changeNotifier = { dispose: vi.fn() }; const closeJsonConflictDialogs = vi.fn(); @@ -28,6 +29,7 @@ describe("HiddenFileSyncContext lifecycle", () => { periodicInternalFileScanProcessor, conflictResolution, changeProcessor, + reconciliation, pathAdmission, changeNotifier, eventCount: 4, @@ -40,6 +42,7 @@ describe("HiddenFileSyncContext lifecycle", () => { expect(periodicInternalFileScanProcessor.disable).toHaveBeenCalledOnce(); expect(conflictResolution.dispose).toHaveBeenCalledOnce(); expect(changeProcessor.dispose).toHaveBeenCalledOnce(); + expect(reconciliation.dispose).toHaveBeenCalledOnce(); expect(pathAdmission.dispose).toHaveBeenCalledOnce(); expect(changeNotifier.dispose).toHaveBeenCalledOnce(); expect(closeJsonConflictDialogs).toHaveBeenCalledOnce(); @@ -60,35 +63,6 @@ describe("HiddenFileSyncContext lifecycle", () => { expect(callPrivate<() => boolean>(hiddenFileSync, "isReady")()).toBe(false); }); - it("keeps subordinate initialisation phases below Notice level so one progress Notice owns the scan", async () => { - const progress = { - log: vi.fn(), - once: vi.fn(), - done: vi.fn(), - }; - const rebuildMerging = vi.fn(async () => []); - const adoptCurrentStorageFilesAsProcessed = vi.fn(async () => undefined); - const adoptCurrentDatabaseFilesAsProcessed = vi.fn(async () => undefined); - const scanAllStorageChanges = vi.fn(async () => undefined); - const scanAllDatabaseChanges = vi.fn(async () => undefined); - const hiddenFileSync = Object.create(HiddenFileSyncContext.prototype) as HiddenFileSyncContext; - Object.assign(hiddenFileSync, { - _progress: vi.fn(() => progress), - rebuildMerging, - adoptCurrentStorageFilesAsProcessed, - adoptCurrentDatabaseFilesAsProcessed, - scanAllStorageChanges, - scanAllDatabaseChanges, - }); - - await hiddenFileSync.initialiseInternalFileSync("safe", true); - - expect(rebuildMerging).toHaveBeenCalledWith(false, false); - expect(scanAllStorageChanges).toHaveBeenCalledWith(false, true, false); - expect(scanAllDatabaseChanges).toHaveBeenCalledWith(false, true, false); - expect(progress.done).toHaveBeenCalledOnce(); - }); - it("retirement guard: does not restore separate gathering and restart Notices", async () => { vi.mocked(configureHiddenFileSyncMode).mockImplementation(async (_mode, handlers) => { await handlers.enable(); diff --git a/src/features/HiddenFileSync/reconciliation.ts b/src/features/HiddenFileSync/reconciliation.ts new file mode 100644 index 00000000..e05cc7b6 --- /dev/null +++ b/src/features/HiddenFileSync/reconciliation.ts @@ -0,0 +1,683 @@ +import { + type AnyEntry, + type DocumentID, + type FilePath, + type FilePathWithPrefix, + type LoadedEntry, + LOG_LEVEL_INFO, + LOG_LEVEL_NOTICE, + LOG_LEVEL_VERBOSE, + type LOG_LEVEL, + type MetaEntry, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; +import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; +import { tryGetFilePath } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc"; +import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; +import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; +import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; + +import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "@/common/types.ts"; +import { + BASE_IS_NEW, + compareMTime, + EVEN, + getLogLevel, + isInternalMetadata, + onlyInNTimes, + TARGET_IS_NEW, +} from "@/common/utils.ts"; +import { + collectOptionalFileSyncFiles, + type OptionalFileSyncFileTreeDependencies, +} from "@/features/optionalFileSyncFileTree.ts"; +import type { HiddenFileSyncChangeProcessor } from "./hiddenFileSyncChangeProcessor.ts"; +import type { HiddenFileSyncProcessedState } from "./hiddenFileSyncProcessedState.ts"; +import { describeHiddenFileSyncDocument, getHiddenFileSyncComparisonMTime } from "./hiddenFileSyncState.ts"; +import type { + HiddenFileSyncInitialisationDirection as InitialisationDirection, + HiddenFileSyncTestingRebuild, + HiddenFileSyncTestingRebuildInterceptor, +} from "./hiddenFileSyncViews.ts"; + +export type { HiddenFileSyncInitialisationDirection as InitialisationDirection } from "./hiddenFileSyncViews.ts"; + +export type ReconciliationProgress = { + log(message: string): void; + once(message: string): void; + done(message?: string): void; +}; + +type ReconciliationDatabase = Pick; +type ReconciliationStorage = Pick; + +type ReconciliationProcessedState = Pick< + HiddenFileSyncProcessedState, + | "databaseStateKey" + | "getLastProcessedDatabaseKey" + | "getLastProcessedFileKey" + | "getLastProcessedFileMTime" + | "hasLastProcessedDatabase" + | "hasLastProcessedFile" + | "getLastProcessedFileKeys" + | "resetLastProcessedDatabase" + | "resetLastProcessedFile" + | "storageStateKey" + | "updateLastProcessed" + | "updateLastProcessedAsActualDatabase" + | "updateLastProcessedAsActualFile" +>; + +type ReconciliationChangeProcessor = Pick< + HiddenFileSyncChangeProcessor, + "processStorageChange" | "processDatabaseChange" +>; + +export type ReconciliationDependencies = OptionalFileSyncFileTreeDependencies & { + getLocalDatabase(): ReconciliationDatabase; + storageAccess: ReconciliationStorage; + getRootPath(): string; + getPath(entry: AnyEntry): FilePathWithPrefix; + isTargetFile(path: FilePath): Promise; + isIgnoredByIgnoreFile(path: string): Promise; + createProgress(prefix?: string, level?: LOG_LEVEL): ReconciliationProgress; + processedState: ReconciliationProcessedState; + changeProcessor: ReconciliationChangeProcessor; + log: LogFunction; +}; + +export type Reconciliation = { + processStorageChange( + path: FilePath, + onlyNew?: boolean, + forceWrite?: boolean, + includeDeleted?: boolean + ): Promise; + processDatabaseDocument(doc: LoadedEntry): Promise; + scanInternalFiles(): Promise; + scanAllStorageChanges( + showNotice?: boolean, + onlyNew?: boolean, + forceWriteAll?: boolean, + includeDeleted?: boolean + ): Promise; + scanAllDatabaseChanges( + showNotice?: boolean, + onlyNew?: boolean, + forceWriteAll?: boolean, + includeDeletion?: boolean + ): Promise; + applyOfflineChanges(showNotice: boolean): Promise; + initialiseInternalFileSync( + direction: InitialisationDirection, + showMessage: boolean, + targetFilesSrc?: string[] | false, + initialisationProgress?: ReconciliationProgress + ): Promise; + interceptRebuildMerging(interceptor: HiddenFileSyncTestingRebuildInterceptor): () => void; + dispose(): void; +}; + +class ReconciliationOwner implements Reconciliation { + private rebuildMergingHook: HiddenFileSyncTestingRebuild | undefined; + + constructor(private readonly dependencies: ReconciliationDependencies) {} + + private get localDatabase() { + return this.dependencies.getLocalDatabase(); + } + + private get storageAccess() { + return this.dependencies.storageAccess; + } + + private getPath(entry: AnyEntry): FilePathWithPrefix { + return this.dependencies.getPath(entry); + } + + private _log(message: unknown, level?: LOG_LEVEL, key?: string): void { + this.dependencies.log(message, level, key); + } + + private _verbose(message: unknown, key?: string): void { + this._log(message, LOG_LEVEL_VERBOSE, key); + } + + private _progress(prefix: string = "", level: LOG_LEVEL = LOG_LEVEL_NOTICE): ReconciliationProgress { + return this.dependencies.createProgress(prefix, level); + } + + async processStorageChange( + path: FilePath, + onlyNew = false, + forceWrite = false, + includeDeleted = true + ): Promise { + if (!(await this.dependencies.isTargetFile(path))) { + this._log( + `Storage file tracking: Hidden file skipped: ${path} is filtered out by the defined patterns.`, + LOG_LEVEL_VERBOSE + ); + return false; + } + return await this.dependencies.changeProcessor.processStorageChange(path, onlyNew, forceWrite, includeDeleted); + } + + async processDatabaseDocument(doc: LoadedEntry): Promise { + const info = describeHiddenFileSyncDocument(doc, this.getPath(doc)); + const path = info.path; + const headerLine = `Tracking DB ${info.path} (${info.revDisplay}) :`; + const ret = await this.trackDatabaseFileModification(path, headerLine); + this._log(`${headerLine} Done: ${info.shortenedId})`, LOG_LEVEL_VERBOSE); + return ret; + } + + private async scanInternalFileNames(): Promise { + const findRoot = this.dependencies.getRootPath(); + + const filenames = await collectOptionalFileSyncFiles(this.dependencies, findRoot, { + shouldInclude: (path) => this.dependencies.isTargetFile(path as FilePath), + onError: (path, error) => { + this._log(`Could not traverse(HiddenSync):${path}`, LOG_LEVEL_INFO); + this._log(error, LOG_LEVEL_VERBOSE); + }, + }); + + return filenames as FilePath[]; + } + + async scanInternalFiles(): Promise { + const fileNames = await this.scanInternalFileNames(); + const files = fileNames.map(async (e) => { + return { + path: e, + stat: await this.storageAccess.statHidden(e), + }; + }); + const result: InternalFileInfo[] = []; + for (const f of files) { + const w = await f; + if (await this.dependencies.isIgnoredByIgnoreFile(w.path)) { + continue; + } + const mtime = w.stat?.mtime ?? 0; + const ctime = w.stat?.ctime ?? mtime; + const size = w.stat?.size ?? 0; + result.push({ + ...w, + mtime, + ctime, + size, + }); + } + return result; + } + + private async adoptCurrentStorageFilesAsProcessed(targetFiles: FilePath[] | false): Promise { + const allFiles = await this.scanInternalFileNames(); + const files = targetFiles ? allFiles.filter((e) => targetFiles.some((t) => e.indexOf(t) !== -1)) : allFiles; + for (const file of files) { + await this.dependencies.processedState.updateLastProcessedAsActualFile(file); + } + } + + private async adoptCurrentDatabaseFilesAsProcessed(targetFiles: FilePath[] | false): Promise { + const allFiles = await this.getAllDatabaseFiles(); + const files = targetFiles + ? allFiles.filter((e) => targetFiles.some((t) => e.path.indexOf(t) !== -1)) + : allFiles; + for (const file of files) { + const path = stripAllPrefixes(this.getPath(file)); + await this.dependencies.processedState.updateLastProcessedAsActualDatabase(path, file); + } + } + + private async trackScannedStorageChanges( + processFiles: FilePath[], + showNotice: boolean = false, + onlyNew = false, + forceWriteAll = false, + includeDeleted = true + ): Promise { + const logLevel = getLogLevel(showNotice); + const p = this._progress(`[⚙ Storage -> DB ]\n`, logLevel); + const notifyProgress = onlyInNTimes(100, (progress) => p.log(`${progress}/${processFiles.length}`)); + const processes = processFiles.map(async (file, i) => { + try { + await this.processStorageChange(file, onlyNew, forceWriteAll, includeDeleted); + notifyProgress(); + } catch (ex) { + p.once(`Failed to process storage change file:${file}`); + this._log(ex, LOG_LEVEL_VERBOSE); + } + }); + await Promise.all(processes); + p.done(); + } + + async scanAllStorageChanges( + showNotice: boolean = false, + onlyNew = false, + forceWriteAll = false, + includeDeleted = true + ): Promise { + return await skipIfDuplicated("scanAllStorageChanges", async () => { + const logLevel = getLogLevel(showNotice); + const p = this._progress(`[⚙ Scanning Storage -> DB ]\n`, logLevel); + p.log(`Scanning storage files...`); + const knownNames = [...this.dependencies.processedState.getLastProcessedFileKeys()] as FilePath[]; + const existNames = await this.scanInternalFileNames(); + const files = new Set([...knownNames, ...existNames]); + + this._log( + `Known/Exist ${knownNames.length}/${existNames.length}, Totally ${files.size} files.`, + LOG_LEVEL_VERBOSE + ); + const taskNameAndMeta = [...files].map(async (e) => [e, await this.storageAccess.statHidden(e)] as const); + const nameAndMeta = await Promise.all(taskNameAndMeta); + const processFiles = nameAndMeta + .filter(([path, stat]) => { + if (forceWriteAll) return true; + const key = this.dependencies.processedState.getLastProcessedFileKey(path); + const newKey = this.dependencies.processedState.storageStateKey(stat); + return key != newKey; + }) + .map(([path, stat]) => path); + + const staticsMessage = `[Storage hidden file statics] +Known files: ${knownNames.length} +Actual files: ${existNames.length} +All files: ${files.size} +Offline Changed files: ${processFiles.length}`; + // this._log(staticsMessage, logLevel, "scan-changes"); + p.once(staticsMessage); + await this.trackScannedStorageChanges(processFiles, showNotice, onlyNew, forceWriteAll, includeDeleted); + p.done(); + }); + } + + private async trackScannedDatabaseChange( + processFiles: MetaEntry[], + showNotice: boolean = false, + onlyNew = false, + forceWriteAll = false, + includeDeletion = true + ): Promise { + const logLevel = getLogLevel(showNotice); + const p = this._progress(`[⚙ DB -> Storage ]\n`, logLevel); + const notifyProgress = onlyInNTimes(100, (progress) => p.log(`${progress}/${processFiles.length}`)); + const processes = processFiles.map(async (file) => { + try { + const path = stripAllPrefixes(this.getPath(file)); + if (!(await this.dependencies.isTargetFile(path))) { + this._log( + `Database file tracking: Hidden file skipped: ${path} is filtered out by the defined patterns.`, + LOG_LEVEL_VERBOSE + ); + } else { + await this.trackDatabaseFileModification( + path, + "[Hidden file scan]", + !forceWriteAll, + onlyNew, + file, + includeDeletion + ); + } + notifyProgress(); + } catch (ex) { + this._log(`Failed to process storage change file:${tryGetFilePath(file)}`, logLevel); + this._log(ex, LOG_LEVEL_VERBOSE); + } + }); + await Promise.all(processes); + p.done(); + } + + async applyOfflineChanges(showNotice: boolean): Promise { + const logLevel = getLogLevel(showNotice); + return await serialized("applyOfflineChanges", async () => { + const p = this._progress("[⚙ Apply untracked changes ]\n", logLevel); + this._log(`Track changes.`, logLevel); + p.log("Enumerating local files..."); + const currentStorageFiles = await this.scanInternalFileNames(); + p.log("Enumerating database files..."); + const currentDatabaseFiles = await this.getAllDatabaseFiles(); + const allDatabaseMap = Object.fromEntries( + currentDatabaseFiles.map((e) => [stripAllPrefixes(this.getPath(e)), e]) + ); + const currentDatabaseFileNames = [...Object.keys(allDatabaseMap)] as FilePath[]; + const untrackedLocal = currentStorageFiles.filter( + (e) => !this.dependencies.processedState.hasLastProcessedFile(e) + ); + const untrackedDatabase = currentDatabaseFileNames.filter( + (e) => !this.dependencies.processedState.hasLastProcessedDatabase(e) + ); + const bothUntracked = untrackedLocal.filter((e) => untrackedDatabase.indexOf(e) !== -1); + p.log("Applying untracked changes..."); + const stat = `Tracking statics: +Local files: ${currentStorageFiles.length} +Database files: ${currentDatabaseFileNames.length} +Untracked local files: ${untrackedLocal.length} +Untracked database files: ${untrackedDatabase.length} +Common untracked files: ${bothUntracked.length}`; + p.once(stat); + const semaphores = Semaphore(10); + const notifyProgress = onlyInNTimes(25, (progress) => p.log(`${progress}/${bothUntracked.length}`)); + const allProcesses = bothUntracked.map(async (file) => { + notifyProgress(); + const rel = await semaphores.acquire(); + try { + const fileStat = await this.storageAccess.statHidden(file); + if (fileStat == null) { + // This should not be happened. But, if it happens, we should skip this. + this._log(`Unexpected error: Failed to stat file during applyOfflineChange :${file}`); + return; + } + const dbInfo = allDatabaseMap[file]; + if (dbInfo.deleted || dbInfo._deleted) { + // Applying deletion can be harmful if the local file is not tracked. + // So, we should skip this. + return; + } + const fileMTime = getHiddenFileSyncComparisonMTime(fileStat); + const dbMTime = getHiddenFileSyncComparisonMTime(dbInfo); + const diff = compareMTime(fileMTime, dbMTime); + if (diff == BASE_IS_NEW) { + // Local file is newer than the database file. + // So, we should apply the local file to the database. + await this.processStorageChange(file, true); + } else if (diff == TARGET_IS_NEW) { + // Database file is newer than the local file. + // So, we should apply the database file to the local file. + await this.trackDatabaseFileModification(file, "[Apply]", true, true, dbInfo); + } else if (diff == EVEN) { + // Both are same, we may skip this but should update the last processed key. + this.dependencies.processedState.updateLastProcessed(file, dbInfo, fileStat); + } + } finally { + rel(); + } + }); + await Promise.all(allProcesses); + await this.scanAllStorageChanges(showNotice); + await this.scanAllDatabaseChanges(showNotice); + + p.done(); + }); + } + + async scanAllDatabaseChanges( + showNotice: boolean = false, + onlyNew = false, + forceWriteAll = false, + includeDeletion = true + ): Promise { + return await skipIfDuplicated("scanAllDatabaseChanges", async () => { + const databaseFiles = await this.getAllDatabaseFiles(); + const files = databaseFiles.filter((e) => { + const doc = e; + const key = this.dependencies.processedState.databaseStateKey(doc); + const path = stripAllPrefixes(this.getPath(doc)); + const lastKey = this.dependencies.processedState.getLastProcessedDatabaseKey(path); + return lastKey != key; + }); + const logLevel = getLogLevel(showNotice); + const staticsMessage = `[Database hidden file statics] +All files: ${databaseFiles.length} +Offline Changed files: ${files.length}`; + this._log(staticsMessage, logLevel, "scan-changes"); + return await this.trackScannedDatabaseChange(files, showNotice, onlyNew, forceWriteAll, includeDeletion); + }); + } + + private async useDatabaseFiles(files: MetaEntry[], showNotice = false, onlyNew = false): Promise { + const logLevel = getLogLevel(showNotice); + const p = this._progress(`[⚙ Scanning DB -> Storage ]\n`, logLevel); + p.log("Scanning database files..."); + const notifyProgress = onlyInNTimes(25, (progress) => p.log(`${progress}/${files.length}`)); + const processFiles = files.map(async (file) => { + try { + const path = stripAllPrefixes(this.getPath(file)); + await this.trackDatabaseFileModification(path, "[Scanning]", true, onlyNew, file); + notifyProgress(); + } catch (ex) { + this._log(`Failed to process database changes:${tryGetFilePath(file)}`); + this._log(ex, LOG_LEVEL_VERBOSE); + } + return; + }); + await Promise.all(processFiles); + p.done(); + return true; + } + + private async trackDatabaseFileModification( + path: FilePath, + headerLine: string, + preventDoubleProcess = false, + onlyNew = false, + meta: MetaEntry | false = false, + includeDeletion = true + ): Promise { + return await this.dependencies.changeProcessor.processDatabaseChange(path, headerLine, { + preventDoubleProcess, + onlyNew, + metaEntry: meta, + includeDeletion, + }); + } + + private async rebuildMerging(showNotice: boolean, targetFiles: FilePath[] | false = false): Promise { + const logLevel = getLogLevel(showNotice); + const p = this._progress("[⚙ Rebuild by Merge ]\n", logLevel); + this._log(`Rebuilding hidden files from the storage and the local database.`, logLevel); + p.log("Enumerating local files..."); + const currentStorageFilesAll = await this.scanInternalFileNames(); + const currentStorageFiles = targetFiles + ? currentStorageFilesAll.filter((e) => targetFiles.some((f) => f == e)) + : currentStorageFilesAll; + p.log("Enumerating database files..."); + const allDatabaseFiles = await this.getAllDatabaseFiles(); + const allDatabaseMap = new Map(allDatabaseFiles.map((e) => [stripAllPrefixes(this.getPath(e)), e])); + const currentDatabaseFiles = targetFiles + ? allDatabaseFiles.filter((e) => targetFiles.some((f) => f == stripAllPrefixes(this.getPath(e)))) + : allDatabaseFiles; + + const allFileNames = new Set([ + ...currentStorageFiles, + ...currentDatabaseFiles.map((e) => stripAllPrefixes(this.getPath(e))), + ]); + const storageToDatabase = [] as FilePath[]; + const databaseToStorage = [] as MetaEntry[]; + + const eachProgress = onlyInNTimes(100, (progress) => p.log(`Checking ${progress}/${allFileNames.size}`)); + for (const file of allFileNames) { + eachProgress(); + const storageMTime = await this.storageAccess.statHidden(file); + const mtimeStorage = getHiddenFileSyncComparisonMTime(storageMTime); + const dbEntry = allDatabaseMap.get(file)!; + const mtimeDB = getHiddenFileSyncComparisonMTime(dbEntry); + const diff = compareMTime(mtimeStorage, mtimeDB); + if (diff == BASE_IS_NEW) { + storageToDatabase.push(file); + } else if (diff == TARGET_IS_NEW) { + databaseToStorage.push(dbEntry); + } else if (diff == EVEN) { + // For safety, storage to database. + storageToDatabase.push(file); + } + } + p.once( + `Storage to Database: ${storageToDatabase.length} files\n Database to Storage: ${databaseToStorage.length} files` + ); + this.dependencies.processedState.resetLastProcessedDatabase(targetFiles); + this.dependencies.processedState.resetLastProcessedFile(targetFiles); + const processes = [ + this.trackScannedStorageChanges(storageToDatabase, showNotice, false, true), + this.useDatabaseFiles(databaseToStorage, showNotice, false), + ]; + p.log("Start processing..."); + await Promise.all(processes); + p.done(); + return [...allFileNames]; + } + + private async runRebuildMerging(showNotice: boolean, targetFiles: FilePath[] | false = false): Promise { + return this.rebuildMergingHook + ? await this.rebuildMergingHook(showNotice, targetFiles) + : await this.rebuildMerging(showNotice, targetFiles); + } + + private async rebuildFromStorage( + showNotice: boolean, + targetFiles: FilePath[] | false = false, + onlyNew = false + ): Promise { + // reset processed file markers + const logLevel = getLogLevel(showNotice); + this._verbose(`Rebuilding hidden files from the storage.`); + this._log(`Rebuilding hidden files from the storage.`, logLevel); + const p = this._progress("[⚙ Rebuild by Storage ]\n", logLevel); + p.log("Enumerating local files..."); + const currentFilesAll = await this.scanInternalFileNames(); + const currentFiles = targetFiles + ? currentFilesAll.filter((e) => targetFiles.some((f) => f == e)) + : currentFilesAll; + p.once(`Storage to Database: ${currentFiles.length} files.`); + p.log("Start processing..."); + this.dependencies.processedState.resetLastProcessedFile(targetFiles); + await this.trackScannedStorageChanges(currentFiles, showNotice, onlyNew, true); + p.done(); + return currentFiles; + } + + private async getAllDatabaseFiles(): Promise { + const allFiles = ( + await this.localDatabase.allDocsRaw({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true }) + ).rows + .filter((e) => isInternalMetadata(e.id as DocumentID)) + .map((e) => e.doc) as MetaEntry[]; + const files = [] as MetaEntry[]; + for (const file of allFiles) { + if (await this.dependencies.isTargetFile(stripAllPrefixes(this.getPath(file)))) { + files.push(file); + } + } + return files; + } + + private async rebuildFromDatabase( + showNotice: boolean, + targetFiles: FilePath[] | false = false, + onlyNew = false + ): Promise { + const logLevel = getLogLevel(showNotice); + this._verbose(`Rebuilding hidden files from the local database.`); + this._log(`Rebuilding hidden files from the local database.`, logLevel); + const p = this._progress("[⚙ Rebuild by Database ]\n", logLevel); + p.log("Enumerating database files..."); + const allFiles = await this.getAllDatabaseFiles(); + + // THINKING: Should we exclude conflicted or deleted files? + // Current implementation is to include all files, and following processes will handle for them. + // However, in perspective of performance and future-proofing, I feel somewhat justified in doing it here. + + const currentFiles = targetFiles + ? allFiles.filter((e) => targetFiles.some((f) => f == stripAllPrefixes(this.getPath(e)))) + : allFiles; + + p.once(`Database to Storage: ${currentFiles.length} files.`); + this.dependencies.processedState.resetLastProcessedDatabase(targetFiles); + p.log("Start processing..."); + await this.useDatabaseFiles(currentFiles, showNotice, onlyNew); + p.done(); + return currentFiles; + } + + async initialiseInternalFileSync( + direction: InitialisationDirection, + showMessage: boolean, + // filesAll: InternalFileInfo[] | false = false, + targetFilesSrc: string[] | false = false, + initialisationProgress?: ReconciliationProgress + ): Promise { + const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; + const p = initialisationProgress ?? this._progress("[⚙ Initialise]\n", logLevel); + // Compatibility question: the legacy preflight was already disabled. + // Enabling it would change initialisation timing and could open a + // conflict dialogue while the feature is being configured. + // p.log("Resolving conflicts before starting..."); + // await this.conflictResolution.resolveAll(); + p.log("Initialising hidden files sync..."); + // The initialisation progress owns the user-visible Notice. Its child + // rebuild and scan operations still write ordinary log entries, but + // must not each create another keep-alive Notice. + const showChildNotices = false; + // TODO: Handling ignore files cannot be performed to the hidden files. + + const targetFiles = targetFilesSrc + ? targetFilesSrc.map((e) => stripAllPrefixes(e as FilePathWithPrefix)) + : false; + if (direction == "pushForce" || direction == "push") { + const onlyNew = direction == "push"; + p.log(`Started: Storage --> Database ${onlyNew ? "(Only New)" : ""}`); + const updatedFiles = await this.rebuildFromStorage(showChildNotices, targetFiles, onlyNew); + // making doubly sure, No more losing files. + // I did so many times during the development. + await this.adoptCurrentStorageFilesAsProcessed(updatedFiles); + await this.adoptCurrentDatabaseFilesAsProcessed(updatedFiles); + // And, scan other changes on the database (i.e. files which are on only other devices) + p.log("Checking for remaining storage and database changes..."); + await this.scanAllStorageChanges(showChildNotices, true, false); + await this.scanAllDatabaseChanges(showChildNotices, true, false); + } + if (direction == "pullForce" || direction == "pull") { + const onlyNew = direction == "pull"; + p.log(`Started: Database --> Storage ${onlyNew ? "(Only New)" : ""}`); + const updatedEntries = await this.rebuildFromDatabase(showChildNotices, targetFiles, onlyNew); + const updatedFiles = updatedEntries.map((e) => stripAllPrefixes(this.getPath(e))); + // making doubly sure, No more losing files. + await this.adoptCurrentStorageFilesAsProcessed(updatedFiles); + await this.adoptCurrentDatabaseFilesAsProcessed(updatedFiles); + // And, scan other changes on the database (i.e. files which are on only other devices) + p.log("Checking for remaining database and storage changes..."); + await this.scanAllDatabaseChanges(showChildNotices, true, false); + await this.scanAllStorageChanges(showChildNotices, true, false); + } + if (direction == "safe") { + p.log(`Started: Database <--> Storage (by modified date)`); + const updatedFiles = await this.runRebuildMerging(showChildNotices, targetFiles); + await this.adoptCurrentStorageFilesAsProcessed(updatedFiles); + await this.adoptCurrentDatabaseFilesAsProcessed(updatedFiles); + // And, scan other changes on the database (i.e. files which are on only other devices) + p.log("Checking for remaining storage and database changes..."); + await this.scanAllStorageChanges(showChildNotices, true, false); + await this.scanAllDatabaseChanges(showChildNotices, true, false); + } + p.done(); + } + + interceptRebuildMerging(interceptor: HiddenFileSyncTestingRebuildInterceptor): () => void { + const previousHook = this.rebuildMergingHook; + const runRebuild = async (showNotice: boolean, targetFiles?: FilePath[] | false) => + await this.rebuildMerging(showNotice, targetFiles); + const hook: HiddenFileSyncTestingRebuild = async (showNotice, targetFiles) => + await interceptor(runRebuild, showNotice, targetFiles); + this.rebuildMergingHook = hook; + return () => { + if (this.rebuildMergingHook === hook) { + this.rebuildMergingHook = previousHook; + } + }; + } + + dispose(): void { + this.rebuildMergingHook = undefined; + } +} + +export function createReconciliation(dependencies: ReconciliationDependencies): Reconciliation { + return new ReconciliationOwner(dependencies); +} diff --git a/src/features/HiddenFileSync/reconciliation.unit.spec.ts b/src/features/HiddenFileSync/reconciliation.unit.spec.ts new file mode 100644 index 00000000..0dde39a7 --- /dev/null +++ b/src/features/HiddenFileSync/reconciliation.unit.spec.ts @@ -0,0 +1,168 @@ +import { describe, expect, it, vi } from "vitest"; +import { + LOG_LEVEL_INFO, + LOG_LEVEL_NOTICE, + type FilePath, + type MetaEntry, + type UXStat, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; + +vi.mock("@/deps.ts", () => ({})); + +import { + createReconciliation, + type ReconciliationProgress, + type ReconciliationDependencies, +} from "./reconciliation.ts"; + +const targetPath = ".obsidian/app.json" as FilePath; +const filteredPath = ".obsidian/plugins/other/data.json" as FilePath; + +function createFixture(options: { files?: FilePath[]; databaseFiles?: MetaEntry[] } = {}) { + const files = options.files ?? [targetPath]; + const databaseFiles = options.databaseFiles ?? []; + const processedFiles = new Map(); + const progress = { + log: vi.fn(), + once: vi.fn(), + done: vi.fn(), + } satisfies ReconciliationProgress; + const createProgress = vi.fn((_prefix = "", _level = LOG_LEVEL_NOTICE) => progress); + const statHidden = vi.fn( + async (path: FilePath): Promise => ({ + ctime: 10, + mtime: path == targetPath ? 20 : 30, + size: 10, + type: "file", + }) + ); + const isTargetFile = vi.fn(async (path: FilePath) => path == targetPath); + const allDocsRaw = vi.fn(async () => ({ + rows: databaseFiles.map((doc) => ({ id: doc._id, doc })), + })); + const processedState = { + databaseStateKey: vi.fn((entry: MetaEntry) => `${entry._rev}`), + getLastProcessedDatabaseKey: vi.fn(() => undefined as string | undefined), + getLastProcessedFileKey: vi.fn(() => undefined as string | undefined), + getLastProcessedFileMTime: vi.fn(() => 0), + hasLastProcessedDatabase: vi.fn(() => false), + hasLastProcessedFile: vi.fn(() => false), + getLastProcessedFileKeys: vi.fn(() => processedFiles.keys()), + resetLastProcessedDatabase: vi.fn(), + resetLastProcessedFile: vi.fn(), + storageStateKey: vi.fn((stat: UXStat | null) => `${stat?.mtime ?? 0}`), + updateLastProcessed: vi.fn(), + updateLastProcessedAsActualDatabase: vi.fn(async () => undefined), + updateLastProcessedAsActualFile: vi.fn(async () => undefined), + }; + const changeProcessor = { + processStorageChange: vi.fn(async () => true), + processDatabaseChange: vi.fn(async () => true), + }; + const dependencies = { + listFiles: vi.fn(async () => ({ files, folders: [] })), + getLocalDatabase: () => ({ allDocsRaw }), + storageAccess: { statHidden }, + getRootPath: () => "root", + getPath: (entry: MetaEntry) => entry.path, + isTargetFile, + isIgnoredByIgnoreFile: vi.fn(async () => false), + createProgress, + processedState, + changeProcessor, + log: vi.fn(), + } as unknown as ReconciliationDependencies; + return { + dependencies, + progress, + createProgress, + isTargetFile, + statHidden, + processedState, + changeProcessor, + }; +} + +function metadata(path: FilePath = targetPath): MetaEntry { + return { + _id: `i:${path}`, + _rev: "2-current", + path: `i:${path}`, + type: "plain", + datatype: "plain", + ctime: 10, + mtime: 20, + size: 10, + children: [], + eden: {}, + deleted: false, + } as unknown as MetaEntry; +} + +describe("Reconciliation", () => { + it("keeps push initialisation direction and follow-up scan order", async () => { + const fixture = createFixture(); + const reconciliation = createReconciliation(fixture.dependencies); + const order: string[] = []; + const scanStorageChanges = vi.spyOn(reconciliation, "scanAllStorageChanges").mockImplementation(async () => { + order.push("storage-scan"); + }); + const scanDatabaseChanges = vi.spyOn(reconciliation, "scanAllDatabaseChanges").mockImplementation(async () => { + order.push("database-scan"); + }); + + await reconciliation.initialiseInternalFileSync("push", true); + + expect(fixture.changeProcessor.processStorageChange).toHaveBeenCalledWith(targetPath, true, true, true); + expect(order).toEqual(["storage-scan", "database-scan"]); + expect(scanStorageChanges).toHaveBeenCalledWith(false, true, false); + expect(scanDatabaseChanges).toHaveBeenCalledWith(false, true, false); + expect(fixture.createProgress).toHaveBeenCalledWith("[⚙ Initialise]\n", LOG_LEVEL_NOTICE); + expect(fixture.createProgress).toHaveBeenCalledWith("[⚙ Rebuild by Storage ]\n", LOG_LEVEL_INFO); + expect(fixture.progress.done).toHaveBeenCalledTimes(3); + }); + + it("restores rebuild interception in stack order without clobbering a newer hook", async () => { + const fixture = createFixture({ files: [] }); + const reconciliation = createReconciliation(fixture.dependencies); + const events: string[] = []; + const first = vi.fn(async (run, showNotice, targetFiles) => { + events.push("first:start"); + const result = await run(showNotice, targetFiles); + events.push("first:end"); + return result; + }); + const second = vi.fn(async (run, showNotice, targetFiles) => { + events.push("second:start"); + const result = await run(showNotice, targetFiles); + events.push("second:end"); + return result; + }); + const restoreFirst = reconciliation.interceptRebuildMerging(first); + const restoreSecond = reconciliation.interceptRebuildMerging(second); + + restoreFirst(); + await reconciliation.initialiseInternalFileSync("safe", false); + expect(second).toHaveBeenCalledOnce(); + + restoreSecond(); + await reconciliation.initialiseInternalFileSync("safe", false); + expect(first).toHaveBeenCalledOnce(); + expect(events[0]).toBe("second:start"); + expect(events[events.length - 1]).toBe("first:end"); + }); + + it("uses admission and processed-state keys when selecting storage scan work", async () => { + const fixture = createFixture({ files: [targetPath, filteredPath] }); + const reconciliation = createReconciliation(fixture.dependencies); + + await reconciliation.scanAllStorageChanges(false); + + expect(fixture.isTargetFile).toHaveBeenCalledWith(targetPath); + expect(fixture.isTargetFile).toHaveBeenCalledWith(filteredPath); + expect(fixture.processedState.getLastProcessedFileKey).toHaveBeenCalledWith(targetPath); + expect(fixture.changeProcessor.processStorageChange).toHaveBeenCalledWith(targetPath, false, false, true); + expect(fixture.changeProcessor.processStorageChange).not.toHaveBeenCalledWith(filteredPath, false, false, true); + expect(fixture.progress.once).toHaveBeenCalledWith(expect.stringContaining("Offline Changed files: 1")); + }); +}); diff --git a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts index ba31bf49..c6076528 100644 --- a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts +++ b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts @@ -202,7 +202,8 @@ async function autoMergeHiddenJsonConflict(cliBinary: string, env: NodeJS.Proces "return JSON.stringify({ok:true,path});", "})()", ].join(""), - env + env, + hiddenFileCliTimeoutMs ); } @@ -567,7 +568,9 @@ async function runMixedOwnership(context: RunnerContext, vault: TemporaryVault): "const hidden=plugin.optionalFileSync.testing.hiddenFileSync;", "core.services.setting.setDeviceAndVaultName('mixed-ownership');", "await customisation.scanAllConfigFiles(false);", - "await hidden.scanAllStorageChanges(false,false,true,true);", + // The testing view exposes the command-level one-argument scan. + // Extra internal scan flags passed here were previously ignored by the view wrapper. + "await hidden.scanAllStorageChanges(false);", "const customisationPaths=[];", "for await(const entry of core.localDatabase.findEntries('ix:','ix;')){customisationPaths.push(entry.path);}", "const hiddenPaths=[];",