From ca4e6d38b934f908caca47d93db67db3b163cdf1 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 1 Nov 2026 10:19:08 +0000 Subject: [PATCH] Expose NFC path identity across supported transports --- docs/adr/2026_10_unicode_path_identity.md | 4 + docs/design_docs/unicode_path_identity.md | 93 ++++- docs/recovery.md | 6 +- docs/settings.md | 26 +- package-lock.json | 16 +- package.json | 2 +- src/apps/cli/README.md | 10 +- .../messages/LiveSyncProvisionalMessages.ts | 3 + .../SettingDialogue/AdvancedSettingSpecs.ts | 13 - .../SettingDialogue/GeneralSettingSpecs.ts | 18 +- ...iveSyncSettingTab.declarative.unit.spec.ts | 18 +- .../ObsidianLiveSyncSettingTab.ts | 13 +- .../ObsidianLiveSyncSettingTab.unit.spec.ts | 28 +- .../SettingDialogue/PaneAdvanced.unit.spec.ts | 4 +- .../SettingDialogue/PaneGeneral.unit.spec.ts | 57 +++ .../SettingDialogue/SettingSpec.unit.spec.ts | 114 ++--- .../SettingsPageCatalogue.unit.spec.ts | 21 +- test/e2e-cli/test-p2p-sync.ts | 393 ++++++++++++------ .../runner/cliJournalMigration.ts | 243 +++++++++++ test/e2e-obsidian/runner/remoteActivity.ts | 25 +- test/e2e-obsidian/scripts/minio-upload.ts | 161 ++++++- updates.md | 3 + 22 files changed, 1003 insertions(+), 268 deletions(-) create mode 100644 src/modules/features/SettingDialogue/PaneGeneral.unit.spec.ts create mode 100644 test/e2e-obsidian/runner/cliJournalMigration.ts diff --git a/docs/adr/2026_10_unicode_path_identity.md b/docs/adr/2026_10_unicode_path_identity.md index 2bc5a036..69885922 100644 --- a/docs/adr/2026_10_unicode_path_identity.md +++ b/docs/adr/2026_10_unicode_path_identity.md @@ -13,6 +13,10 @@ Accepted on 2026-10-17. Unicode NFC path identity was released in Self-hosted LiveSync 1.0.38 with Commonlib 0.1.42. Local validation evidence and platform limits are recorded in the design document. +The maintained design now also describes an unreleased Journal and P2P +extension and the move to General Settings. The decision below retains the +initial CouchDB support boundary and original acceptance date. + ## Context Self-hosted LiveSync derives normal-file Metadata document IDs from Vault paths. diff --git a/docs/design_docs/unicode_path_identity.md b/docs/design_docs/unicode_path_identity.md index c5808f92..b03319e1 100644 --- a/docs/design_docs/unicode_path_identity.md +++ b/docs/design_docs/unicode_path_identity.md @@ -22,14 +22,18 @@ and is included in current Commonlib `0.2.2`, used by Self-hosted LiveSync `1.0.42`. This later handling is distinct from the initial NFC identity release. +An unreleased extension adds Journal and P2P for the same maintained CLI and +Obsidian hosts, and moves the setting to General Settings. The original dates +and reviewed versions above remain the historical baseline. + ## Scope | Area | Contract | | --- | --- | | Logical identity | Version `1` uses NFC for Metadata ID and path comparison keys. Existing case handling remains a separate rule. Missing and `0` retain legacy behaviour. | | Stored paths and content | Keep the spelling in `Metadata.path` and existing physical Vault names. NFC does not change Chunk IDs or content. | -| Initial hosts and remote | Support CLI and Obsidian with CouchDB. | -| Other transports | Journal and P2P reject version `1`; Direct file access refuses a remote declaring the NFC feature. Other host adapters remain outside the support matrix. | +| Maintained hosts and transports | CLI and Obsidian with CouchDB; Journal and P2P are added by the unreleased extension. | +| Other consumers | Direct file access refuses a remote declaring the NFC feature. Other host adapters remain outside the support matrix. | | Migration | Rebuild both databases from one complete authoritative Vault, then Fetch each remaining client. Do not rewrite IDs or merge histories in place. | ## Path identity contract @@ -133,8 +137,8 @@ check until a remote deletion has been replicated. ## Reconstruction and client operations Mode changes are never applied by an ordinary Metadata write. In Obsidian, the -Advanced page offers **Legacy path identity** (`0`) and **Unicode NFC path -identity** (`1`) for CouchDB. A changed `pathNormalizationVersion` remains +General Settings β†’ **File identity** offers **Legacy path identity** (`0`) +and **Unicode NFC path identity** (`1`) independently of the selected transport. A changed `pathNormalizationVersion` remains pending through ordinary settings saves, the settings-only Apply fallback, and reload. It becomes active only after the existing Rebuild or Fetch operation has been successfully reserved; cancellation or failed reservation leaves it @@ -144,7 +148,8 @@ the settings-only fallback is not offered. The [user migration procedure](../recovery.md#migrate-to-unicode-nfc-path-identity) owns the operational sequence: updated and stopped clients, a complete backed-up authoritative Vault, source Rebuild, then Fetch on each remaining client. The -source operation reconstructs both databases; a receiving client's Fetch +source operation reconstructs both databases for CouchDB and Journal. With +P2P, it reconstructs only the source database. A receiving client's Fetch replaces only its local database. The [CLI guide](../../src/apps/cli/README.md) provides the equivalent `rebuild` and `fetch` commands. @@ -190,11 +195,48 @@ updates and deletions. | --- | --- | --- | | CLI with CouchDB | Supported with legacy identity. | Supported with NFC path resolution. | | Obsidian with CouchDB | Supported with legacy identity. | Supported with NFC path resolution. | -| Journal or P2P | Existing behaviour. | Rejected. | +| CLI or Obsidian with Journal | Existing behaviour. | Added by the unreleased extension; Rebuild remote journals and Fetch other devices. | +| CLI or Obsidian with P2P | Existing behaviour. | Added by the unreleased extension; update and migrate every peer. | | Direct file access | Existing behaviour. | Refuses a remote requiring `nfc-path-identity-v1`. | | Other host adapters, including WebApp | Existing behaviour, subject to their existing support. | Outside the initial support matrix until physical-path handling is verified. | | Older client without this feature | Can use a legacy database. | Cannot join a database declaring `nfc-path-identity-v1`. | +## Journal and P2P extension + +The transport moves the Metadata IDs already generated in the local database; +it does not derive a second remote ID. Journal uses its existing compatibility +range with NFC value `4`, plus a required path-version comparison. A legacy +client with ordinary version checking cannot join the rebuilt Journal. The +explicit version override in older clients remains outside this guarantee. +Journal refuses to initialise NFC compatibility when journal objects exist +without their milestone. + +Journal shares one read-only identity assessment across ordinary admission, +Fetch before local reset, and ordinary milestone updates. It checks stored +node ranges even when a Rebuild lock has retired the previous accepted nodes. +Rebuild locks the existing milestone for its source node while retaining an +existing compatibility range, and declares the replacement identity only after +the remote journals have +been deleted. If the milestone is missing, the pre-reset lock does not create +one over the existing journals. A failed or interrupted reset therefore keeps +the old identity or the missing declaration. + +P2P compares the effective path version through its existing Tweak exchange, +checks local database identity, and reads the peer version document before +replication. Fresh control-only databases can declare the existing NFC feature. +Every participating peer must be updated and migrated; this is not a new +authentication protocol for already deployed clients. + +The existing Rebuild and Fetch paths own migration and state reset. Journal +reconstructs the local database and remote objects; P2P Rebuild reconstructs +only the source database. Other devices Fetch the source. Local reset clears +Journal KV checkpoints, while P2P uses its existing source mark and target +checkpoint. No new migration store, checkpoint format, or persistent admission +state is added. An explicit Journal bucket reset also clears the existing +transfer checkpoint when the bucket is already empty, so a remote-only Rebuild +resends the complete local database. P2P retains its existing peer selection +during Fetch. + ## Verification contract and limits The implementation acceptance matrix covers: @@ -211,8 +253,9 @@ The implementation acceptance matrix covers: - refusal to write when an existing legacy database has Metadata, refusal of version-`0` clients against an NFC database, and enforcement when ordinary mismatch warnings are disabled; -- URI round trips, checkpoint invalidation across path versions, and transport - rejection for Journal, P2P, and Direct file access; +- URI round trips, checkpoint invalidation across path versions, and + rejection for Direct file access; Journal and P2P acceptance and identity + mismatch checks belong to the unreleased extension; - refusal for ambiguous file or parent-directory candidates before local or remote reset, preserving both databases on rejection, within the paths returned by each host's supported listing; and @@ -251,6 +294,38 @@ Streaming Fast Fetch. Fast Fetch removed the obsolete Metadata and retained the current NFC document, its active state, and its stored path. The existing content, creation-time, physical-name, update, and deletion checks also passed. + +### Unreleased Journal and P2P validation + +The extension was checked on Linux with a locally packed Commonlib `0.2.3` +candidate and real Obsidian `1.12.7`. Publication remains pending. The checks +covered: + +- real S3-compatible Journal transfer with and without E2EE and Path + Obfuscation, including restart, logical rename and deletion, both conflict + leaves and their revision ancestry, and a full resend after bucket reset; +- real Obsidian and CLI Journal round trips with plain IDs and independent + encrypted IDs: Standard Fetch, identical Metadata IDs and Chunk content, + retention of a pre-existing NFD physical name, edits after CLI process + restart, and deletion reflected back into Obsidian; and +- real CLI P2P peers in legacy mode and NFC mode, with and without encryption + in NFC mode: canonical path reads, incompatible-peer refusal, reverse edits + after restart, retained original revision content, logical rename, and + deletion. Existing automatic-host, adapter-replacement, and explicit + disconnect scenarios also passed. + +The Journal migration scenarios also start with existing legacy CLI databases. +The source Rebuild changes its Metadata ID and declares compatibility range +`4`. A receiving client's legacy Fetch is refused while preserving its local +Metadata ID and revision and the remote milestone. After selecting NFC, its +Fetch converges to the source ID and content while preserving the NFD physical +path. Both plain and independently encrypted ID scenarios passed. + +P2P runtime evidence is limited to CLI peers; a real Obsidian P2P peer remains +unverified. The settings and reconstruction paths are covered by unit tests. +These checks do not extend NFC support to other hosts or establish native iOS +or macOS filesystem behaviour. + ## Non-goals This design does not: @@ -262,7 +337,7 @@ This design does not: - choose among ambiguous NFC-equivalent files or parents, or merge their content; - change the default away from legacy path identity; -- enable version `1` for Journal, P2P, Direct file access, WebApp, or an +- enable version `1` for Direct file access, WebApp, or an unverified host adapter; or - claim native iOS or macOS filesystem compatibility. diff --git a/docs/recovery.md b/docs/recovery.md index b73cf9fd..57942598 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -20,13 +20,13 @@ Do not switch transport, enable P2P, or run Garbage Collection as a substitute f ## Migrate to Unicode NFC path identity -Use this coordinated migration when CLI and Obsidian clients share a CouchDB database and composed and decomposed Unicode spellings should identify the same path. Version `0` remains the default and preserves legacy identity. Version `1` is supported with CouchDB only; it is unsupported with Object Storage and P2P. +Use this coordinated migration when CLI and Obsidian clients should identify composed and decomposed Unicode spellings as the same path. Version `0` remains the default and preserves legacy identity. CouchDB support is released; Object Storage and P2P support belong to the unreleased extension described in the [Unicode path identity design](design_docs/unicode_path_identity.md). This migration reconstructs the databases from one authoritative Vault. It does not rewrite document IDs in place or rename files. -1. Update every participating CLI and Obsidian client to a version which supports Unicode NFC path identity. Back up a complete authoritative Vault and the CouchDB database. Stop synchronisation and editing on every device until the migration is complete. +1. Update every participating CLI and Obsidian client to a version which supports Unicode NFC path identity. Back up a complete authoritative Vault and the selected remote database or Object Storage bucket. For P2P, preserve the participating devices because there is no central remote backup. Stop synchronisation and editing on every device until the migration is complete. 2. Choose the complete Vault which contains the intended files. Review any composed and decomposed names which would map to the same path identity, and resolve them before reconstruction. If Rebuild or Fetch reports an ambiguous path or Metadata ID, stop and resolve that collision before retrying; LiveSync does not choose or merge the files for you. -3. On the source client, select **Unicode NFC path identity** (`pathNormalizationVersion: 1`). In Obsidian, choose **Overwrite Server Data with This Device's Files**. In the CLI, set the same value in its settings and run `rebuild` with `--force --write-settings`. +3. On the source client, open **General Settings** β†’ **File identity** and select **Unicode NFC path identity** (`pathNormalizationVersion: 1`). In Obsidian, choose **Overwrite Server Data with This Device's Files**. In the CLI, set the same value in its settings and run `rebuild` with `--force --write-settings`. For Object Storage, this replaces the remote journals as well as the local database. For P2P, it prepares only the source device's local database; keep that device online for the other devices to Fetch. 4. On each remaining client, select **Unicode NFC path identity** (`pathNormalizationVersion: 1`) before fetching. In Obsidian, choose **Reset Synchronisation on This Device**. In the CLI, set the same value in its settings and run `fetch` with `--force --write-settings`. 5. Confirm that every client uses version `1` and that the expected files are present before resuming synchronisation. Do not reconnect a version `0` client to the migrated database. diff --git a/docs/settings.md b/docs/settings.md index e197d669..19e88101 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -236,6 +236,21 @@ Setting key: showVerboseLog Show verbose log. Please enable when you report the logs +### 3. File identity + +#### Unicode path identity + +Setting key: pathNormalizationVersion + +Version `0` preserves legacy file identity and is the default for existing and new settings. Version `1` uses Unicode NFC when deriving document IDs, so equivalent composed and decomposed paths identify the same file across the CLI and Obsidian. Existing physical file names are retained. CouchDB support is released. The unreleased extension also supports Object Storage and P2P for the maintained CLI and Obsidian hosts. This is a common local database setting, independently of the active transport. + +Version `1` requires a coordinated migration: set it on the source client before Rebuild, then set it on each remaining client before Fetch. Changing the setting alone does not migrate existing Metadata. Follow [Migrate to Unicode NFC path identity](recovery.md#migrate-to-unicode-nfc-path-identity) for the complete procedure. Do not resume a version `0` client against the migrated remote. + +The selected value stays pending through ordinary settings saves and reloads. It becomes active only when the Rebuild or Fetch operation is successfully reserved; cancellation leaves it pending. + +Distinct files or retained Metadata documents which derive the same identity stop reconstruction before either database is reset. When replacing the local database, Rebuild and Fetch exclude logical deletions without unresolved conflicts; a deleted winner with unresolved conflicts still participates in the check. This check uses the host's supported file listing; Obsidian checks the paths exposed by its Vault APIs and cannot detect physical aliases hidden by its index. Resolve the duplicate names or synchronisation scope first; Self-hosted LiveSync does not merge them automatically. Case-insensitive handling also applies the existing letter-case rule. + + ## 3. Remote Configuration ### 1. Connection settings @@ -1166,17 +1181,6 @@ When this setting is disabled, changing only the letter case of a file name with New Vaults use case-insensitive handling for cross-platform compatibility. Existing settings with an explicit value preserve that choice. Earlier releases also followed the case-insensitive branch when this value was absent, so 1.0 saves a missing legacy value as `false` without requiring a compatibility review or database rebuild. -#### Unicode path identity - -Setting key: pathNormalizationVersion - -Version `0` preserves legacy file identity and is the default for existing and new settings. Version `1` uses Unicode NFC when deriving document IDs, so equivalent composed and decomposed paths identify the same file across the CLI and Obsidian. Existing physical file names are retained. This mode currently supports CouchDB only. - -Version `1` requires a coordinated migration: set it on the source client before Rebuild, then set it on each remaining client before Fetch. Changing the setting alone does not migrate existing Metadata. Follow [Migrate to Unicode NFC path identity](recovery.md#migrate-to-unicode-nfc-path-identity) for the complete procedure. Do not resume a version `0` client against the migrated remote. - -The selected value stays pending through ordinary settings saves and reloads. It becomes active only when the Rebuild or Fetch operation is successfully reserved; cancellation leaves it pending. - -Distinct files or retained Metadata documents which derive the same identity stop reconstruction before either database is reset. When replacing the local database, Rebuild and Fetch exclude logical deletions without unresolved conflicts; a deleted winner with unresolved conflicts still participates in the check. This check uses the host's supported file listing; Obsidian checks the paths exposed by its Vault APIs and cannot detect physical aliases hidden by its index. Resolve the duplicate names or synchronisation scope first; Self-hosted LiveSync does not merge them automatically. Case-insensitive handling also applies the existing letter-case rule. ### 4. Compatibility (Internal API Usage) diff --git a/package-lock.json b/package-lock.json index bd8cd0f7..6407b1a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.2", + "@vrtmrz/livesync-commonlib": "file:_testdata/nfc/vrtmrz-livesync-commonlib-0.2.3-nfc-21c639f65430.tgz", "@vrtmrz/obsidian-plugin-kit": "0.1.6", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", @@ -4121,9 +4121,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.2.tgz", - "integrity": "sha512-+nl4ntmRtUmSUuh9mOpicE0U/wpzhSPSQQXy9eg/2xdTBL58jkrQ6bQjbVKEhqUj0bUvg0ESjZm8YX/x+4zXiA==", + "version": "0.2.3", + "resolved": "file:_testdata/nfc/vrtmrz-livesync-commonlib-0.2.3-nfc-21c639f65430.tgz", + "integrity": "sha512-xv+6GLjpmtJwY6hYOD7FIIqAOYEnjYuD5Vmwfj/+tAvKNdDKrvJCFW+veTXyWMs+MheEDg5wGCPHdddXL88XqA==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -4141,7 +4141,7 @@ "idb": "^8.0.3", "markdown-it": "^14.3.3", "minimatch": "^10.2.6", - "octagonal-wheels": "^0.1.55", + "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-idb": "^9.0.0", "pouchdb-adapter-indexeddb": "^9.0.0", @@ -8850,9 +8850,9 @@ "license": "MIT" }, "node_modules/octagonal-wheels": { - "version": "0.1.55", - "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.55.tgz", - "integrity": "sha512-mHsQv/h4Tw4ZGJCmTxXFlgeRvPuL9s47C0arD8Njqld+DKr3ppaWjVteEBP+9bWFSOpv/a1u2yY3LYNFGmuexg==", + "version": "0.1.56", + "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.56.tgz", + "integrity": "sha512-c4xQqdeEGE//ZExMJDEsqPsUHsRVHMNuxudux6scYI46eqA/54ao3WJmEZj3fbSsUhR7uCRmJ5ddd1cfRybIfg==", "license": "MIT", "dependencies": { "idb": "^8.0.3" diff --git a/package.json b/package.json index 29cd5e20..080b4489 100644 --- a/package.json +++ b/package.json @@ -206,7 +206,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.2", + "@vrtmrz/livesync-commonlib": "file:_testdata/nfc/vrtmrz-livesync-commonlib-0.2.3-nfc-21c639f65430.tgz", "@vrtmrz/obsidian-plugin-kit": "0.1.6", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index fb2a7827..79e0a590 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -382,7 +382,8 @@ The CLI uses the same settings format as the Obsidian plug-in. Create a `.livesy #### Unicode path identity The `pathNormalizationVersion` setting controls how file paths identify -Metadata documents when using CouchDB. An absent value or `0` keeps the legacy +Metadata documents. CouchDB support is released; the unreleased extension also +supports Object Storage and P2P. An absent value or `0` keeps the legacy identity. Version `1` uses Unicode Normalization Form C (NFC) for Metadata IDs and path comparison keys. It does not rewrite `Metadata.path`, rename existing files, or change Chunks or file content. `handleFilenameCaseSensitive` remains @@ -406,8 +407,11 @@ with `--force --write-settings`. Set the same value on each remaining CLI client, then run `fetch` with `--force --write-settings`. Use **Overwrite Server Data with This Device's Files** and **Reset Synchronisation on This Device** for the corresponding Obsidian Rebuild and Fetch operations. Resolve -any ambiguous names before reconstruction. Version `1` is supported with -CouchDB only; do not resume a legacy client against the rebuilt database. +any ambiguous names before reconstruction. Object Storage Rebuild replaces the +remote journals as well as the local database. P2P Rebuild prepares only the +source local database; keep that peer online for the other devices to Fetch. +Every P2P participant must be updated and migrated. Do not resume a legacy +client against the rebuilt database. ### Command-line Reference diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index f081cd13..4733ab44 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,9 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "File identity": "File identity", + "Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild from a complete authoritative Vault and Fetch on the other devices. With P2P, Rebuild prepares only the source device's local database. Resolve colliding file names first.": + "Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild from a complete authoritative Vault and Fetch on the other devices. With P2P, Rebuild prepares only the source device's local database. Resolve colliding file names first.", "Synchronisation protection": "Synchronisation protection", "Synchronising your Vault": "Synchronising your Vault", "Receiving changes from the server…": "Receiving changes from the server…", diff --git a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts index 59e09c17..e431bf53 100644 --- a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts +++ b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts @@ -1,5 +1,4 @@ import { ChunkAlgorithmNames } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import { $msg } from "@/common/translation"; import type { SettingSpecGroup } from "./SettingSpec.ts"; export type AdvancedSettingSpecContext = { @@ -24,18 +23,6 @@ export function createAdvancedSettingSpecGroups({ }, { key: "customChunkSize", control: { type: "number", min: 0, allowZero: true } }, { key: "strictContentCheckOnStartup", control: { type: "toggle" } }, - { - key: "pathNormalizationVersion", - holdValue: true, - control: { - type: "numeric-dropdown", - options: () => ({ - "0": $msg("Legacy path identity"), - "1": $msg("Unicode NFC path identity"), - }), - }, - visible: isCouchDB, - }, ], }, { diff --git a/src/modules/features/SettingDialogue/GeneralSettingSpecs.ts b/src/modules/features/SettingDialogue/GeneralSettingSpecs.ts index e3af3906..11befc23 100644 --- a/src/modules/features/SettingDialogue/GeneralSettingSpecs.ts +++ b/src/modules/features/SettingDialogue/GeneralSettingSpecs.ts @@ -8,7 +8,7 @@ export type GeneralSettingSpecContext = { showVerboseLog: () => boolean; }; -/** Build the shared Appearance and Logging controls. */ +/** Build the shared General Settings controls. */ export function createGeneralSettingSpecGroups({ showEditorStatusDetails, showVerboseLog, @@ -61,6 +61,22 @@ export function createGeneralSettingSpecGroups({ }, ], }, + { + heading: "File identity", + items: [ + { + key: "pathNormalizationVersion", + holdValue: true, + control: { + type: "numeric-dropdown", + options: () => ({ + "0": $msg("Legacy path identity"), + "1": $msg("Unicode NFC path identity"), + }), + }, + }, + ], + }, ]; } diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts index afe7986f..765b0b53 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts @@ -319,7 +319,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { ).toEqual(["Copy the current settings to a Setup URI", "Show QR code"]); }); - it("keeps Appearance, Logging, and Extra menus inside General Settings", () => { + it("keeps Appearance, Logging, File identity, and Extra menus inside General Settings", () => { const tab = createSettingsTab(); const definitions = tab.getSettingDefinitions(); const general = definitions.find( @@ -332,7 +332,19 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { (item): item is SettingDefinitionPage => isPage(item) && item.name === "🎚️ Extra menus" ); - expect(generalPages?.map(({ name }) => name)).toEqual(["🎨 Appearance", "πŸ“ Logging", "🎚️ Extra menus"]); + expect(generalPages?.map(({ name }) => name)).toEqual([ + "🎨 Appearance", + "πŸ“ Logging", + "πŸ—‚οΈ File identity", + "🎚️ Extra menus", + ]); + const fileIdentity = generalPages?.find(({ name }) => name === "πŸ—‚οΈ File identity"); + expect( + fileIdentity?.items?.flatMap((item) => ("control" in item && item.control ? [item.control.key] : [])) + ).toEqual(["pathNormalizationVersion"]); + expect( + fileIdentity?.items?.filter((item) => "action" in item && typeof item.action === "function") + ).toHaveLength(1); expect( appearance?.items?.flatMap((item) => ("control" in item && item.control ? [item.control.key] : [])) ).toEqual([ @@ -390,7 +402,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { const tab = createSettingsTab(); const pages = collectPages(tab.getSettingDefinitions()); - expect(pages).toHaveLength(14); + expect(pages).toHaveLength(15); expect(pages.map(({ name }) => name)).toEqual( expect.arrayContaining( createSettingsPageCatalogue() diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index e5680d8e..ec1beb99 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -892,9 +892,9 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { showEditorStatusDetails: () => this.isConfiguredAs("showStatusOnEditor", true), showVerboseLog: () => this.isConfiguredAs("lessInformationInLog", false), }); - const [appearance, logging] = groups; - if (!appearance || !logging) { - throw new Error("General settings must define Appearance and Logging groups"); + const [appearance, logging, fileIdentity] = groups; + if (!appearance || !logging || !fileIdentity) { + throw new Error("General settings must define Appearance, Logging, and File identity groups"); } return this.createRootGroup("general-settings", [ { @@ -907,6 +907,11 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { name: `πŸ“ ${logging.heading}`, items: logging.items, }, + { + type: "page", + name: `πŸ—‚οΈ ${fileIdentity.heading}`, + items: [this.createRebuildRequiredAction(), ...(fileIdentity.items ?? [])], + }, this.createExtraMenusPage(), ]); } @@ -1128,7 +1133,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { if (method === "remoteOnly" && this.isDirty("pathNormalizationVersion")) { Logger( $msg( - "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1." + "Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild from a complete authoritative Vault and Fetch on the other devices. With P2P, Rebuild prepares only the source device's local database. Resolve colliding file names first." ), LOG_LEVEL_NOTICE ); diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts index df8f6b23..4db8f99a 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts @@ -581,11 +581,11 @@ describe("ObsidianLiveSyncSettingTab pending-setting initialisation", () => { expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); expect(tab.isDirty("pathNormalizationVersion")).toBe(true); expect(vi.mocked($msg)).toHaveBeenCalledWith( - "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1." + "Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild from a complete authoritative Vault and Fetch on the other devices. With P2P, Rebuild prepares only the source device's local database. Resolve colliding file names first." ); expect(loggerMocks.Logger).toHaveBeenCalledWith( expect.stringContaining( - "Rebuild both databases from a complete authoritative Vault and Fetch on the other devices." + "Rebuild from a complete authoritative Vault and Fetch on the other devices." ), LOG_LEVEL_NOTICE ); @@ -692,6 +692,30 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { return { tab, saveSettingData }; } + it("includes path identity once in the declarative General settings group", () => { + const { tab } = createSettingsTab(); + const group = ( + tab as unknown as { + createGeneralSettingsGroup: () => { + items: Array<{ + name: string; + items?: Array<{ control?: { key: string }; action?: () => void; visible?: () => boolean }>; + }>; + }; + } + ).createGeneralSettingsGroup(); + const fileIdentity = group.items.find(({ name }) => name === "πŸ—‚οΈ File identity"); + + expect(fileIdentity).toBeDefined(); + const apply = fileIdentity?.items?.find(({ action }) => typeof action === "function"); + expect(apply).toBeDefined(); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + expect(apply?.visible?.()).toBe(true); + expect(fileIdentity?.items?.flatMap(({ control }) => (control ? [control.key] : []))).toEqual([ + "pathNormalizationVersion", + ]); + }); + it("awaits the stored passphrase before initialising the editing buffer", async () => { const { tab } = createSettingsTab(); tab.resetEditingSettings(); diff --git a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts index 85597fbf..5b50cba4 100644 --- a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts +++ b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts @@ -43,7 +43,7 @@ describe("paneAdvanced", () => { }; paneAdvanced.call(host as never, {} as HTMLElement, { addPanel } as never); - await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(11)); + await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(10)); expect(addPanel.mock.calls.map(([, heading]) => heading)).toEqual([ "Memory cache", @@ -56,7 +56,6 @@ describe("paneAdvanced", () => { panelElements.get("Local Database Tweak"), panelElements.get("Local Database Tweak"), panelElements.get("Local Database Tweak"), - panelElements.get("Local Database Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), @@ -69,7 +68,6 @@ describe("paneAdvanced", () => { "chunkSplitterVersion", "customChunkSize", "strictContentCheckOnStartup", - "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", diff --git a/src/modules/features/SettingDialogue/PaneGeneral.unit.spec.ts b/src/modules/features/SettingDialogue/PaneGeneral.unit.spec.ts new file mode 100644 index 00000000..73582d49 --- /dev/null +++ b/src/modules/features/SettingDialogue/PaneGeneral.unit.spec.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { paneGeneral } from "./PaneGeneral.ts"; + +const settingHarness = vi.hoisted(() => ({ + rendered: [] as unknown[], +})); + +vi.mock("@/common/translation", () => ({ + $msg: (key: string) => key, + $t: (key: string) => key, +})); + +vi.mock("./LiveSyncSetting.ts", () => ({ + LiveSyncSetting: class LiveSyncSetting {}, +})); + +vi.mock("./SettingSpec.ts", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + renderLegacySettingSpec: vi.fn((_renderer: unknown, spec: unknown) => { + settingHarness.rendered.push(spec); + }), + }; +}); + +afterEach(() => { + settingHarness.rendered.length = 0; + vi.clearAllMocks(); +}); + +describe("paneGeneral", () => { + it("renders path identity once in the File identity panel and preserves its held value", async () => { + const addPanel = vi.fn((_parent: HTMLElement, heading: string) => + Promise.resolve({ heading } as unknown as HTMLElement) + ); + const host = { + isConfiguredAs: vi.fn(() => false), + }; + + paneGeneral.call(host as never, {} as HTMLElement, { addPanel } as never); + await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(14)); + + expect(addPanel.mock.calls.map(([, heading]) => heading)).toEqual([ + "obsidianLiveSyncSettingTab.titleAppearance", + "obsidianLiveSyncSettingTab.titleLogging", + "File identity", + "obsidianLiveSyncSettingTab.titleExtraMenus", + ]); + const pathIdentity = settingHarness.rendered.filter( + (spec) => (spec as { key: string }).key === "pathNormalizationVersion" + ); + expect(pathIdentity).toHaveLength(1); + expect(pathIdentity[0]).toMatchObject({ holdValue: true }); + expect((pathIdentity[0] as { visible?: () => boolean }).visible).toBeUndefined(); + }); +}); diff --git a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts index 0261a2da..214d7892 100644 --- a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts @@ -17,7 +17,7 @@ const rangeMessages = { }; describe("Advanced setting specifications", () => { - it("lists only the eleven controls currently exposed by the Advanced page", () => { + it("lists only the ten controls currently exposed by the Advanced page", () => { const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => true }); expect(groups.map(({ heading }) => heading)).toEqual([ @@ -31,7 +31,6 @@ describe("Advanced setting specifications", () => { "chunkSplitterVersion", "customChunkSize", "strictContentCheckOnStartup", - "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", @@ -52,13 +51,12 @@ describe("Advanced setting specifications", () => { expect(spec.disabled).toBeUndefined(); }); - it("keeps path identity and the four remote chunk controls CouchDB-only", () => { + it("keeps the four remote chunk controls CouchDB-only", () => { let couchDB = false; const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => couchDB }); const conditional = groups.flatMap(({ items }) => items).filter(({ visible }) => visible !== undefined); expect(conditional.map(({ key }) => key)).toEqual([ - "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", @@ -70,18 +68,67 @@ describe("Advanced setting specifications", () => { expect(conditional.every(({ visible }) => visible?.() === true)).toBe(true); }); - it("exposes CouchDB path identity as a two-choice numeric dropdown", () => { - let couchDB = false; - const spec = createAdvancedSettingSpecGroups({ isCouchDB: () => couchDB }) + it("excludes transient settings-dialogue keys from the persisted key type", () => { + type OnDialogOverlap = Extract; + expectTypeOf().toEqualTypeOf(); + }); + + it("resolves translated Commonlib metadata for every explicitly exposed key", () => { + const specs = [ + ...createAdvancedSettingSpecGroups({ isCouchDB: () => true }), + ...createGeneralSettingSpecGroups({ + showEditorStatusDetails: () => true, + showVerboseLog: () => true, + }), + ].flatMap(({ items }) => items); + + for (const spec of specs) { + const metadata = getConfig(spec.key); + expect(metadata, spec.key).not.toBe(false); + if (!metadata) { + throw new Error(`Missing setting metadata for ${spec.key}`); + } + const definition = toObsidianSettingDefinition(spec, metadata, rangeMessages); + expect(definition.name).toBe(`${metadata.name}${statusDisplay(metadata.status)}`); + } + }); +}); + +describe("General setting specifications", () => { + it("shares every General, Logging, and File identity control between the imperative and declarative renderers", () => { + const groups = createGeneralSettingSpecGroups({ + showEditorStatusDetails: () => true, + showVerboseLog: () => true, + }); + + expect(groups.map(({ heading }) => heading)).toEqual(["Appearance", "Logging", "File identity"]); + expect(groups.flatMap(({ items }) => items.map(({ key }) => key))).toEqual([ + "displayLanguage", + "showStatusOnEditor", + "showOnlyIconsOnEditor", + "showStatusOnStatusbar", + "hideFileWarningNotice", + "syncGuardEnabled", + "syncGuardMinimumFiles", + "networkWarningStyle", + "lessInformationInLog", + "showVerboseLog", + "pathNormalizationVersion", + ]); + }); + + it("exposes path identity for every remote type as a reconstruction-required dropdown", () => { + const spec = createGeneralSettingSpecGroups({ + showEditorStatusDetails: () => false, + showVerboseLog: () => false, + }) .flatMap(({ items }) => items) .find(({ key }) => key === "pathNormalizationVersion"); - if (!spec) throw new Error("Path identity mode is not exposed on the Advanced page"); + if (!spec) throw new Error("Path identity mode is not exposed in General settings"); if (spec.control.type !== "numeric-dropdown") throw new Error("Expected a numeric dropdown"); - expect(spec.control).toMatchObject({ type: "numeric-dropdown", options: expect.any(Function) }); - expect(spec.visible?.()).toBe(false); - couchDB = true; - expect(spec.visible?.()).toBe(true); + expect(spec).toMatchObject({ key: "pathNormalizationVersion", holdValue: true }); + expect(spec.visible).toBeUndefined(); expect(spec.control.options()).toEqual({ "0": "Legacy path identity", "1": "Unicode NFC path identity", @@ -102,54 +149,11 @@ describe("Advanced setting specifications", () => { key: "pathNormalizationVersion", options: { holdValue: true, - onUpdate: expect.any(Function), options: { "0": "Legacy path identity", "1": "Unicode NFC path identity" }, }, }); }); - it("excludes transient settings-dialogue keys from the persisted key type", () => { - type OnDialogOverlap = Extract; - expectTypeOf().toEqualTypeOf(); - }); - - it("resolves translated Commonlib metadata for every explicitly exposed key", () => { - const specs = createAdvancedSettingSpecGroups({ isCouchDB: () => true }).flatMap(({ items }) => items); - - for (const spec of specs) { - const metadata = getConfig(spec.key); - expect(metadata, spec.key).not.toBe(false); - if (!metadata) { - throw new Error(`Missing setting metadata for ${spec.key}`); - } - const definition = toObsidianSettingDefinition(spec, metadata, rangeMessages); - expect(definition.name).toBe(`${metadata.name}${statusDisplay(metadata.status)}`); - } - }); -}); - -describe("General setting specifications", () => { - it("shares every General and Logging control between the imperative and declarative renderers", () => { - const groups = createGeneralSettingSpecGroups({ - showEditorStatusDetails: () => true, - showVerboseLog: () => true, - }); - - expect(groups.map(({ heading }) => heading)).toEqual(["Appearance", "Logging"]); - expect(groups.flatMap(({ items }) => items.map(({ key }) => key))).toEqual([ - "displayLanguage", - "showStatusOnEditor", - "showOnlyIconsOnEditor", - "showStatusOnStatusbar", - "hideFileWarningNotice", - "syncGuardEnabled", - "syncGuardMinimumFiles", - "networkWarningStyle", - "lessInformationInLog", - "showVerboseLog", - ]); - }); - it("keeps the three feature-level controls together under Extra menus", () => { const group = createExtraMenuSettingSpecGroup(); diff --git a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts index a2605f41..002bc174 100644 --- a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; vi.mock("@/common/translation", () => ({ $msg: (key: string) => key, + $t: (key: string) => key, translateLiveSyncMessage: (key: string) => key, })); vi.mock("./PaneChangeLog.ts", () => ({ paneChangeLog: vi.fn() })); @@ -20,6 +21,7 @@ vi.mock("./PaneMaintenance.ts", () => ({ paneMaintenance: vi.fn() })); import { createAdvancedSettingDefinitionGroups, + createGeneralSettingDefinitionGroups, createSettingsPageCatalogue, getSettingsRootGroupEntry, type SettingsRootGroupId, @@ -87,7 +89,6 @@ describe("settings page catalogue", () => { "chunkSplitterVersion", "customChunkSize", "strictContentCheckOnStartup", - "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", @@ -97,4 +98,22 @@ describe("settings page catalogue", () => { ]); expect(new Set(keys).size).toBe(keys.length); }); + + it("registers path identity once in General settings and leaves it out of Advanced", () => { + const generalGroups = createGeneralSettingDefinitionGroups({ + showEditorStatusDetails: () => true, + showVerboseLog: () => true, + }); + const generalKeys = generalGroups.flatMap(({ items = [] }) => + items.flatMap((item) => ("control" in item && item.control ? [item.control.key] : [])) + ); + const advancedKeys = createAdvancedSettingDefinitionGroups({ isCouchDB: () => true }).flatMap( + ({ items = [] }) => + items.flatMap((item) => ("control" in item && item.control ? [item.control.key] : [])) + ); + + expect(generalGroups.map(({ heading }) => heading)).toContain("File identity"); + expect(generalKeys.filter((key) => key === "pathNormalizationVersion")).toHaveLength(1); + expect(advancedKeys).not.toContain("pathNormalizationVersion"); + }); }); diff --git a/test/e2e-cli/test-p2p-sync.ts b/test/e2e-cli/test-p2p-sync.ts index 0392ef6f..b0ac1718 100644 --- a/test/e2e-cli/test-p2p-sync.ts +++ b/test/e2e-cli/test-p2p-sync.ts @@ -19,152 +19,271 @@ import { import { runCli, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; import { getOptimalLoopbackIp } from "./helpers/net.ts"; -test("p2p-sync: transfers with the same ID key and rejects a different document ID key", async () => { - const loopbackIp = await getOptimalLoopbackIp(); - const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; +for (const [pathNormalizationVersion, encrypted] of [ + [0, true], + [1, true], + [1, false], +] as const) { + test(`p2p-sync: identity mode ${pathNormalizationVersion}, encryption ${encrypted} transfers matching IDs and rejects incompatible peers`, async () => { + const loopbackIp = await getOptimalLoopbackIp(); + const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; - const relay = process.env["RELAY"] ?? `ws://${loopbackHost}:4000/`; - const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`; - const passphrase = process.env["PASSPHRASE"] ?? "test"; - const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "12"); - const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "15"); - const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; - const hostPeerName = process.env["HOST_PEER_NAME"] ?? `p2p-host-${nonce}`; - const clientPeerName = process.env["CLIENT_PEER_NAME"] ?? `p2p-client-${nonce}`; - const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0"; - const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); + const relay = process.env["RELAY"] ?? `ws://${loopbackHost}:4000/`; + const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`; + const passphrase = process.env["PASSPHRASE"] ?? "test"; + const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "12"); + const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "15"); + const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; + const hostPeerName = process.env["HOST_PEER_NAME"] ?? `p2p-host-${nonce}`; + const clientPeerName = process.env["CLIENT_PEER_NAME"] ?? `p2p-client-${nonce}`; + const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0"; + const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); - await using workDir = await TempDir.create("livesync-cli-p2p-sync"); - const hostVault = workDir.join("vault-host"); - const hostSettings = workDir.join("settings-host.json"); - const clientVault = workDir.join("vault-sync"); - const clientSettings = workDir.join("settings-sync.json"); - const rejectedVault = workDir.join("vault-rejected"); - const rejectedSettings = workDir.join("settings-rejected.json"); - await mkdir(hostVault, { recursive: true }); - await mkdir(clientVault, { recursive: true }); - await mkdir(rejectedVault, { recursive: true }); + await using workDir = await TempDir.create("livesync-cli-p2p-sync"); + const hostVault = workDir.join("vault-host"); + const hostSettings = workDir.join("settings-host.json"); + const clientVault = workDir.join("vault-sync"); + const clientSettings = workDir.join("settings-sync.json"); + const rejectedVault = workDir.join("vault-rejected"); + const rejectedSettings = workDir.join("settings-rejected.json"); + await mkdir(hostVault, { recursive: true }); + await mkdir(clientVault, { recursive: true }); + await mkdir(rejectedVault, { recursive: true }); - const relayStarted = await maybeStartLocalRelay(relay); - const coturnStarted = await maybeStartCoturn(turnServers); - try { - await initSettingsFile(hostSettings); - await initSettingsFile(clientSettings); - await initSettingsFile(rejectedSettings); - await applyP2pSettings( - hostSettings, - roomId, - passphrase, - "self-hosted-livesync-cli-tests", - relay, - "~.*", - turnServers - ); - await applyP2pSettings( - clientSettings, - roomId, - passphrase, - "self-hosted-livesync-cli-tests", - relay, - "~.*", - turnServers - ); - await applyP2pSettings( - rejectedSettings, - roomId, - passphrase, - "self-hosted-livesync-cli-tests", - relay, - "~.*", - turnServers - ); - await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase); - await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase); - await applyP2pTestTweaks(rejectedSettings, "p2p-rejected-" + nonce, passphrase); - for (const [vault, path, key, label] of [ - [hostVault, hostSettings, "ab".repeat(32), "host"], - [clientVault, clientSettings, "ab".repeat(32), "client"], - [rejectedVault, rejectedSettings, "cd".repeat(32), "rejected"], - ]) { - const settings = JSON.parse(await readFile(path, "utf8")); - settings.idDerivationVersion = 1; - settings.idDerivationKey = key; - const sourcePath = workDir.join("setup-source-" + label + ".json"); - await writeFile(sourcePath, JSON.stringify(settings)); - const setupPassphrase = "independent-id-setup-passphrase"; - const setupUri = await generateSetupUriFromSettings(sourcePath, setupPassphrase, true); - await runCliWithInputOrFail(setupPassphrase + "\n", vault, "--settings", path, "setup", setupUri); - const persisted = JSON.parse(await readFile(path, "utf8")); - assert(persisted.idDerivationVersion === 1, "The Setup URI lost the ID derivation version."); - assert(persisted.idDerivationKey === "", "The CLI stored the ID key in plain text."); - assert( - typeof persisted.encryptedIdDerivationKey === "string" && persisted.encryptedIdDerivationKey.length > 0, - "The CLI did not encrypt the saved ID key." - ); - assert(persisted.P2P_Enabled === true, "The Setup URI disabled P2P."); - assert(persisted.P2P_roomID === roomId, "The Setup URI changed the P2P room."); - assert(persisted.P2P_relays === relay, "The Setup URI changed the P2P relay."); - assert(persisted.remoteType === "ONLY_P2P", "The Setup URI changed the remote type."); - } - const notePath = "p2p/independent-id-note.md"; - await runCliWithInputOrFail( - "A note transferred with the saved ID key.\n", - clientVault, - "--settings", - clientSettings, - "put", - notePath - ); - - const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host"); + const relayStarted = await maybeStartLocalRelay(relay); + const coturnStarted = await maybeStartCoturn(turnServers); try { - await host.waitUntilContains("P2P host is running", 20000); - const peer = await discoverPeer( - clientVault, + await initSettingsFile(hostSettings); + await initSettingsFile(clientSettings); + await initSettingsFile(rejectedSettings); + await applyP2pSettings( + hostSettings, + roomId, + passphrase, + "self-hosted-livesync-cli-tests", + relay, + "~.*", + turnServers + ); + await applyP2pSettings( clientSettings, - peersTimeout, - process.env["TARGET_PEER"] ?? hostPeerName + roomId, + passphrase, + "self-hosted-livesync-cli-tests", + relay, + "~.*", + turnServers ); - const syncResult = await runCli( - clientVault, - "--settings", - clientSettings, - "p2p-sync", - peer.id, - String(syncTimeout) - ); - assert( - syncResult.code === 0, - `p2p-sync failed\nstdout: ${syncResult.stdout}\nstderr: ${syncResult.stderr}` - ); - const rejectedPeer = await discoverPeer(rejectedVault, rejectedSettings, peersTimeout, hostPeerName); - const rejectedSync = await runCli( - rejectedVault, - "--settings", + await applyP2pSettings( rejectedSettings, - "p2p-sync", - rejectedPeer.id, - String(syncTimeout) + roomId, + passphrase, + "self-hosted-livesync-cli-tests", + relay, + "~.*", + turnServers ); + await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase); + await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase); + await applyP2pTestTweaks(rejectedSettings, "p2p-rejected-" + nonce, passphrase); + for (const [vault, path, key, label] of [ + [hostVault, hostSettings, "ab".repeat(32), "host"], + [clientVault, clientSettings, "ab".repeat(32), "client"], + [ + rejectedVault, + rejectedSettings, + (pathNormalizationVersion === 1 ? "ab" : "cd").repeat(32), + "rejected", + ], + ]) { + const settings = JSON.parse(await readFile(path, "utf8")); + settings.pathNormalizationVersion = + pathNormalizationVersion === 1 && label === "rejected" ? 0 : pathNormalizationVersion; + settings.encrypt = encrypted; + settings.usePathObfuscation = encrypted; + settings.idDerivationVersion = 1; + settings.idDerivationKey = key; + const sourcePath = workDir.join("setup-source-" + label + ".json"); + await writeFile(sourcePath, JSON.stringify(settings)); + const setupPassphrase = "independent-id-setup-passphrase"; + const setupUri = await generateSetupUriFromSettings(sourcePath, setupPassphrase, true); + await runCliWithInputOrFail(setupPassphrase + "\n", vault, "--settings", path, "setup", setupUri); + const persisted = JSON.parse(await readFile(path, "utf8")); + assert(persisted.idDerivationVersion === 1, "The Setup URI lost the ID derivation version."); + assert( + persisted.pathNormalizationVersion === settings.pathNormalizationVersion, + "The Setup URI lost the path identity version." + ); + assert(persisted.idDerivationKey === "", "The CLI stored the ID key in plain text."); + assert( + typeof persisted.encryptedIdDerivationKey === "string" && + persisted.encryptedIdDerivationKey.length > 0, + "The CLI did not encrypt the saved ID key." + ); + assert(persisted.P2P_Enabled === true, "The Setup URI disabled P2P."); + assert(persisted.P2P_roomID === roomId, "The Setup URI changed the P2P room."); + assert(persisted.P2P_relays === relay, "The Setup URI changed the P2P relay."); + assert(persisted.remoteType === "ONLY_P2P", "The Setup URI changed the remote type."); + } + const notePath = + pathNormalizationVersion === 1 ? "Cafe\u0301/re\u0301sume\u0301.md" : "p2p/independent-id-note.md"; + const receivedPath = notePath.normalize("NFC"); + await runCliWithInputOrFail( + "A note transferred with the saved ID key.\n", + clientVault, + "--settings", + clientSettings, + "put", + notePath + ); + + const originalInfo = JSON.parse( + await runCliOrFail(clientVault, "--settings", clientSettings, "info", notePath) + ) as { id: string; revision: string }; + if (pathNormalizationVersion === 1) { + assert.equal( + sanitiseCatStdout( + await runCliOrFail(clientVault, "--settings", clientSettings, "cat", receivedPath) + ), + "A note transferred with the saved ID key.\n", + "The NFC spelling did not address the NFD Metadata." + ); + } + const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host"); + try { + await host.waitUntilContains("P2P host is running", 20000); + const peer = await discoverPeer( + clientVault, + clientSettings, + peersTimeout, + process.env["TARGET_PEER"] ?? hostPeerName + ); + const syncResult = await runCli( + clientVault, + "--settings", + clientSettings, + "p2p-sync", + peer.id, + String(syncTimeout) + ); + assert( + syncResult.code === 0, + `p2p-sync failed\nstdout: ${syncResult.stdout}\nstderr: ${syncResult.stderr}` + ); + const rejectedPeer = await discoverPeer(rejectedVault, rejectedSettings, peersTimeout, hostPeerName); + const rejectedSync = await runCli( + rejectedVault, + "--settings", + rejectedSettings, + "p2p-sync", + rejectedPeer.id, + String(syncTimeout) + ); + assert( + rejectedSync.code !== 0, + `P2P accepted incompatible peer settings.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}` + ); + + assert( + rejectedSync.combined.includes("Tweak values are not matched"), + `P2P failed before checking peer settings.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}` + ); + } finally { + await host.stop(); + } + const received = sanitiseCatStdout( + await runCliOrFail(hostVault, "--settings", hostSettings, "cat", receivedPath) + ).trimEnd(); assert( - rejectedSync.code !== 0, - `P2P accepted a different key for obfuscated document IDs.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}` - ); - assert( - rejectedSync.combined.includes("Tweak values are not matched"), - `P2P failed before checking peer settings.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}` + received === "A note transferred with the saved ID key.", + "The host did not receive the keyed note." ); + const rejectedRead = await runCli(rejectedVault, "--settings", rejectedSettings, "cat", notePath); + assert(rejectedRead.code !== 0, "The rejected device received the keyed note."); + if (pathNormalizationVersion === 1) { + const canonicalRead = await runCli(rejectedVault, "--settings", rejectedSettings, "cat", receivedPath); + assert( + canonicalRead.code !== 0, + "The rejected device received canonical Metadata despite its different identity mode." + ); + } + if (pathNormalizationVersion === 1) { + const exchange = async ( + senderVault: string, + senderSettings: string, + receiverVault: string, + receiverSettings: string, + receiverName: string + ) => { + const restartedHost = startCliInBackground( + receiverVault, + "--settings", + receiverSettings, + "p2p-host" + ); + try { + await restartedHost.waitUntilContains("P2P host is running", 20000); + const peer = await discoverPeer(senderVault, senderSettings, peersTimeout, receiverName); + await runCliOrFail( + senderVault, + "--settings", + senderSettings, + "p2p-sync", + peer.id, + String(syncTimeout) + ); + } finally { + await restartedHost.stop(); + } + }; + const editedBody = "Edited through the NFC spelling after process restart.\n"; + await runCliWithInputOrFail(editedBody, hostVault, "--settings", hostSettings, "put", receivedPath); + await exchange(hostVault, hostSettings, clientVault, clientSettings, clientPeerName); + assert.equal( + sanitiseCatStdout(await runCliOrFail(clientVault, "--settings", clientSettings, "cat", notePath)), + editedBody + ); + const storedPath = (await runCliOrFail(clientVault, "--settings", clientSettings, "ls")) + .trim() + .split("\t")[0]; + assert.equal(storedPath.normalize("NFC"), receivedPath); + const editedInfo = JSON.parse( + await runCliOrFail(clientVault, "--settings", clientSettings, "info", storedPath) + ) as { id: string }; + assert.equal(editedInfo.id, originalInfo.id, "The reverse edit changed the canonical Metadata ID."); + assert.equal( + sanitiseCatStdout( + await runCliOrFail( + clientVault, + "--settings", + clientSettings, + "cat-rev", + notePath, + originalInfo.revision + ) + ), + "A note transferred with the saved ID key.\n", + "P2P discarded the original revision body." + ); + + const renamedPath = "Cafe\u0301/renamed.md"; + await runCliWithInputOrFail(editedBody, clientVault, "--settings", clientSettings, "put", renamedPath); + await runCliOrFail(clientVault, "--settings", clientSettings, "rm", receivedPath); + await exchange(clientVault, clientSettings, hostVault, hostSettings, hostPeerName); + assert.notEqual((await runCli(hostVault, "--settings", hostSettings, "cat", notePath)).code, 0); + assert.equal( + sanitiseCatStdout( + await runCliOrFail(hostVault, "--settings", hostSettings, "cat", renamedPath.normalize("NFC")) + ), + editedBody + ); + await runCliOrFail(clientVault, "--settings", clientSettings, "rm", renamedPath.normalize("NFC")); + await exchange(clientVault, clientSettings, hostVault, hostSettings, hostPeerName); + assert.notEqual((await runCli(hostVault, "--settings", hostSettings, "cat", renamedPath)).code, 0); + } } finally { - await host.stop(); + await stopLocalRelayIfStarted(relayStarted); + await stopCoturnIfStarted(coturnStarted); } - const received = sanitiseCatStdout( - await runCliOrFail(hostVault, "--settings", hostSettings, "cat", notePath) - ).trimEnd(); - assert(received === "A note transferred with the saved ID key.", "The host did not receive the keyed note."); - const rejectedRead = await runCli(rejectedVault, "--settings", rejectedSettings, "cat", notePath); - assert(rejectedRead.code !== 0, "The rejected device received the keyed note."); - } finally { - await stopLocalRelayIfStarted(relayStarted); - await stopCoturnIfStarted(coturnStarted); - } -}); + }); +} diff --git a/test/e2e-obsidian/runner/cliJournalMigration.ts b/test/e2e-obsidian/runner/cliJournalMigration.ts new file mode 100644 index 00000000..3019b65d --- /dev/null +++ b/test/e2e-obsidian/runner/cliJournalMigration.ts @@ -0,0 +1,243 @@ +import { deepStrictEqual } from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { assertEqual } from "./liveSyncWorkflow.ts"; +import { deleteObjectStoragePrefix, readObjectStorageJson, type ObjectStorageConfig } from "./objectStorage.ts"; + +const notePath = "Cafe\u0301/re\u0301sume\u0301.md"; +const noteContent = "# CLI journal migration\n\nThe NFD physical path must keep its identity during migration.\n"; + +type CliFileInfo = { + id: string; + revision: string; + path: string; +}; + +type JournalMilestone = { + node_chunk_info?: Record; +}; + +type CliResult = { + stdout: string; + stderr: string; +}; + +type CliRunner = (...args: string[]) => Promise; + +function cliErrorOutput(error: unknown): string { + if (error === null || typeof error !== "object") { + return String(error); + } + const details = error as { message?: unknown; stdout?: unknown; stderr?: unknown }; + return [details.message, details.stdout, details.stderr] + .filter((value): value is string => typeof value === "string") + .join("\n"); +} + +async function readSingleMetadata(run: CliRunner, label: string): Promise { + const output = (await run("ls")).stdout.trimEnd(); + const paths = output === "" ? [] : output.split(/\r?\n/u).map((line) => line.split("\t", 1)[0] ?? ""); + assertEqual(paths.length, 1, `${label} did not contain exactly one Metadata document.`); + const storedPath = paths[0]; + if (!storedPath) { + throw new Error(`${label} did not report a stored path.`); + } + return JSON.parse((await run("info", storedPath)).stdout) as CliFileInfo; +} + +export async function runCliJournalMigration( + objectStorage: ObjectStorageConfig, + bucketPrefix: string, + sourceSettings: Record +): Promise { + const migrationPrefix = `${bucketPrefix}migration/`; + const cliRoot = await mkdtemp(join(tmpdir(), "livesync-cli-journal-migration-")); + const sourceRoot = join(cliRoot, "source"); + const targetRoot = join(cliRoot, "target"); + const sourceDatabasePath = join(sourceRoot, "database"); + const targetDatabasePath = join(targetRoot, "database"); + const sourceVaultPath = join(sourceRoot, "vault"); + const targetVaultPath = join(targetRoot, "vault"); + const sourceSettingsPath = join(sourceRoot, "settings.json"); + const targetSettingsPath = join(targetRoot, "settings.json"); + const physicalParent = notePath.split("/")[0] ?? ""; + const physicalFilename = notePath.split("/").slice(-1)[0] ?? ""; + + const writeCliSettings = async (settingsPath: string, pathNormalizationVersion: number) => { + const previous = await readFile(settingsPath, "utf8").then( + (text) => JSON.parse(text) as Record, + (error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + return undefined; + } + ); + await writeFile( + settingsPath, + JSON.stringify({ + ...(previous ?? { + ...DEFAULT_SETTINGS, + ...objectStorage, + ...sourceSettings, + remoteType: "MINIO", + bucketPrefix: migrationPrefix, + isConfigured: true, + }), + pathNormalizationVersion, + liveSync: false, + syncOnStart: false, + syncOnSave: false, + syncOnEditorSave: false, + syncOnFileOpen: false, + syncAfterMerge: false, + periodicReplication: false, + }) + ); + }; + + try { + await mkdir(sourceDatabasePath, { recursive: true }); + await mkdir(targetDatabasePath, { recursive: true }); + await mkdir(dirname(join(sourceVaultPath, notePath)), { recursive: true }); + await mkdir(dirname(join(targetVaultPath, notePath)), { recursive: true }); + await writeFile(join(sourceVaultPath, notePath), noteContent); + await writeFile(join(targetVaultPath, notePath), noteContent); + await writeCliSettings(sourceSettingsPath, 0); + await writeCliSettings(targetSettingsPath, 0); + + const execute = promisify(execFile); + const createRunner = + (databasePath: string, vaultPath: string, settingsPath: string): CliRunner => + async (...args: string[]) => + await execute( + process.execPath, + [ + resolve("src/apps/cli/dist/index.cjs"), + databasePath, + "--vault", + vaultPath, + "--settings", + settingsPath, + ...args, + ], + { timeout: 60_000, maxBuffer: 4 * 1024 * 1024 } + ); + + const source = createRunner(sourceDatabasePath, sourceVaultPath, sourceSettingsPath); + const target = createRunner(targetDatabasePath, targetVaultPath, targetSettingsPath); + + await source("--force", "--write-settings", "rebuild"); + await target("mirror"); + + const legacySource = await readSingleMetadata(source, "Legacy CLI source"); + const legacyTarget = await readSingleMetadata(target, "Legacy CLI target"); + const targetVaultEntriesBeforeFetch = await readdir(targetVaultPath); + const targetParentEntriesBeforeFetch = await readdir(dirname(join(targetVaultPath, notePath))); + assertEqual( + targetVaultEntriesBeforeFetch.includes(physicalParent), + true, + "The legacy CLI target did not retain the NFD physical parent." + ); + assertEqual( + targetParentEntriesBeforeFetch.includes(physicalFilename), + true, + "The legacy CLI target did not retain the NFD physical filename." + ); + + await writeCliSettings(sourceSettingsPath, 1); + await source("--force", "--write-settings", "rebuild"); + + const migratedSource = await readSingleMetadata(source, "Migrated CLI source"); + if (migratedSource.id === legacySource.id) { + throw new Error("The NFC migration did not change the source Metadata ID."); + } + assertEqual( + (await source("cat", notePath.normalize("NFC"))).stdout, + noteContent, + "The migrated CLI source did not retain the canonical Chunk content." + ); + + const milestoneKey = `${migrationPrefix}_00000000-milestone.json`; + const milestoneBeforeFailedFetch = await readObjectStorageJson(objectStorage, milestoneKey); + const migrationVersions = Object.values(milestoneBeforeFailedFetch.node_chunk_info ?? {}).map( + (node) => node.current + ); + assertEqual(migrationVersions.includes(4), true, "The migrated Journal milestone did not declare version 4."); + + let legacyFetchError: unknown; + try { + await target("--force", "--write-settings", "fetch"); + } catch (error) { + legacyFetchError = error; + } + if (legacyFetchError === undefined) { + throw new Error("The legacy CLI target unexpectedly fetched a Journal with NFC path identity."); + } + const legacyFetchOutput = cliErrorOutput(legacyFetchError); + if (!/path identity/iu.test(legacyFetchOutput)) { + throw new Error( + `The legacy CLI Fetch failed without the expected path identity error. Output: ${legacyFetchOutput}` + ); + } + + const targetAfterFailedFetch = await readSingleMetadata(target, "Legacy CLI target after failed Fetch"); + assertEqual( + targetAfterFailedFetch.id, + legacyTarget.id, + "The failed legacy Fetch changed the target Metadata ID." + ); + assertEqual( + targetAfterFailedFetch.revision, + legacyTarget.revision, + "The failed legacy Fetch changed the target Metadata revision." + ); + const milestoneAfterFailedFetch = await readObjectStorageJson(objectStorage, milestoneKey); + deepStrictEqual( + milestoneAfterFailedFetch, + milestoneBeforeFailedFetch, + "The failed legacy Fetch changed the remote Journal milestone." + ); + + await writeCliSettings(targetSettingsPath, 1); + await target("--force", "--write-settings", "fetch"); + + const migratedTarget = await readSingleMetadata(target, "Migrated CLI target"); + assertEqual(migratedTarget.id, migratedSource.id, "The CLI source and target Metadata IDs did not converge."); + assertEqual( + (await target("cat", notePath.normalize("NFC"))).stdout, + noteContent, + "The migrated CLI target did not retain the canonical Chunk content." + ); + assertEqual( + await readFile(join(targetVaultPath, notePath), "utf8"), + noteContent, + "The migrated CLI Fetch changed the NFD physical file contents." + ); + + const targetVaultEntriesAfterFetch = await readdir(targetVaultPath); + const targetParentEntriesAfterFetch = await readdir(dirname(join(targetVaultPath, notePath))); + deepStrictEqual( + targetVaultEntriesAfterFetch, + targetVaultEntriesBeforeFetch, + "The migrated CLI Fetch changed the physical Vault parent names." + ); + deepStrictEqual( + targetParentEntriesAfterFetch, + targetParentEntriesBeforeFetch, + "The migrated CLI Fetch changed the physical NFD filename." + ); + + console.log( + "CLI_JOURNAL_MIGRATION: legacy Fetch rejection, safe migration, and NFD physical path preservation verified." + ); + } finally { + try { + await rm(cliRoot, { recursive: true, force: true }); + } finally { + await deleteObjectStoragePrefix(objectStorage, migrationPrefix); + } + } +} diff --git a/test/e2e-obsidian/runner/remoteActivity.ts b/test/e2e-obsidian/runner/remoteActivity.ts index 355d7fa7..5a44dc09 100644 --- a/test/e2e-obsidian/runner/remoteActivity.ts +++ b/test/e2e-obsidian/runner/remoteActivity.ts @@ -34,6 +34,7 @@ export type RemoteActivitySnapshot = { responseCount: number; statusBarFound: boolean; statusBarText: string; + indicatorElements?: Array<{ text: string; inactive: boolean; ariaHidden: string | null; display: string }>; }; export type ExpectedRemoteActivityState = @@ -88,14 +89,26 @@ async function readRemoteActivitySnapshotFromPage(page: Page): Promise - (element.textContent ?? "").includes(operationIcon) - ), - remoteRequestIndicatorVisible: statusBars.some((element) => - (element.textContent ?? "").includes(requestIcon) + element.innerText.includes(operationIcon) ), + remoteRequestIndicatorVisible: statusBars.some((element) => element.innerText.includes(requestIcon)), responseCount: Number(core.services?.API?.responseCount?.value ?? -1), statusBarFound: statusBars.length > 0, - statusBarText: statusBars.map((element) => element.textContent ?? "").join("\n"), + statusBarText: statusBars.map((element) => element.innerText).join("\n"), + indicatorElements: Array.from( + document.querySelectorAll(".syncstatusbar .livesync-stat-item") + ) + .filter( + (element) => + (element.textContent ?? "").includes(operationIcon) || + (element.textContent ?? "").includes(requestIcon) + ) + .map((element) => ({ + text: element.textContent ?? "", + inactive: element.classList.contains("livesync-stat-inactive"), + ariaHidden: element.getAttribute("aria-hidden"), + display: getComputedStyle(element).display, + })), } satisfies RemoteActivitySnapshot; }, { @@ -141,7 +154,7 @@ export async function waitForRemoteActivityState( if (!core) return false; const gate = (globalThis as unknown as Record)[stateKey]; const statusBarText = Array.from(document.querySelectorAll(".syncstatusbar")) - .map((element) => element.textContent ?? "") + .map((element) => element.innerText) .join("\n"); const bounded = Number(core.services?.replicator?.boundedRemoteActivityCount?.value ?? -1); const finite = Number(core.services?.replicator?.finiteReplicationActivityCount?.value ?? -1); diff --git a/test/e2e-obsidian/scripts/minio-upload.ts b/test/e2e-obsidian/scripts/minio-upload.ts index 7de062e8..48aa05b0 100644 --- a/test/e2e-obsidian/scripts/minio-upload.ts +++ b/test/e2e-obsidian/scripts/minio-upload.ts @@ -16,8 +16,15 @@ * same upload. */ import { randomBytes } from "node:crypto"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { DEFAULT_SETTINGS, TweakValuesTemplate } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings"; import { evalObsidianJson } from "../runner/cli.ts"; +import { runCliJournalMigration } from "../runner/cliJournalMigration.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { assertEqual, @@ -43,7 +50,8 @@ import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../r process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000"; -const notePath = "E2E/minio-upload.md"; +const unicodePaths = process.env.E2E_OBSIDIAN_UNICODE_PATHS === "true"; +const notePath = unicodePaths ? "Cafe\u0301/re\u0301sume\u0301.md" : "E2E/minio-upload.md"; const useIndependentIds = process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true"; const useCustomRequestHandler = process.env.E2E_OBSIDIAN_CUSTOM_HTTP_HANDLER === "true"; const noteContent = [ @@ -63,10 +71,11 @@ async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.Proces `const path=${JSON.stringify(notePath)};`, `const content=${JSON.stringify(noteContent)};`, "const core=app.plugins.plugins['obsidian-livesync'].core;", - "if(!(await app.vault.adapter.exists('E2E'))) await app.vault.createFolder('E2E');", + "const parent=path.slice(0,path.lastIndexOf('/'));", + "if(!(await app.vault.adapter.exists(parent))) await app.vault.createFolder(parent);", "const existing=app.vault.getAbstractFileByPath(path);", "if(existing) await app.vault.delete(existing);", - "await app.vault.create(path,content);", + "const createdFile=await app.vault.create(path,content);", "const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));", "let entry=false;", "for(let i=0;i<40;i++){", @@ -76,7 +85,7 @@ async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.Proces "await sleep(250);", "}", "if(!entry||!entry._id) throw new Error('Timed out waiting for local database entry');", - "return JSON.stringify({id:entry._id,path:entry.path,type:entry.type,children:entry.children||[]});", + "return JSON.stringify({id:entry._id,path:createdFile.path,type:entry.type,children:entry.children||[]});", "})()", ].join(""), env @@ -124,10 +133,13 @@ async function main(): Promise { cliBinary: cli.binary, vault, startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), - pluginData: createE2eObjectStoragePluginData({ - ...objectStorage, - bucketPrefix, - }), + pluginData: createE2eObjectStoragePluginData( + { + ...objectStorage, + bucketPrefix, + }, + { pathNormalizationVersion: unicodePaths ? 1 : 0 } + ), localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), }); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); @@ -137,6 +149,7 @@ async function main(): Promise { session.cliEnv, { ...objectStorage, bucketPrefix }, { + pathNormalizationVersion: unicodePaths ? 1 : 0, ...(useIndependentIds ? { encrypt: true, @@ -173,6 +186,13 @@ async function main(): Promise { ); } } + if (unicodePaths && !useIndependentIds) { + assertEqual( + localEntry.id, + notePath.normalize("NFC").toLowerCase(), + "Obsidian did not derive the canonical Metadata ID." + ); + } await pushLocalChanges(cli.binary, session.cliEnv); const activityAfterUpload = await waitForRemoteActivityState( session.remoteDebuggingPort, @@ -198,6 +218,131 @@ async function main(): Promise { } } + if (unicodePaths) { + const cliRoot = await mkdtemp(join(tmpdir(), "livesync-nfc-journal-")); + try { + const databasePath = join(cliRoot, "db"); + const vaultPath = join(cliRoot, "vault"); + const settingsPath = join(cliRoot, "settings.json"); + await mkdir(databasePath, { recursive: true }); + await mkdir(dirname(join(vaultPath, notePath)), { recursive: true }); + await writeFile(join(vaultPath, notePath), noteContent); + const sourceSettings = await evalObsidianJson>( + cli.binary, + [ + "(async()=>{", + "const service=app.plugins.plugins['obsidian-livesync'].core.services.setting;", + "const s=service.currentSettings();", + `const keys=${JSON.stringify([...Object.keys(TweakValuesTemplate), "encrypt", "passphrase", "usePathObfuscation", "E2EEAlgorithm", "idDerivationKey"])};`, + "const values=Object.fromEntries(keys.map(key=>[key,s[key]]));", + "if(s.idDerivationVersion===1){", + "values.encryptedIdDerivationKey=await service.encryptConfigurationItem(s.idDerivationKey,s,'*');", + "values.idDerivationKey='';values.configPassphraseStore='';", + "}", + "return JSON.stringify(values);", + "})()", + ].join(""), + session.cliEnv + ); + await runCliJournalMigration(objectStorage, bucketPrefix, sourceSettings); + await writeFile( + settingsPath, + JSON.stringify({ + ...DEFAULT_SETTINGS, + ...objectStorage, + ...sourceSettings, + remoteType: "MINIO", + bucketPrefix, + pathNormalizationVersion: 1, + isConfigured: true, + }) + ); + const execute = promisify(execFile); + const run = async (...args: string[]) => + await execute( + process.execPath, + [ + resolve("src/apps/cli/dist/index.cjs"), + databasePath, + "--vault", + vaultPath, + "--settings", + settingsPath, + ...args, + ], + { timeout: 60_000, maxBuffer: 4 * 1024 * 1024 } + ); + await run("--force", "--write-settings", "fetch"); + assertEqual( + await readFile(join(vaultPath, notePath), "utf8"), + noteContent, + "CLI Fetch changed the file contents." + ); + assertEqual( + (await run("cat", notePath.normalize("NFC"))).stdout, + noteContent, + "CLI Fetch did not retain the remote Chunk content." + ); + const listedPaths = (await run("ls")).stdout + .trim() + .split("\n") + .map((line) => line.split("\t")[0]); + assertEqual(listedPaths.length, 1, "CLI Fetch retained duplicate Metadata."); + assertEqual( + listedPaths[0].normalize("NFC"), + notePath.normalize("NFC"), + "CLI Fetch changed the logical path." + ); + const fileInfo = JSON.parse((await run("info", listedPaths[0])).stdout) as { id: string }; + assertEqual(fileInfo.id, localEntry.id, "CLI Fetch used a different Metadata ID."); + const expectedNames = await readdir(dirname(join(vaultPath, notePath))); + assertEqual(expectedNames.length, 1, "CLI Fetch created a duplicate physical file."); + assertEqual( + expectedNames[0], + notePath.split("/").slice(-1)[0], + "CLI Fetch renamed the NFD physical file." + ); + const editedContent = noteContent + "\nEdited after CLI restart.\n"; + await writeFile(join(vaultPath, notePath), editedContent); + const nextTime = new Date(Date.now() + 5000); + await utimes(join(vaultPath, notePath), nextTime, nextTime); + await run("mirror"); + await run("sync"); + await pushLocalChanges(cli.binary, session.cliEnv); + const sourceFile = join(vault.path, localEntry.path); + const deadline = Date.now() + 20_000; + while ((await readFile(sourceFile, "utf8")) !== editedContent && Date.now() < deadline) { + await new Promise((resolveWait) => setTimeout(resolveWait, 200)); + } + assertEqual( + await readFile(sourceFile, "utf8"), + editedContent, + "Obsidian did not receive the CLI edit." + ); + await run("rm", notePath.normalize("NFC")); + await run("sync"); + await pushLocalChanges(cli.binary, session.cliEnv); + const deleteDeadline = Date.now() + 20_000; + let exists = true; + while (exists && Date.now() < deleteDeadline) { + exists = await readFile(sourceFile).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false; + throw error; + } + ); + if (exists) await new Promise((resolveWait) => setTimeout(resolveWait, 200)); + } + assertEqual(exists, false, "Obsidian did not reflect the CLI deletion."); + console.log( + "NFC_JOURNAL_ROUND_TRIP: canonical ID, NFD physical name, CLI Fetch, edit after process restart, and deletion verified." + ); + } finally { + await rm(cliRoot, { recursive: true, force: true }); + } + } + console.log( `Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}` ); diff --git a/updates.md b/updates.md index 3d01a759..f10e76ed 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,9 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +- Unicode NFC path identity can be used with Object Storage and P2P on CLI and Obsidian clients. The setting is available under General Settings β†’ File identity. Existing databases require a coordinated Rebuild and Fetch; physical file names and Chunk IDs stay unchanged. All P2P participants must be updated and migrated. + - Object Storage checks the selected identity before Fetch replaces the local database and before ordinary remote administration changes its milestone. Rebuild preserves the old identity until the remote journals have been deleted. + - The CLI mirror now starts correctly on Windows when the Vault path is omitted and the database directory is also used as the Vault. It checks logical Vault paths using `/` separators, correctly excluding internal database files and nested hidden directories even when the host uses Windows path separators. ## 1.0.43