From 6c50505096b13e9ded2131fd4ad1fb936003c8df Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 27 Sep 2026 09:51:53 +0000 Subject: [PATCH 1/8] Integrate encrypted internal metadata in LiveSync --- ...elease_notes_and_database_compatibility.md | 4 +- .../internal_metadata_encryption.md | 198 ++++++++++++ docs/settings.md | 8 + docs/troubleshooting.md | 6 + package.json | 1 + src/common/replicatorConfigurationIdentity.ts | 2 + ...plicatorConfigurationIdentity.unit.spec.ts | 16 + .../ObsidianLiveSyncSettingTab.ts | 1 + .../SettingDialogue/PaneRemoteConfig.ts | 34 ++- .../PaneRemoteConfig.unit.spec.ts | 58 +++- .../dialogs/SetupRemoteE2EE.svelte | 23 +- .../replication/ReplicateResultProcessor.ts | 255 ++++++++++++++-- .../ReplicateResultProcessor.unit.spec.ts | 285 +++++++++++++++++- .../centralCompatibilityRecovery.ts | 22 +- .../centralCompatibilityRecovery.unit.spec.ts | 58 +++- src/serviceFeatures/replication/index.ts | 14 +- .../replicationFeature.unit.spec.ts | 71 ++++- test/e2e-obsidian/README.md | 2 + .../scripts/cli-to-obsidian-sync.ts | 2 + .../scripts/customisation-sync.ts | 19 ++ .../scripts/hidden-file-snippet-sync.ts | 17 ++ .../scripts/remote-feature-change.ts | 178 +++++++++++ updates.md | 7 + 23 files changed, 1220 insertions(+), 61 deletions(-) create mode 100644 docs/design_docs/internal_metadata_encryption.md create mode 100644 test/e2e-obsidian/scripts/remote-feature-change.ts diff --git a/docs/adr/2026_07_release_notes_and_database_compatibility.md b/docs/adr/2026_07_release_notes_and_database_compatibility.md index 5a7d80aa..4985008d 100644 --- a/docs/adr/2026_07_release_notes_and_database_compatibility.md +++ b/docs/adr/2026_07_release_notes_and_database_compatibility.md @@ -57,6 +57,8 @@ Keep configured-state inference separate from new-Vault initialisation. If an ex - On resume, clear `versionUpFlash` and persist that fail-closed change before recording the current `VER` as acknowledged. If saving fails, restore the gate. Reapply settings only after the marker has advanced so that the previously configured synchronisation behaviour can resume without reconstruction. - Preserve the original legacy review message as a structured reason when no more specific database or settings-schema reason is available. Escape it before including it in Markdown UI. - Continue to reject a remote version document which is newer than the running implementation. That receiver-side check is independent of the local upgrade review. +- From remote generation 13, assess the `used_features` list in that document as a separate compatibility dimension. A client must recognise every listed feature before it interprets the database or runs maintenance which depends on Metadata. Declare a feature before writing its representation, and retain the declaration while older data may depend on it. An unknown identifier is reported as text without requiring a descriptive label in that client. +- Do not advance the device-local `VER` acknowledgement merely because a remote feature is introduced. The remote generation and its feature list govern remote admission; `VER` remains the local compatibility review gate. Connecting to a generation-12 database does not promote it solely because the client understands generation 13. ### Onboarding activation and initialisation @@ -90,7 +92,7 @@ Keep configured-state inference separate from new-Vault initialisation. If an ex - Accepted new-device and existing-device setup cannot enable ordinary processing before the selected Rebuild or Fetch has been reserved. - An older installation cannot dismiss evidence that a newer implementation or settings schema has already been used on the device. - The Obsidian-specific dialogue depends only on a host-neutral compatibility result and the injected confirmation capability. Commonlib remains responsible for settings migration, device-local storage, and the replication gate. -- A future incompatible database change must increment `VER`, provide an actionable review message, verify the remote version negotiation, and test both the pending and acknowledged states. A major SemVer increase without those changes has no database-compatibility effect. +- A future local database change which requires compatibility review must increment `VER`, provide an actionable review message, and test both the pending and acknowledged states. A new remote representation must declare its feature before use and verify remote admission independently. A major SemVer increase alone has no database-compatibility effect. ## Verification diff --git a/docs/design_docs/internal_metadata_encryption.md b/docs/design_docs/internal_metadata_encryption.md new file mode 100644 index 00000000..c73d0d01 --- /dev/null +++ b/docs/design_docs/internal_metadata_encryption.md @@ -0,0 +1,198 @@ +--- +date: 2026-09-27 +commonlib-version: "0.1.30" +self-hosted-livesync-version: "1.0.32" +status: unreleased +--- + +# Internal Metadata encryption and remote feature changes + +This document defines the LiveSync integration of Commonlib's remote feature +contract and encrypted Metadata for Hidden File Sync and Customisation Sync. +It describes unreleased behaviour being implemented in this branch. + +Commonlib's companion `docs/remote-feature-compatibility.md` is the +source of truth for the wire document, identifiers, validation, and shared +assessment. This document owns the application behaviour, settings, Doctor +recommendation, and verification of the Obsidian and CLI integrations. + +## Scope and settings + +Add `encryptInternalMetadata` to the shared encryption settings. A genuinely new +Vault or CLI configuration defaults to true. Existing stored settings and old +Setup URI or QR imports complete an absent value as false. Ordinary partial +setting updates retain the current value. + +The preference applies to CouchDB with E2EE V2 and Property Encryption enabled. +Show the preference as unavailable and explain its prerequisites when they are +absent. Keep Journal and P2P's existing +transport protection and avoid unrelated setting mismatches for those remotes. + +Use the existing HKDF Metadata representation to protect path, creation and +modification times, size, and Chunk references for obfuscated internal entries. +Keep the `i:`, `ix:`, and supported legacy `ps:` document IDs, path conversion, +and content Chunk representation. Read encrypted Metadata independently of the +write preference, including after that preference is disabled. + +The protection leaves document IDs, namespaces, revisions, deletion state, +document counts, and ciphertext lengths visible. It does not encrypt device or +Vault names stored in separate participant records. + +## Enabling the preference + +Changing the preference does not automatically reconstruct a database or gather +all devices' data. It affects subsequent Metadata writes. Unchanged documents +and old revisions can retain plaintext; mixed plaintext and encrypted Metadata +are a supported transition state. + +Strongly recommend the existing manual remote Rebuild workflow when the person +wants existing Metadata protected as well. The person prepares the authoritative +data for that workflow. Describe this distinction in the setting, Doctor reason, +and operational documentation. Do not advertise complete historical protection +merely because the preference is enabled. + +Copy the preference with the other encryption settings when preparing a remote +profile. Recreate a connection when its effective encryption settings change. +Use the existing Tweak assessment and manual mismatch resolution; do not change +the remote's shared policy silently when importing or loading settings. + +## Doctor + +Use Commonlib's existing conditional recommendation rules. Recommend true when +the selected CouchDB settings have E2EE V2 and Property Encryption enabled and +the new preference is false. Do not require Hidden File Sync or Customisation +Sync to be active before offering the recommendation. + +Retain the existing E2EE V2 recommendation for a legacy algorithm. After that +change, ensure the newly applicable Metadata recommendation is not hidden by a +premature `doctorProcessedVersion` update. Advance the Doctor rule revision so +an older completed consultation does not suppress this new recommendation. + +Apply the preference only when the person accepts the recommendation. Include +the existing-data limitation, the manual Rebuild recommendation, and the need +for compatible clients in the explanation. Do not set `requireRebuild` or +`requireRebuildLocal` for this rule: the current host wrapper can schedule those +operations and restart. `recommendRebuild` currently exists only as an unused +rule field, so setting it alone does not display an explanation. + +## Receiving a changed version document + +The existing path is: + +1. Commonlib receives a CouchDB replication change and calls + `parseSynchroniseResult` with the received documents. +2. The replication service feature passes them to + `ReplicateResultProcessor.enqueueAll`. +3. `processIfNonDocumentChange` recognises `type: versioninfo` and requests active + Replicator retirement when `version > VER`. +4. The owner closes admission, requests transfer cancellation, drains its work, + and closes the instance. The result callback does not wait for that transition. + +Retain this observation path, but use Commonlib's complete assessment of the +identified control document. A changed `used_features` list must be inspected +even when the numeric version is unchanged. A mere revision change, list reorder, +or duplicate known identifier does not require an interruption. + +Inspect the entire batch's control information before passing any file entries +to normal or optional processing. Recognise the fixed control-document ID and +validate its type and contents. A deleted or malformed control document is a +rejection, not a successful empty update. + +When all requirements remain supported, refresh the assessment and affected +shared-setting checks; a newly added feature retires the current writer so its +next admission rechecks the shared Tweak policy. When a requirement is +unknown or incompatible, synchronously record the block for the affected +database, stop admitting new reflection and database operations, and request +retirement through the existing owner. Notify with the unknown identifiers as +text, using a generic message when no descriptive label exists. + +File application and remote transfer have separate lifetimes. Requesting owner +retirement alone is not the application block. Keep the block separate from +temporary lifecycle suspension so an ordinary resume event cannot clear it. +Queued or waiting work checks it before starting another write; notifications +from an old physical database must not affect its replacement. + +Do not await `onCloseActiveReplication` inside the callback which delivered the +change. That callback can belong to work which retirement must drain. Establish +the block immediately, request retirement without awaiting it there, and let the +owner perform cancellation and close in its existing order. + +## Persistence and recovery boundaries + +A CouchDB replication notification can arrive after the documents have entered +the local DB. Already-started network and filesystem operations may settle. +This feature does not promise rollback or atomic revocation of those operations. + +Preserve pending work or durable reconciliation information when stopping. A +checkpoint may already include the documents which have not reached the Vault. +Do not drop those documents or depend on an ordinary reconnect to send them +again. A compatible client must reassess and reprocess or explicitly reacquire +them before lifting the applicable block. +Persist a blocked pending-work snapshot before the received-change callback +settles, while requesting owner retirement separately to avoid a circular wait. + +Restore compatibility checks before replication result application and the next +ordinary synchronisation. Retain observed feature requirements with the pending +work snapshot so that a shortened version-document list does not release a +blocked local database on restart. A dismissed Notice or a changed connection +does not establish that the affected local data has become interpretable. +An older local generation without feature declarations remains readable during +normal application and cleaned-remote recovery; remote migration remains the +responsibility of the replication admission check. + +Garbage Collection V3 is a beta manual operation which begins with an ordinary +bidirectional synchronisation. That admission checks the remote feature +contract; no additional per-step GC checks are introduced. The separate +cleaned-remote recovery path checks the local version document before its +first Chunk-reference count because it does not start with that synchronisation. + +Use the same Commonlib assessment at the CLI, Fast Fetch, and direct-access +boundaries. The Obsidian result processor is one consumer, not the only place +which determines compatibility. Keep unrelated Vaults and databases operational. + +## Verification and documentation + +Keep focused tests for settings defaults and imports, the Doctor condition +matrix, acceptance and dismissal, connection replacement, and absence of an +automatic Rebuild, Fetch, or restart for this rule. + +Add deterministic runtime tests for feature-only changes, version documents +first and last in a batch, unknown-name presentation, duplicate notifications, +queued and waiting reflection, stale database callbacks, restart, checkpointed +but unapplied documents, and cancellation without a circular wait. + +Before this implementation, the host processor was checked with a focused unit probe: a numeric +incompatibility requests retirement, but the processor still applies a note in +the same batch when its host remains ready. A same-version document with an +unknown feature does not request retirement. These observations motivated the +new checks. + +Extend real Obsidian Hidden File Sync and Customisation Sync scenarios and the +CLI interoperability checks. Inspect raw CouchDB documents as well as restored +files. Exercise an active connection when another client changes the feature +requirements, and verify that previously accepted data survives the stop. +Pending-work restoration and recovery with a compatible client are separate +boundaries. + +The local packed Commonlib 0.1.30 candidate passed Commonlib unit and boundary +tests and the LiveSync build, type checks, and unit tests. Real Obsidian 1.12.7 +passed the Hidden File Sync, Customisation Sync, and encrypted CLI-to-Obsidian +scenarios. The dedicated active-connection scenario changed a generation 12 +remote to generation 13 with an unknown feature whilst continuous replication +was running. The local control document arrived, the active Replicator retired, +a subsequent replication was refused, and an earlier accepted note stayed in +the Vault. Focused host tests cover both batch orders, pending-work snapshots, +restart, stale callbacks, and the older local-generation case. Recovery after +upgrading to a future client that understands the unknown feature has not been +exercised in real Obsidian. + +Keep the primary-language settings and troubleshooting guides, the +database-compatibility ADR, and Unreleased notes aligned with this behaviour. +Keep the detailed shared protocol in Commonlib and link to it after publication; +do not maintain another copy of its wire schema here. Translations are a separate +change. Update tested-version evidence when the implementation and its +validation have been accepted. + +Related application contracts: [Replicator architecture](replicator_architecture.md), +[Tweak compatibility](tweak_compatibility.md), and +[database compatibility](../adr/2026_07_release_notes_and_database_compatibility.md). diff --git a/docs/settings.md b/docs/settings.md index 2ce7e275..d6622d91 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -245,6 +245,14 @@ Setting key: usePathObfuscation In default, the path of the file is not obfuscated to improve the performance. If you enable this, the path of the file will be obfuscated. This is useful when you want to hide the path of the file. +#### Encrypt internal file Metadata + +Setting key: encryptInternalMetadata + +For CouchDB, this encrypts paths, times, sizes, and Chunk references in the Metadata used by Hidden File Sync and Customisation Sync. It requires E2EE V2 and **Property Encryption**. New Vaults enable the preference by default, but it has no effect until those prerequisites are enabled. Existing Vaults and older Setup URIs and QR codes keep it disabled unless you enable it. + +Enabling the preference protects future Metadata writes. Existing Metadata and earlier revisions can remain readable in the remote database. If you want to protect existing Metadata too, prepare the authoritative data, update every device to a compatible version, and manually Rebuild the remote database. LiveSync does not gather data or start a Rebuild when you change this preference. Plaintext and encrypted Metadata can coexist during the transition. Document IDs, revision information, document counts, and ciphertext lengths remain visible. + #### Encryption Algorithm Setting key: E2EEAlgorithm diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d8f128b5..b2a00a68 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -96,6 +96,8 @@ Current releases automatically align compatible settings which control how new c A missing legacy file-name case setting means case-insensitive handling. It matches an explicit disabled setting and does not require a rebuild for that difference. An explicitly enabled setting can use different document IDs and still requires a compatibility decision against either value. Other configuration differences shown in the dialogue must still be resolved. +If the mismatch names **Encrypt internal file Metadata**, update every device before accepting that preference. It affects subsequent Metadata writes for Hidden File Sync and Customisation Sync; it does not automatically protect existing Metadata. A manual remote Rebuild is strongly recommended if you need to protect existing paths, times, sizes, and Chunk references. + The `Sync now` command keeps routine replication progress quiet so that it is convenient to assign to a keyboard shortcut; assign one in Obsidian if that suits your workflow. A quiet command may still open this dialogue when a mismatch or another decision requires your attention. The available actions depend on when the mismatch is found: @@ -109,6 +111,10 @@ The available actions depend on when the mismatch is found: Historic defect notices and renamed controls are retained in the [0.25 release history](releases/0.25.md) and [legacy release history](releases/legacy.md), rather than in the current troubleshooting path. +## The remote database uses an unknown feature + +When a notice identifies an unknown feature, update this device and every other client of the same CouchDB database, including the CLI. The notice includes the feature identifier even if this version has no descriptive name for it. Synchronisation and pending file reflection pause because an older client may not interpret the Metadata and its Chunk references correctly. The cleaned-remote recovery path also checks compatibility before counting Chunk references. Do not remove the feature name from the remote version document to bypass the check. After updating, reconnect and review any pending file changes before running Garbage Collection. + ## Setup and settings questions ### Share a configuration with another device diff --git a/package.json b/package.json index d87efd90..d270849e 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "test:e2e:obsidian:security-seed-reconnect": "tsx test/e2e-obsidian/scripts/security-seed-reconnect.ts", "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", + "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", "test:e2e:obsidian:setting-markdown-export": "tsx test/e2e-obsidian/scripts/setting-markdown-export.ts", "test:e2e:obsidian:upgrade-from-stable": "tsx test/e2e-obsidian/scripts/upgrade-from-stable.ts", "test:e2e:obsidian:local-suite": "tsx test/e2e-obsidian/scripts/local-suite.ts", diff --git a/src/common/replicatorConfigurationIdentity.ts b/src/common/replicatorConfigurationIdentity.ts index ff6221d8..af72f6d6 100644 --- a/src/common/replicatorConfigurationIdentity.ts +++ b/src/common/replicatorConfigurationIdentity.ts @@ -1,4 +1,5 @@ import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { usesEncryptedInternalMetadata } from "@vrtmrz/livesync-commonlib/replication"; type EndpointProjection = readonly [kind: "url" | "invalid-url", value: string]; @@ -77,6 +78,7 @@ export function getCouchDBReplicatorConfigurationIdentity(settings: RemoteDBSett settings.useRequestAPI, settings.disableRequestURI, projectRemoteSecurity(settings), + usesEncryptedInternalMetadata(settings), settings.enableCompression, ]); } diff --git a/src/common/replicatorConfigurationIdentity.unit.spec.ts b/src/common/replicatorConfigurationIdentity.unit.spec.ts index 719db130..a5ef0f9b 100644 --- a/src/common/replicatorConfigurationIdentity.unit.spec.ts +++ b/src/common/replicatorConfigurationIdentity.unit.spec.ts @@ -67,6 +67,22 @@ describe("active Replicator configuration identity", () => { ); }); + it("recreates the CouchDB connection when internal Metadata encryption becomes effective", () => { + const active = configuredSettings({ usePathObfuscation: true, encryptInternalMetadata: false }); + const enabled = { ...active, encryptInternalMetadata: true }; + + expect(getCouchDBReplicatorConfigurationIdentity(enabled)).not.toBe( + getCouchDBReplicatorConfigurationIdentity(active) + ); + const inactive = { ...active, usePathObfuscation: false }; + expect(getCouchDBReplicatorConfigurationIdentity({ ...inactive, encryptInternalMetadata: true })).toBe( + getCouchDBReplicatorConfigurationIdentity(inactive) + ); + expect(getObjectStorageReplicatorConfigurationIdentity(enabled)).toBe( + getObjectStorageReplicatorConfigurationIdentity(active) + ); + }); + it("projects only the active CouchDB authentication mode", () => { const basic = configuredSettings({ useJWT: false, jwtKey: "inactive-a" }); expect(getCouchDBReplicatorConfigurationIdentity({ ...basic, jwtKey: "inactive-b" })).toBe( diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 032caf76..5bf463d1 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -569,6 +569,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { } } + // Internal Metadata encryption affects future Metadata writes and is not a rebuild requirement. isNeedRebuildLocal() { return this.isSomeDirty([ "useIndexedDBAdapter", diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts index 260ec3c0..3f93b00c 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts @@ -5,6 +5,7 @@ import { DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, type ObsidianLiveSyncSettings, + type EncryptionSettings, LOG_LEVEL_VERBOSE, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { Menu, type ButtonComponent } from "@/deps.ts"; @@ -31,11 +32,13 @@ import { import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; import type { RemoteConfigurationResult } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; import SetupRemote from "@/modules/features/SetupWizard/dialogs/SetupRemote.svelte"; +import SetupRemoteE2EE from "@/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte"; import SetupRemoteCouchDB from "@/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte"; import SetupRemoteBucket from "@/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte"; import type { SetupRemoteCouchDBInitialData, SetupRemoteCouchDBResultType, + SetupRemoteE2EEResultType, } from "@/modules/features/SetupWizard/dialogs/setupDialogTypes.ts"; import { syncActivatedRemoteSettings } from "./remoteConfigBuffer.ts"; @@ -116,7 +119,35 @@ export function paneRemoteConfig( .onClick(async () => { const setupManager = this.core.getModule(SetupManager); const originalSettings = getSettingsFromEditingSettings(this.editingSettings); - await setupManager.onlyE2EEConfiguration(UserMode.Update, originalSettings); + const e2eeConf = await setupManager.dialogManager.openWithExplicitCancel< + SetupRemoteE2EEResultType, + EncryptionSettings + >(SetupRemoteE2EE, originalSettings); + if (e2eeConf === "cancelled") { + return; + } + const onlyInternalMetadataPreferenceChanged = + originalSettings.encryptInternalMetadata !== e2eeConf.encryptInternalMetadata && + originalSettings.encrypt === e2eeConf.encrypt && + originalSettings.passphrase === e2eeConf.passphrase && + originalSettings.E2EEAlgorithm === e2eeConf.E2EEAlgorithm && + originalSettings.usePathObfuscation === e2eeConf.usePathObfuscation; + if (onlyInternalMetadataPreferenceChanged) { + await this.services.setting.applyPartial( + { encryptInternalMetadata: e2eeConf.encryptInternalMetadata }, + true + ); + this.editingSettings.encryptInternalMetadata = e2eeConf.encryptInternalMetadata; + if (this.initialSettings) { + this.initialSettings.encryptInternalMetadata = e2eeConf.encryptInternalMetadata; + } + this.requestUpdate(); + } else { + await setupManager.onConfirmApplySettingsFromWizard( + { ...originalSettings, ...e2eeConf }, + UserMode.Update + ); + } updateE2EESummary(); }) .setButtonText("Configure") @@ -243,6 +274,7 @@ export function paneRemoteConfig( ...DEFAULT_SETTINGS, encrypt: this.editingSettings.encrypt, usePathObfuscation: this.editingSettings.usePathObfuscation, + encryptInternalMetadata: this.editingSettings.encryptInternalMetadata, passphrase: this.editingSettings.passphrase, configPassphraseStore: this.editingSettings.configPassphraseStore, }); diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts index d61a998a..8d259210 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const runtime = vi.hoisted(() => ({ buttonClasses: [] as string[], + clickHandlers: [] as Array<() => Promise | void>, panels: [] as Array<{ destroy: ReturnType }>, settingClasses: [] as string[], })); @@ -51,7 +52,8 @@ vi.mock("./LiveSyncSetting.ts", () => ({ setDestructive() { return this; }, - onClick() { + onClick(callback: () => Promise | void) { + runtime.clickHandlers.push(callback); return this; }, setButtonText() { @@ -97,6 +99,7 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/common/ConnectionString", () => ({ }, })); vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemote.svelte", () => ({ default: {} })); +vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte", () => ({ default: {} })); vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte", () => ({ default: {} })); vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte", () => ({ default: {} })); vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte", () => ({ default: {} })); @@ -114,6 +117,7 @@ function createPanelElement(): HTMLElement { afterEach(() => { runtime.buttonClasses.length = 0; + runtime.clickHandlers.length = 0; runtime.panels.length = 0; runtime.settingClasses.length = 0; vi.clearAllMocks(); @@ -148,4 +152,56 @@ describe("paneRemoteConfig", () => { expect(runtime.panels[0].destroy).toHaveBeenCalledOnce(); }); + + it("applies an internal Metadata preference change without scheduling setup initialisation", async () => { + const originalSettings = { + encrypt: true, + passphrase: "passphrase", + E2EEAlgorithm: "v2", + usePathObfuscation: true, + encryptInternalMetadata: false, + remoteConfigurations: {}, + }; + const applyPartial = vi.fn(async () => {}); + const onConfirmApplySettingsFromWizard = vi.fn(async () => {}); + const setupManager = { + dialogManager: { + openWithExplicitCancel: vi.fn(async () => ({ + encrypt: true, + passphrase: "passphrase", + E2EEAlgorithm: "v2", + usePathObfuscation: true, + encryptInternalMetadata: true, + })), + }, + onConfirmApplySettingsFromWizard, + }; + const host = { + editingSettings: { ...originalSettings }, + initialSettings: { ...originalSettings }, + services: { setting: { applyPartial } }, + core: { + settings: { ...originalSettings }, + getModule: vi.fn(() => setupManager), + }, + lifetimeComponent: { register: vi.fn() }, + requestUpdate: vi.fn(), + }; + const addPanel = vi.fn((_parent: HTMLElement, heading: string) => ({ + then(callback: (paneEl: HTMLElement) => void) { + if (heading === "E2EE Configuration") { + callback(createPanelElement()); + } + }, + })); + + paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never); + await runtime.clickHandlers[0](); + + expect(applyPartial).toHaveBeenCalledWith({ encryptInternalMetadata: true }, true); + expect(onConfirmApplySettingsFromWizard).not.toHaveBeenCalled(); + expect(host.editingSettings.encryptInternalMetadata).toBe(true); + expect(host.initialSettings.encryptInternalMetadata).toBe(true); + expect(host.requestUpdate).toHaveBeenCalledOnce(); + }); }); diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte index 7340f31d..1b02f79c 100644 --- a/src/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte +++ b/src/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte @@ -26,7 +26,8 @@ passphrase: "", E2EEAlgorithm: DEFAULT_SETTINGS.E2EEAlgorithm, usePathObfuscation: true, - } as EncryptionSettings; + encryptInternalMetadata: true, + }; let encryptionSettings = $state({ ...default_encryption }); @@ -42,6 +43,11 @@ if (!encryptionSettings.encrypt) return true; return encryptionSettings.passphrase.trim().length >= 1; }); + let canEncryptInternalMetadata = $derived( + encryptionSettings.encrypt && + encryptionSettings.E2EEAlgorithm === E2EEAlgorithms.V2 && + encryptionSettings.usePathObfuscation + ); function commit() { setResult(pickEncryptionSettings(encryptionSettings)); @@ -87,6 +93,21 @@ {/if} + + + + + This option applies only to CouchDB and requires End-to-End Encryption, the V2 algorithm, and Property Encryption + (Obfuscate Properties). The remote type is selected later in this setup wizard. +
+ It protects Metadata written after the option is enabled; existing Metadata is not rewritten. A manual remote + Rebuild is strongly recommended to protect existing Metadata. + + - This option applies only to CouchDB and requires End-to-End Encryption, the V2 algorithm, and Property Encryption + This option encrypts file properties used by Hidden File Sync and Customisation Sync. +
+ It applies only to CouchDB and requires End-to-End Encryption, the V2 algorithm, and Property Encryption (Obfuscate Properties). The remote type is selected later in this setup wizard.
- It protects Metadata written after the option is enabled; existing Metadata is not rewritten. A manual remote - Rebuild is strongly recommended to protect existing Metadata. Update every other synchronising device to a compatible + It protects properties written after the option is enabled; existing properties are not rewritten. A manual remote + Rebuild is strongly recommended to protect existing properties. Update every other synchronising device to a compatible version before enabling this option, including devices currently running LiveSync. diff --git a/test/e2e-obsidian/scripts/internal-metadata-migration.ts b/test/e2e-obsidian/scripts/internal-metadata-migration.ts index 416dd436..3b45b3af 100644 --- a/test/e2e-obsidian/scripts/internal-metadata-migration.ts +++ b/test/e2e-obsidian/scripts/internal-metadata-migration.ts @@ -164,9 +164,9 @@ async function main(): Promise { .getByRole("button", { name: "Configure", exact: true }) .click(); const dialog = await waitForVisibleObsidianDialogue(navigator.page, "End-to-End Encryption"); - await dialog.getByLabel("Encrypt internal file Metadata", { exact: true }).check(); + await dialog.getByLabel("Encrypt internal file Properties", { exact: true }).check(); await dialog.getByRole("button", { name: "Proceed", exact: true }).click(); - const warning = await waitForVisibleObsidianDialogue(navigator.page, "Encrypt internal file Metadata"); + const warning = await waitForVisibleObsidianDialogue(navigator.page, "Encrypt internal file Properties"); await warning .getByRole("button", { name: "Enable without rebuilding — update every other device first", @@ -221,7 +221,7 @@ async function main(): Promise { await withObsidianPage(session!.remoteDebuggingPort, async (page) => { const dialog = await waitForVisibleObsidianDialogue(page, "Configuration Mismatch Detected"); await dialog - .getByText("Encrypt internal file Metadata", { exact: false }) + .getByText("Encrypt internal file Properties", { exact: false }) .first() .waitFor({ state: "visible" }); await dialog.getByRole("button", { name: "Dismiss", exact: true }).click(); From f954ce6f92c61f2b746fe1bc5bce4f7e6db59cc2 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 27 Sep 2026 16:34:36 +0000 Subject: [PATCH 6/8] Use released Commonlib 0.1.30 --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 78086c95..248985a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.29", + "@vrtmrz/livesync-commonlib": "0.1.30", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4567,9 +4567,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.29", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.29.tgz", - "integrity": "sha512-PeBUUQNSuyS+ISPIvwK3rzaritHi0XDfNlFMOqaRpIbxGAWbWK0YDtE1Htbde3r8Lr3ZdTNjpY+Nmr//OxzgTA==", + "version": "0.1.30", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.30.tgz", + "integrity": "sha512-GCVLn/qGGJnJdm9gdM8MigvbE/2tjpgLuekqqr0iPsSOAUrIZNYcWq31KfUwrNsn5L4lLy+aMy0LhyWyBvK6yw==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index e66b0717..20d89dd7 100644 --- a/package.json +++ b/package.json @@ -185,7 +185,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.29", + "@vrtmrz/livesync-commonlib": "0.1.30", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", From c1f4376ccc31633f9c3f1758d244bcdcd5156351 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 28 Sep 2026 01:01:25 +0000 Subject: [PATCH 7/8] Explain internal file privacy benefits in release notes --- updates.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/updates.md b/updates.md index 151f8806..cd0c4c1e 100644 --- a/updates.md +++ b/updates.md @@ -16,8 +16,12 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi #### New Feature -- Hidden File Sync and Customisation Sync can now encrypt their paths, times, sizes, and Chunk references in CouchDB Metadata when E2EE V2 and Property Encryption are enabled. The preference is enabled for new Vaults and remains off for existing configurations until selected. It protects future writes; protecting existing Metadata also requires a manual remote Rebuild after all devices have been updated. -- CouchDB records the features its data uses. Clients check these requirements before synchronisation and show any unknown feature identifiers. Receiving an unsupported requirement also stops the active replication. +- We can now keep the file properties used by Hidden File Sync and Customisation Sync private in CouchDB. + - **Encrypt internal file Properties** extends E2EE V2 and Property Encryption to their paths, times, sizes, and Chunk references. + - Existing configurations keep this preference disabled. New Vaults enable it for use when the required encryption settings are active. + - Update every synchronising device before enabling it. It protects future writes; a manual remote Rebuild is strongly recommended to protect existing properties. +- We can now see which unsupported feature prevents a client from synchronising with CouchDB. + - Clients check the features required by the remote before transferring data or resetting the local database for Fast Fetch. Receiving an unsupported requirement also stops active replication. ## 1.0.32 From 126d6eadb858a79a08ad7f600061e54fc8d31196 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 28 Sep 2026 08:09:41 +0000 Subject: [PATCH 8/8] Fix percent-prefixed E2EE passphrase persistence --- package-lock.json | 8 +++--- package.json | 2 +- test/e2e-obsidian/README.md | 2 +- .../scripts/couchdb-manual-setup-workflow.ts | 26 ++++++++++++++++--- updates.md | 5 ++++ 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 248985a4..6b81bcf3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.30", + "@vrtmrz/livesync-commonlib": "0.1.31", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4567,9 +4567,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.30", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.30.tgz", - "integrity": "sha512-GCVLn/qGGJnJdm9gdM8MigvbE/2tjpgLuekqqr0iPsSOAUrIZNYcWq31KfUwrNsn5L4lLy+aMy0LhyWyBvK6yw==", + "version": "0.1.31", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.31.tgz", + "integrity": "sha512-ZFspZQmVlsCD41us7OdVc6Fl7T8ckCMiz/BKjif7pE7FO6/vJ4hhEGfcYRSYY+2iyuWyUklJcMgW7nEsO4p2UA==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 20d89dd7..6162c67d 100644 --- a/package.json +++ b/package.json @@ -185,7 +185,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.30", + "@vrtmrz/livesync-commonlib": "0.1.31", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index d557388e..39128c1b 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -137,7 +137,7 @@ The mobile pass uses Obsidian's `app.emulateMobile(true)`, a 390 by 844 CSS-pixe The same workflow checks the two remote-activity status boundaries. It first holds a real CouchDB request at the selected fetch implementation and confirms that `🌐N` is visible while `📲` is absent. It then holds the real one-shot replication immediately before its replicator call, confirms that `📲` is visible while no physical request is active, releases it, and requires the finite and bounded activity counts to return to zero, the request and response counts to balance, and both indicators to disappear. Finally, it creates a remote-only chunk, holds the real on-demand fetch immediately before its remote call, makes the same logical active and idle assertions, and verifies that the fetched chunk is written into the local database. These gates make the active states deterministic without replacing the remote request or operation. -`test:e2e:obsidian:couchdb-manual-setup-workflow` follows the visible onboarding path for the first device when no Setup URI is available. It enters end-to-end encryption and CouchDB details, runs the read-only `Check server requirements` step, requires the prepared fixture to pass without applying a server fix, and lets the onboarding connection test create the named database. After Rebuild completes on the first device, it creates an ordinary note, asks that working device to generate a Setup URI for a second device, completes Fetch there, and verifies a bidirectional note round-trip. The workflow captures each decision point and the expanded server-check result; password controls remain visually masked. +`test:e2e:obsidian:couchdb-manual-setup-workflow` follows the visible onboarding path for the first device when no Setup URI is available. It enters end-to-end encryption and CouchDB details, runs the read-only `Check server requirements` step, requires the prepared fixture to pass without applying a server fix, and lets the onboarding connection test create the named database. After Rebuild completes on the first device, it creates an ordinary note, asks that working device to generate a Setup URI for a second device, completes Fetch there, and verifies a bidirectional note round-trip. The workflow captures each decision point and the expanded server-check result; password controls remain visually masked. It uses an E2EE passphrase beginning with `%`, confirms that the saved settings do not contain it in plain text, and checks that Obsidian restores it after restarting with the first Vault. If this status workflow fails while Obsidian is running, it writes a full-page screenshot and a JSON snapshot of the status text and counters under `/tmp/obsidian-livesync-e2e`. The dialogue-mount workflow leaves desktop and mobile screenshots for both representative Svelte routes, the Hidden File Sync workflow captures the successfully displayed JSON Resolve dialogue before selecting an option, and the Security Seed reconnect workflow captures each significant application state. The suite therefore records representative evidence without capturing every interaction. Set `E2E_OBSIDIAN_DIAGNOSTICS_DIR` to use another directory. diff --git a/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts b/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts index f604059c..4e1945b4 100644 --- a/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts +++ b/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts @@ -39,6 +39,7 @@ process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "90000"; process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "30000"; const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_SETUP_URI_TIMEOUT_MS ?? 30000); +const e2eePassphrase = `%${randomBytes(24).toString("base64url")}`; const notePath = "E2E/manual-couchdb/from-first-device.md"; const noteContent = "# Manual CouchDB setup\n\nThis note was sent by the manually configured first device.\n"; const returnNotePath = "E2E/manual-couchdb/from-second-device.md"; @@ -147,8 +148,7 @@ async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig, .locator('input[type="checkbox"]') .first() .check({ timeout: uiTimeoutMs }); - const passphraseValue = randomBytes(24).toString("base64url"); - await passphraseInput.fill(passphraseValue); + await passphraseInput.fill(e2eePassphrase); const passwordToggle = encryption.locator("button.sls-password-toggle"); await passwordToggle.click({ timeout: uiTimeoutMs }); assertEqual( @@ -158,7 +158,7 @@ async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig, ); assertEqual( await passphraseInput.inputValue(), - passphraseValue, + e2eePassphrase, "Toggling visibility changed the passphrase value." ); await passwordToggle.click({ timeout: uiTimeoutMs }); @@ -169,7 +169,7 @@ async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig, ); assertEqual( await passphraseInput.inputValue(), - passphraseValue, + e2eePassphrase, "Re-masking the passphrase changed its value." ); }); @@ -301,6 +301,23 @@ async function assertPersistedE2EE(vault: TemporaryVault): Promise { if (typeof persisted.encryptedPassphrase !== "string" || persisted.encryptedPassphrase.length === 0) { throw new Error("Manual CouchDB setup did not persist an encrypted E2EE passphrase."); } + if (JSON.stringify(persisted).includes(e2eePassphrase)) { + throw new Error("Manual CouchDB setup persisted the E2EE passphrase in plain text."); + } +} + +async function assertRestoredE2EEPassphrase(session: ObsidianLiveSyncSession, cliBinary: string): Promise { + const restored = await evalObsidianJson( + cliBinary, + [ + "(()=>{", + "const settings=app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings();", + `return JSON.stringify(settings.passphrase === ${JSON.stringify(e2eePassphrase)});`, + "})()", + ].join(""), + session.cliEnv + ); + assertEqual(restored, true, "The E2EE passphrase was not restored after Obsidian restarted."); } async function setRemotePreferredE2EEDisabled(context: RunnerContext): Promise { @@ -454,6 +471,7 @@ async function main(): Promise { session = await startUnconfiguredSession(context, vaultA); try { + await assertRestoredE2EEPassphrase(session, context.cliBinary); await scheduleRemoteOverwrite(session.remoteDebuggingPort); screenshots.push(await confirmRebuild(session.remoteDebuggingPort, e2eeRebuildCaptures)); screenshots.push( diff --git a/updates.md b/updates.md index cd0c4c1e..aec7077d 100644 --- a/updates.md +++ b/updates.md @@ -23,6 +23,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - We can now see which unsupported feature prevents a client from synchronising with CouchDB. - Clients check the features required by the remote before transferring data or resetting the local database for Fast Fetch. Receiving an unsupported requirement also stops active replication. +#### Fixed + +- We can now keep using an E2EE passphrase beginning with `%` after restarting Obsidian. (#1221) + - LiveSync encrypts it before saving the settings. If an earlier version saved it in plain text, re-enter the passphrase used to encrypt the existing data after updating. Treat that passphrase as exposed if the affected `data.json` was shared. + ## 1.0.32 27th September, 2026