From d139e01c4885c434d1ebbaaa505e2be52bdccef3 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 20 Oct 2026 10:08:45 +0000 Subject: [PATCH 01/28] fix: package.json to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-MARKDOWNIT-21277720 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b98c52f8..2e95db5a 100644 --- a/package.json +++ b/package.json @@ -198,7 +198,7 @@ "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", "idb": "^8.0.3", - "markdown-it": "^14.3.2", + "markdown-it": "^14.3.3", "minimatch": "^10.2.5", "obsidian": "^1.13.1", "octagonal-wheels": "^0.1.55", From b35ac635d09a44cb00c0ee5e442be1fd2e89aabe Mon Sep 17 00:00:00 2001 From: Junker der Provinz <133605895+junkerderprovinz@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:25:44 +0200 Subject: [PATCH 02/28] Read Setup URI generator values from LIVESYNC_ variables hostname, username, and the other unprefixed names collide with shell and system variables (fish makes hostname read-only, Windows already defines USERNAME). The generator now reads LIVESYNC_HOST, LIVESYNC_USER, LIVESYNC_PASS, LIVESYNC_DB, and LIVESYNC_PASSPHRASE first and falls back to the old names, so existing instructions keep working. Co-authored-by: rwbru <59871009+rwbru@users.noreply.github.com> --- docs/setup_object_storage.md | 2 +- docs/setup_own_server.md | 17 ++++--- docs/setup_p2p.md | 2 +- updates.md | 7 +++ utils/readme.md | 18 ++++--- utils/setup/generate_setup_uri.test.ts | 70 ++++++++++++++++++++++++++ utils/setup/generate_setup_uri.ts | 22 +++++--- 7 files changed, 114 insertions(+), 24 deletions(-) diff --git a/docs/setup_object_storage.md b/docs/setup_object_storage.md index cad31264..c2a5237b 100644 --- a/docs/setup_object_storage.md +++ b/docs/setup_object_storage.md @@ -24,7 +24,7 @@ export secret_key= export bucket=vault-data export region=auto export bucket_prefix=my-vault -export passphrase= +export LIVESYNC_PASSPHRASE= export uri_passphrase= deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/docs/setup_own_server.md b/docs/setup_own_server.md index 1b7d0e9c..6924ec55 100644 --- a/docs/setup_own_server.md +++ b/docs/setup_own_server.md @@ -128,6 +128,9 @@ export database=obsidiannotes curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | bash ``` +> [!TIP] +> In fish, `hostname` is read-only and cannot be exported. Pass the values through `env` instead: `curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | env hostname=http://localhost:5984 username=<...> password=<...> database=obsidiannotes bash`. + If it results like the following: ``` CouchDB provisioning completed. @@ -173,20 +176,20 @@ Now `https://tiles-photograph-routine-groundwater.trycloudflare.com` is our serv ### 1. Generate the setup URI on a desktop device or server -The `username` and `password` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. +The `LIVESYNC_USER` and `LIVESYNC_PASS` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. ```bash -export hostname=https://tiles-photograph-routine-groundwater.trycloudflare.com -export database=obsidiannotes -export username= -export password= -export passphrase= +export LIVESYNC_HOST=https://tiles-photograph-routine-groundwater.trycloudflare.com +export LIVESYNC_DB=obsidiannotes +export LIVESYNC_USER= +export LIVESYNC_PASS= +export LIVESYNC_PASSPHRASE= export uri_passphrase= # Optional deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` > [!TIP] -> `passphrase` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel. +> `LIVESYNC_PASSPHRASE` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel. > > If `uri_passphrase` is omitted, the generator creates a cryptographically random value and prints it once. diff --git a/docs/setup_p2p.md b/docs/setup_p2p.md index 4797f420..c755ce0a 100644 --- a/docs/setup_p2p.md +++ b/docs/setup_p2p.md @@ -130,7 +130,7 @@ export remote_type=p2p export p2p_relays=wss://relay.example.com export p2p_room_id= # Optional; generated when omitted export p2p_passphrase= # Optional; generated when omitted -export passphrase= +export LIVESYNC_PASSPHRASE= export uri_passphrase= deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/updates.md b/updates.md index ca2e3497..d2575c27 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Setup tools + +#### Fixed + +- We can generate a Setup URI from fish, PowerShell, or cmd without the connection details colliding with `hostname` or `USERNAME`. (#817) + - The generator reads `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE`. The previous names are still read when these are not set. + ## 1.0.39 20th October, 2026 diff --git a/utils/readme.md b/utils/readme.md index 6ee10742..22439215 100644 --- a/utils/readme.md +++ b/utils/readme.md @@ -31,21 +31,23 @@ Authentication and other non-retryable HTTP failures stop immediately. Network a The existing `flyio/generate_setupuri.ts` path remains a CouchDB-only compatibility wrapper for the Fly.io deployment script. +The connection and Vault encryption values use a `LIVESYNC_` prefix so that they do not collide with shell or system variables such as `hostname` and `USERNAME`. The unprefixed names `hostname`, `username`, `password`, `database`, and `passphrase` are still read when the prefixed ones are not set. + The generator creates a fresh random ID key by default and includes it in the encrypted Setup URI. It prints a tagged `sls-id-v1:` recovery code. Set `id_recovery_code` to that code when generating another URI for the same Vault; a new run without it creates a different key. This restores only the ID key: reuse the original connection details too. For P2P, provide the original `p2p_room_id` and `p2p_passphrase` because omitted values are generated afresh. Set `id_mode=legacy` to generate a URI with the previous ID behaviour. `id_mode=legacy` and `id_recovery_code` cannot be combined. Keep the recovery code private and retain it if every device might be lost. ### CouchDB ```sh -export hostname=https://couch.example.com -export username=couchdb-admin-username -export password=couchdb-admin-password -export database=obsidiannotes -export passphrase=a-strong-vault-encryption-passphrase +export LIVESYNC_HOST=https://couch.example.com +export LIVESYNC_USER=couchdb-admin-username +export LIVESYNC_PASS=couchdb-admin-password +export LIVESYNC_DB=obsidiannotes +export LIVESYNC_PASSPHRASE=a-strong-vault-encryption-passphrase export uri_passphrase=a-separate-setup-uri-passphrase # Optional deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` -If `uri_passphrase` is omitted, the tool generates and prints a cryptographically random one. Store the Setup URI and its passphrase separately. The `passphrase` value protects synchronised Vault data and must also be stored safely. +If `uri_passphrase` is omitted, the tool generates and prints a cryptographically random one. Store the Setup URI and its passphrase separately. The `LIVESYNC_PASSPHRASE` value protects synchronised Vault data and must also be stored safely. The generator defaults to `uri_mode=ephemeral`. The URI opens only during the current fixed seven-day UTC window, and the tool prints its exact end time. It may have less than seven days remaining when generated. For an indefinitely reusable URI, set `uri_mode=persistent` before running the command. Persistent uses the existing encrypted format, which older clients that already support that format can read. The time condition controls opening the URI; it does not revoke settings or credentials after import. @@ -59,7 +61,7 @@ export secret_key= export bucket=vault-data export region=auto export bucket_prefix=team-a # Optional -export passphrase= +export LIVESYNC_PASSPHRASE= deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` @@ -69,7 +71,7 @@ Optional Object Storage variables are `use_custom_request_handler`, `force_path_ ```sh export remote_type=p2p -export passphrase= +export LIVESYNC_PASSPHRASE= deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` diff --git a/utils/setup/generate_setup_uri.test.ts b/utils/setup/generate_setup_uri.test.ts index 2bf802ff..66ced671 100644 --- a/utils/setup/generate_setup_uri.test.ts +++ b/utils/setup/generate_setup_uri.test.ts @@ -8,6 +8,76 @@ function assert(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message); } +Deno.test("reads LIVESYNC_ variables before the unprefixed names", async () => { + const generated = await generateSetupURI({ + LIVESYNC_HOST: "https://couch.example.test", + LIVESYNC_USER: "alice", + LIVESYNC_PASS: "couch-secret", + LIVESYNC_DB: "notes", + LIVESYNC_PASSPHRASE: "vault-secret", + hostname: "https://shell.example.test", + username: "shell-user", + password: "shell-secret", + database: "shell-db", + passphrase: "shell-passphrase", + uri_passphrase: "setup-secret", + }); + const decoded = await decodeSettingsFromSetupURI( + generated.setupURI, + generated.setupPassphrase, + ); + assert(decoded, "Commonlib could not decode the CouchDB Setup URI"); + const effective = { ...DEFAULT_SETTINGS, ...decoded }; + assert( + effective.couchDB_URI === "https://couch.example.test" && + effective.couchDB_USER === "alice" && + effective.couchDB_PASSWORD === "couch-secret" && + effective.couchDB_DBNAME === "notes", + "the CouchDB connection did not come from the LIVESYNC_ variables", + ); + assert( + effective.passphrase === "vault-secret", + "the Vault passphrase did not come from LIVESYNC_PASSPHRASE", + ); +}); + +Deno.test("falls back to the unprefixed names when LIVESYNC_ variables are blank", async () => { + const generated = await generateSetupURI({ + LIVESYNC_HOST: " ", + hostname: "https://couch.example.test", + username: "alice", + password: "couch-secret", + database: "notes", + passphrase: "vault-secret", + uri_passphrase: "setup-secret", + }); + const decoded = await decodeSettingsFromSetupURI( + generated.setupURI, + generated.setupPassphrase, + ); + assert(decoded, "Commonlib could not decode the CouchDB Setup URI"); + const effective = { ...DEFAULT_SETTINGS, ...decoded }; + assert( + effective.couchDB_URI === "https://couch.example.test" && + effective.couchDB_USER === "alice" && + effective.couchDB_PASSWORD === "couch-secret" && + effective.couchDB_DBNAME === "notes" && + effective.passphrase === "vault-secret", + "the generator did not fall back to the unprefixed names", + ); +}); + +Deno.test("names both variables when a CouchDB value is missing", async () => { + let rejected = false; + try { + await generateSetupURI({ uri_passphrase: "setup-secret" }); + } catch (error) { + rejected = error instanceof Error && + error.message === "LIVESYNC_HOST (or hostname) is required"; + } + assert(rejected, "the generator did not name both host variables"); +}); + Deno.test("generates an Object Storage Setup URI with a selected S3 profile", async () => { const generated = await generateSetupURI({ remote_type: "s3", diff --git a/utils/setup/generate_setup_uri.ts b/utils/setup/generate_setup_uri.ts index 6bf1ebdd..14c67724 100644 --- a/utils/setup/generate_setup_uri.ts +++ b/utils/setup/generate_setup_uri.ts @@ -64,9 +64,17 @@ function configureIdDerivation( function requireValue( environment: SetupGeneratorEnvironment, name: string, + legacyName?: string, ): string { - const value = environment[name]?.trim(); - if (!value) throw new Error(`${name} is required`); + const value = environment[name]?.trim() || + (legacyName && environment[legacyName]?.trim()); + if (!value) { + throw new Error( + legacyName + ? `${name} (or ${legacyName}) is required` + : `${name} is required`, + ); + } return value; } @@ -97,7 +105,7 @@ function applyEncryptedVaultSettings( Object.assign(settings, { isConfigured: true, encrypt: true, - passphrase: requireValue(environment, "passphrase"), + passphrase: requireValue(environment, "LIVESYNC_PASSPHRASE", "passphrase"), usePathObfuscation: true, }); } @@ -107,10 +115,10 @@ function createCouchDBSettings( ): ObsidianLiveSyncSettings { const settings = createNewVaultSettings(); Object.assign(settings, PREFERRED_SETTING_SELF_HOSTED, { - couchDB_URI: requireValue(environment, "hostname"), - couchDB_USER: requireValue(environment, "username"), - couchDB_PASSWORD: requireValue(environment, "password"), - couchDB_DBNAME: requireValue(environment, "database"), + couchDB_URI: requireValue(environment, "LIVESYNC_HOST", "hostname"), + couchDB_USER: requireValue(environment, "LIVESYNC_USER", "username"), + couchDB_PASSWORD: requireValue(environment, "LIVESYNC_PASS", "password"), + couchDB_DBNAME: requireValue(environment, "LIVESYNC_DB", "database"), batchSave: true, periodicReplication: true, syncOnStart: true, From e83833d4e8663d16ab95d626376c8a7ee45ee188 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 01:00:22 +0000 Subject: [PATCH 03/28] Fix Automatic mode prerequisites and plug-in initialisation targets --- docs/settings.md | 2 + docs/tips/hidden-file-sync.md | 6 + .../messages/LiveSyncProvisionalMessages.ts | 3 + src/features/ConfigSync/PluginPane.svelte | 17 +- .../ConfigSync/getAutomaticSyncTargetFiles.ts | 13 + .../getAutomaticSyncTargetFiles.unit.spec.ts | 58 ++++ .../HiddenFileSync/CmdHiddenFileSync.ts | 14 +- .../CmdHiddenFileSync.unit.spec.ts | 190 ++++++++++- test/e2e-obsidian/README.md | 4 + .../scripts/customisation-sync.ts | 316 +++++++++++++++++- .../scripts/hidden-file-snippet-sync.ts | 124 +++++++ updates.md | 7 + 12 files changed, 735 insertions(+), 19 deletions(-) create mode 100644 src/features/ConfigSync/getAutomaticSyncTargetFiles.ts create mode 100644 src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts diff --git a/docs/settings.md b/docs/settings.md index 0885589c..bdad0692 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -783,6 +783,8 @@ When a saved setting changes whether a normal file can be reflected, LiveSync re Customisation Sync is a supported, advanced opt-in feature. Its current per-file implementation is covered by a two-Vault real-Obsidian workflow for snippets, configuration files, and plug-in files. Hidden File Sync is a separate feature with different setup, selection, and conflict behaviour; do not use both features to manage the same files. +**Automatic** delegates an item's files to Hidden File Sync. Enable and initialise Hidden File Sync in **Sync Settings** on this device first, after reviewing its [target and ignore patterns](./tips/hidden-file-sync.md#review-the-file-selection). Choosing Automatic while Hidden File Sync is disabled displays a Notice and leaves the current mode unchanged. Existing Automatic assignments, including those imported from another device, remain saved while Hidden File Sync is disabled. + ### 1. Customisation Sync #### Device name diff --git a/docs/tips/hidden-file-sync.md b/docs/tips/hidden-file-sync.md index 6b270da2..c8dccf1c 100644 --- a/docs/tips/hidden-file-sync.md +++ b/docs/tips/hidden-file-sync.md @@ -68,6 +68,12 @@ Hidden File Sync must be enabled independently after ordinary setup on each devi If you later run Rebuild or Fetch as a recovery operation, expect optional features to be disabled again. Complete ordinary recovery first, then repeat this guide deliberately. +## Automatic in Customisation Sync + +**Automatic** delegates an item's files to Hidden File Sync. Enable Hidden File Sync on this device through the steps above before choosing Automatic. If Hidden File Sync is disabled, the choice displays a Notice without changing the item's mode or starting initialisation. Previously saved Automatic assignments remain saved, so this prerequisite also applies after importing settings from another device. + +Once Hidden File Sync is enabled, choose the item's initial action in Customisation Sync. Selecting a whole plug-in applies that action to both its **MAIN** and **DATA** selections. + ## Related settings The [settings reference](../settings.md#7-hidden-files-advanced) describes periodic scanning, scanning before replication, notification suppression, selectors, and overwrite patterns. Change those controls only after the basic Hidden File Sync path is working. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 34ae100b..650dea93 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,9 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files.": + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files.", + "Hidden File Sync initialisation was not started.": "Hidden File Sync initialisation was not started.", "Preparing follow-up report…": "Preparing follow-up report…", "Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.", "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.": diff --git a/src/features/ConfigSync/PluginPane.svelte b/src/features/ConfigSync/PluginPane.svelte index 18a65109..5be97ac9 100644 --- a/src/features/ConfigSync/PluginPane.svelte +++ b/src/features/ConfigSync/PluginPane.svelte @@ -20,10 +20,14 @@ MODE_SHINY, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { normalizePath } from "@/deps"; - import { HiddenFileSync } from "@/features/HiddenFileSync/CmdHiddenFileSync.ts"; + import { + HiddenFileSync, + HIDDEN_FILE_SYNC_DISABLED_NOTICE, + } from "@/features/HiddenFileSync/CmdHiddenFileSync.ts"; import { LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger"; import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore.ts"; import { $msg as translateMessage } from "@/common/translation"; + import { getAutomaticSyncTargetFiles } from "./getAutomaticSyncTargetFiles.ts"; import { REPLICATION_PROGRESS_PRESENTATIONS, USER_INITIATED_REPLICATION_AUTHORITY, @@ -139,6 +143,11 @@ async function deleteData(data: IPluginDataExDisplay): Promise { return await addOn.deleteData(data); } + function checkAutomaticSyncAvailable() { + if (addOnHiddenFileSync.isThisModuleEnabled()) return true; + Logger(translateMessage(HIDDEN_FILE_SYNC_DISABLED_NOTICE), LOG_LEVEL_NOTICE); + return false; + } function askMode(evt: MouseEvent, title: string, key: string) { const menu = new Menu(); menu.addItem((item) => item.setTitle(title).setIsLabel(true)); @@ -149,6 +158,7 @@ item.setTitle(`${getIcon(mode as SYNC_MODE)}:${TITLES[mode]}`) .onClick((e) => { if (mode === MODE_AUTOMATIC) { + if (!checkAutomaticSyncAvailable()) return; askOverwriteModeForAutomatic(evt, key); } else { setMode(key, mode as SYNC_MODE); @@ -161,10 +171,11 @@ menu.showAtMouseEvent(evt); } function applyAutomaticSync(key: string, direction: "pushForce" | "pullForce" | "safe") { + if (!checkAutomaticSyncAvailable()) return; setMode(key, MODE_AUTOMATIC); const configDir = normalizePath(plugin.core.services.API.getSystemConfigDir()); - const files = (plugin.core.settings.pluginSyncExtendedSetting[key]?.files ?? []).map((e) => `${configDir}/${e}`); - addOnHiddenFileSync.initialiseInternalFileSync(direction, true, files); + const files = getAutomaticSyncTargetFiles(key, configDir, plugin.core.settings.pluginSyncExtendedSetting); + void addOnHiddenFileSync.initialiseInternalFileSync(direction, true, files); } function askOverwriteModeForAutomatic(evt: MouseEvent, key: string) { const menu = new Menu(); diff --git a/src/features/ConfigSync/getAutomaticSyncTargetFiles.ts b/src/features/ConfigSync/getAutomaticSyncTargetFiles.ts new file mode 100644 index 00000000..181e3669 --- /dev/null +++ b/src/features/ConfigSync/getAutomaticSyncTargetFiles.ts @@ -0,0 +1,13 @@ +import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; + +export function getAutomaticSyncTargetFiles( + key: string, + configDir: string, + entries: ObsidianLiveSyncSettings["pluginSyncExtendedSetting"] +): string[] { + const keys = key.startsWith("PLUGIN_ALL/") + ? [`PLUGIN_DATA/${key.substring("PLUGIN_ALL/".length)}`, `PLUGIN_MAIN/${key.substring("PLUGIN_ALL/".length)}`] + : [key]; + const files = new Set(keys.flatMap((entryKey) => entries[entryKey]?.files ?? [])); + return [...files].map((file) => `${configDir}/${file}`); +} diff --git a/src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts b/src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts new file mode 100644 index 00000000..a1c8265c --- /dev/null +++ b/src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { MODE_AUTOMATIC, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { getAutomaticSyncTargetFiles } from "./getAutomaticSyncTargetFiles.ts"; + +function createEntries(): ObsidianLiveSyncSettings["pluginSyncExtendedSetting"] { + return { + "CONFIG/app.json": { key: "CONFIG/app.json", mode: MODE_AUTOMATIC, files: ["app.json"] }, + "PLUGIN_MAIN/example": { + key: "PLUGIN_MAIN/example", + mode: MODE_AUTOMATIC, + files: ["plugins/example/main.js", "plugins/example/manifest.json"], + }, + "PLUGIN_DATA/example": { + key: "PLUGIN_DATA/example", + mode: MODE_AUTOMATIC, + files: ["plugins/example/data.json"], + }, + "PLUGIN_ETC/example": { + key: "PLUGIN_ETC/example", + mode: MODE_AUTOMATIC, + files: ["plugins/example/extra.json"], + }, + "PLUGIN_MAIN/other": { + key: "PLUGIN_MAIN/other", + mode: MODE_AUTOMATIC, + files: ["plugins/other/main.js"], + }, + }; +} + +describe("getAutomaticSyncTargetFiles", () => { + it("initialises a single selected item in the current configuration directory", () => { + expect(getAutomaticSyncTargetFiles("CONFIG/app.json", ".custom-config", createEntries())).toEqual([ + ".custom-config/app.json", + ]); + }); + + it("initialises both saved child selections when the whole plug-in is selected", () => { + expect(getAutomaticSyncTargetFiles("PLUGIN_ALL/example", ".obsidian", createEntries())).toEqual([ + ".obsidian/plugins/example/data.json", + ".obsidian/plugins/example/main.js", + ".obsidian/plugins/example/manifest.json", + ]); + }); + + it("retains the available child files when the other child has no saved entry", () => { + const entries = createEntries(); + delete entries["PLUGIN_DATA/example"]; + expect(getAutomaticSyncTargetFiles("PLUGIN_ALL/example", ".obsidian", entries)).toEqual([ + ".obsidian/plugins/example/main.js", + ".obsidian/plugins/example/manifest.json", + ]); + }); + + it("keeps an empty selection empty rather than including unrelated plug-ins", () => { + expect(getAutomaticSyncTargetFiles("PLUGIN_ALL/absent", ".obsidian", createEntries())).toEqual([]); + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.ts index 5b945c69..48564566 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.ts @@ -69,6 +69,9 @@ type HiddenFileInitialisationProgress = { const HIDDEN_FILE_NOTICE_GROUP = "hidden-file-changes"; const HIDDEN_FILE_NOTICE_DURATION_MS = 20_000; +export const HIDDEN_FILE_SYNC_DISABLED_NOTICE = + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files."; + function getComparingMTime( doc: (MetaEntry | LoadedEntry | false) | UXFileInfo | UXStat | null | undefined, includeDeleted = false @@ -1444,6 +1447,11 @@ Offline Changed files: ${files.length}`; initialisationProgress?: HiddenFileInitialisationProgress ) { const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; + if (!this.isThisModuleEnabled()) { + this._log($msg(HIDDEN_FILE_SYNC_DISABLED_NOTICE), logLevel); + initialisationProgress?.done($msg("Hidden File Sync initialisation was not started.")); + return false; + } const p = initialisationProgress ?? this._progress("[⚙ Initialise]\n", logLevel); // p.log("Resolving conflicts before starting..."); // await this.resolveConflictOnInternalFiles(); @@ -1494,6 +1502,7 @@ Offline Changed files: ${files.length}`; await this.scanAllDatabaseChanges(showChildNotices, true, false); } p.done(); + return true; } // <-- Initialization functions @@ -1925,6 +1934,7 @@ Offline Changed files: ${files.length}`; async configureHiddenFileSync(mode: OptionalSyncFeatureMode) { let initialisationProgress: HiddenFileInitialisationProgress | undefined; + let initialised = false; let result: ConfigureHiddenFileSyncResult; try { result = await configureHiddenFileSyncMode(mode, { @@ -1954,7 +1964,7 @@ Offline Changed files: ${files.length}`; ); }, initialise: async (direction) => { - await this.initialiseInternalFileSync(direction, true, false, initialisationProgress); + initialised = await this.initialiseInternalFileSync(direction, true, false, initialisationProgress); initialisationProgress = undefined; }, }); @@ -1962,7 +1972,7 @@ Offline Changed files: ${files.length}`; initialisationProgress?.done("Failed"); throw error; } - if (result == "ignored" || result == "disabled") { + if (result == "ignored" || result == "disabled" || !initialised) { return; } // this.plugin.settings.useAdvancedMode = true; diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts index ad664a0f..9018b222 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts @@ -99,6 +99,115 @@ function createHiddenRevisionOperation() { } describe("HiddenFileSync configuration-change notices", () => { + it.each(["pushForce", "pullForce", "safe"] as const)( + "refuses %s initialisation before processing files while Hidden File Sync is disabled", + async (direction) => { + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const path = ".obsidian/example.json" as FilePath; + const operations = { + rebuildFromStorage: vi.fn(async () => [path]), + rebuildFromDatabase: vi.fn(async () => []), + rebuildMerging: vi.fn(async () => [path]), + adoptCurrentStorageFilesAsProcessed: vi.fn(async () => undefined), + adoptCurrentDatabaseFilesAsProcessed: vi.fn(async () => undefined), + scanAllStorageChanges: vi.fn(async () => undefined), + scanAllDatabaseChanges: vi.fn(async () => undefined), + }; + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + const settings = { syncInternalFiles: false }; + const createProgress = vi.fn(() => progress); + const log = vi.fn(); + Object.assign(hiddenFileSync, { + core: { settings }, + _progress: createProgress, + _log: log, + ...operations, + }); + + const result = await hiddenFileSync.initialiseInternalFileSync(direction, true, [path]); + + for (const operation of Object.values(operations)) { + expect(operation).not.toHaveBeenCalled(); + } + expect(result).toBe(false); + expect(settings.syncInternalFiles).toBe(false); + expect(createProgress).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith(expect.stringContaining("Hidden File Sync is disabled"), LOG_LEVEL_NOTICE); + expect(progress.done).not.toHaveBeenCalled(); + } + ); + + it("finishes an existing preparation Notice as not started when initialisation is disabled", async () => { + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const createProgress = vi.fn(); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { settings: { syncInternalFiles: false } }, + _progress: createProgress, + _log: vi.fn(), + }); + + const result = await hiddenFileSync.initialiseInternalFileSync("safe", true, false, progress); + + expect(result).toBe(false); + expect(createProgress).not.toHaveBeenCalled(); + expect(progress.done).toHaveBeenCalledExactlyOnceWith("Hidden File Sync initialisation was not started."); + }); + + it("does not apply a stored deletion when initialisation is requested while disabled", async () => { + const path = ".obsidian/example.json" as FilePath; + const deleted: MetaEntry = { + _id: "i:example" as DocumentID, + _rev: "2-deleted", + path: `i:${path}` as FilePathWithPrefix, + type: "plain", + ctime: 1, + mtime: 10, + size: 0, + deleted: true, + children: [], + eden: {}, + }; + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const deleteFile = vi.fn(async () => "OK" as const); + const updateDeletion = vi.fn(); + const getFiles = vi.fn(async () => [deleted]); + const processedStorage = new Map([[path, "storage-processed"]]); + const processedDatabase = new Map([[path, "database-processed"]]); + const resetProcessed = vi.fn(() => processedDatabase.clear()); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { + settings: { syncInternalFiles: false }, + services: { vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) } }, + }, + _progress: vi.fn(() => progress), + _log: vi.fn(), + _verbose: vi.fn(), + _fileInfoLastProcessed: processedStorage, + _databaseInfoLastProcessed: processedDatabase, + getAllDatabaseFiles: getFiles, + getPath: (doc: MetaEntry) => doc.path, + resetLastProcessedDatabase: resetProcessed, + serializedForEvent: async (_path: unknown, operation: () => Promise) => await operation(), + __deleteFile: deleteFile, + updateLastProcessedDeletion: updateDeletion, + adoptCurrentStorageFilesAsProcessed: vi.fn(), + adoptCurrentDatabaseFilesAsProcessed: vi.fn(), + scanAllStorageChanges: vi.fn(), + scanAllDatabaseChanges: vi.fn(), + }); + + await hiddenFileSync.initialiseInternalFileSync("pullForce", true); + + expect(deleteFile).not.toHaveBeenCalled(); + expect(updateDeletion).not.toHaveBeenCalled(); + expect(getFiles).not.toHaveBeenCalled(); + expect(resetProcessed).not.toHaveBeenCalled(); + expect(processedStorage).toEqual(new Map([[path, "storage-processed"]])); + expect(processedDatabase).toEqual(new Map([[path, "database-processed"]])); + }); + it("shows manual Hidden File Sync commands only when the feature, Advanced mode, and runtime are ready", () => { const commands: Array<{ id: string; @@ -136,6 +245,13 @@ describe("HiddenFileSync configuration-change notices", () => { expect(command?.checkCallback?.(true)).toBe(false); } + settings.useAdvancedMode = true; + for (const commandId of commandIds) { + const command = commands.find(({ id }) => id === commandId); + expect(command?.checkCallback?.(true)).toBe(false); + expect(command?.checkCallback?.(false)).toBe(false); + } + settings.syncInternalFiles = true; settings.useAdvancedMode = true; for (const commandId of commandIds) { @@ -252,6 +368,7 @@ describe("HiddenFileSync configuration-change notices", () => { const scanAllDatabaseChanges = vi.fn(async () => undefined); const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; Object.assign(hiddenFileSync, { + core: { settings: { syncInternalFiles: true } }, _progress: vi.fn(() => progress), rebuildMerging, adoptCurrentStorageFilesAsProcessed, @@ -260,14 +377,83 @@ describe("HiddenFileSync configuration-change notices", () => { scanAllDatabaseChanges, }); - await hiddenFileSync.initialiseInternalFileSync("safe", true); + const result = await hiddenFileSync.initialiseInternalFileSync("safe", true); + expect(result).toBe(true); expect(rebuildMerging).toHaveBeenCalledWith(false, false); expect(scanAllStorageChanges).toHaveBeenCalledWith(false, true, false); expect(scanAllDatabaseChanges).toHaveBeenCalledWith(false, true, false); expect(progress.done).toHaveBeenCalledOnce(); }); + it.each(["pushForce", "pullForce", "safe"] as const)( + "retains the selected files and completes %s initialisation when enabled", + async (direction) => { + const files = [".obsidian/plugins/example/main.js", ".obsidian/plugins/example/data.json"] as FilePath[]; + const entries = files.map((path) => ({ path: `i:${path}` as FilePathWithPrefix })); + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const rebuildFromStorage = vi.fn(async () => files); + const rebuildFromDatabase = vi.fn(async () => entries); + const rebuildMerging = vi.fn(async () => files); + const adoptStorage = vi.fn(); + const adoptDatabase = vi.fn(); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { settings: { syncInternalFiles: true } }, + _progress: vi.fn(() => progress), + getPath: (entry: { path: FilePathWithPrefix }) => entry.path, + rebuildFromStorage, + rebuildFromDatabase, + rebuildMerging, + adoptCurrentStorageFilesAsProcessed: adoptStorage, + adoptCurrentDatabaseFilesAsProcessed: adoptDatabase, + scanAllStorageChanges: vi.fn(), + scanAllDatabaseChanges: vi.fn(), + }); + + const result = await hiddenFileSync.initialiseInternalFileSync(direction, true, files); + + expect(result).toBe(true); + if (direction === "pushForce") { + expect(rebuildFromStorage).toHaveBeenCalledExactlyOnceWith(false, files, false); + expect(rebuildFromDatabase).not.toHaveBeenCalled(); + expect(rebuildMerging).not.toHaveBeenCalled(); + } else if (direction === "pullForce") { + expect(rebuildFromDatabase).toHaveBeenCalledExactlyOnceWith(false, files, false); + expect(rebuildFromStorage).not.toHaveBeenCalled(); + expect(rebuildMerging).not.toHaveBeenCalled(); + } else { + expect(rebuildMerging).toHaveBeenCalledExactlyOnceWith(false, files); + expect(rebuildFromStorage).not.toHaveBeenCalled(); + expect(rebuildFromDatabase).not.toHaveBeenCalled(); + } + expect(adoptStorage).toHaveBeenCalledExactlyOnceWith(files); + expect(adoptDatabase).toHaveBeenCalledExactlyOnceWith(files); + expect(progress.done).toHaveBeenCalledOnce(); + } + ); + + it("does not report completion when the initialisation entry point refuses to start", async () => { + vi.mocked(configureHiddenFileSyncMode).mockImplementation(async (_mode, handlers) => { + await handlers.enable(); + await handlers.initialise("safe"); + return "enabled"; + }); + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const log = vi.fn(); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { services: { setting: { applyPartial: vi.fn() } } }, + _progress: vi.fn(() => progress), + _log: log, + initialiseInternalFileSync: vi.fn(async () => false), + }); + + await hiddenFileSync.configureHiddenFileSync("MERGE"); + + expect(log).not.toHaveBeenCalledWith("Hidden File Sync initialisation completed.", expect.any(Number)); + }); + it("retirement guard: does not restore separate gathering and restart Notices", async () => { vi.mocked(configureHiddenFileSyncMode).mockImplementation(async (_mode, handlers) => { await handlers.enable(); @@ -286,7 +472,7 @@ describe("HiddenFileSync configuration-change notices", () => { const applyPartial = vi.fn(async () => { events.push("apply-settings"); }); - const initialiseInternalFileSync = vi.fn(async () => undefined); + const initialiseInternalFileSync = vi.fn(async () => true); const log = vi.fn(); const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; Object.assign(hiddenFileSync, { diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..77358281 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -248,10 +248,14 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. +Before enablement, the scenario directly requests initialisation while Hidden File Sync is disabled. It requires a refused result, unchanged local files, Metadata revisions, and processed markers, and no initialisation progress Notice. + `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. +`--case=automatic` separately checks that choosing **Automatic** while Hidden File Sync is disabled displays a Notice without changing modes, files, Metadata revisions, or processed markers. The fixture starts with a saved Automatic selection and requires it to survive start-up and both rejected actions. It also checks disabling the feature after opening **Initial Action**. After enablement, selecting a whole plug-in must initialise both **MAIN** and **DATA**, and transfer their files to the other Vault. + `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`. `test:e2e:obsidian:remote-feature-change` starts real Obsidian with continuous CouchDB replication, then changes the remote version document from generation 12 to generation 13 with an unknown feature. It waits for the control document to reach the local database and the active Replicator to retire, checks that another replication is refused, and verifies that an already accepted Vault note remains intact. After restarting the same Vault, the current remote declaration still blocks finite replication and the actual continuous connection attempt. No KV feature history is involved. It is a focused test outside `test:e2e:obsidian:local-suite`; recovery with a future compatible client remains a separate validation boundary. diff --git a/test/e2e-obsidian/scripts/customisation-sync.ts b/test/e2e-obsidian/scripts/customisation-sync.ts index 78fd7a96..f20391e2 100644 --- a/test/e2e-obsidian/scripts/customisation-sync.ts +++ b/test/e2e-obsidian/scripts/customisation-sync.ts @@ -1,6 +1,10 @@ import { mkdir, readFile, rm, utimes, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; -import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + MODE_AUTOMATIC, + VERSIONING_DOCID, + type ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication"; import { evalObsidianJson } from "../runner/cli.ts"; import { @@ -23,11 +27,12 @@ import { prepareRemote, pushLocalChanges, waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, } from "../runner/liveSyncWorkflow.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { waitForVisibleObsidianDialogue, withObsidianPage } from "../runner/ui.ts"; import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; -import type { Locator } from "playwright"; +import type { Locator, Page } from "playwright"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000"; process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "20000"; @@ -59,6 +64,18 @@ const pluginDir = ".obsidian/plugins/livesync-e2e-sample"; const pluginManifestPath = `${pluginDir}/manifest.json`; const pluginMainPath = `${pluginDir}/main.js`; const pluginStylesPath = `${pluginDir}/styles.css`; +const pluginDisplayName = "LiveSync E2E Sample"; +const pluginDataPath = `${pluginDir}/data.json`; +const pluginDataContent = JSON.stringify({ source: "customisation-sync-a", enabled: true }, null, 4) + "\n"; +const targetPluginDataContent = JSON.stringify({ source: "customisation-sync-b", enabled: true }, null, 4) + "\n"; +const existingAutomaticKey = "SNIPPET/livesync-e2e-existing-automatic.css"; +const existingAutomaticSelections: ObsidianLiveSyncSettings["pluginSyncExtendedSetting"] = { + [existingAutomaticKey]: { + key: existingAutomaticKey, + mode: MODE_AUTOMATIC, + files: ["snippets/livesync-e2e-existing-automatic.css"], + }, +}; const identicalSnippetPath = ".obsidian/snippets/livesync-customisation-e2e-identical.css"; const identicalSnippetContent = ".livesync-customisation-identical { color: #3d6f54; }\n"; const differentSnippetPath = ".obsidian/snippets/livesync-customisation-e2e-different.css"; @@ -104,7 +121,7 @@ const pluginFixtures = [ { path: pluginStylesPath, source: pluginStylesContent, target: targetPluginStylesContent }, ] as const; -type CustomisationSyncCase = "all" | "visibility" | "mtime"; +type CustomisationSyncCase = "all" | "visibility" | "mtime" | "automatic"; type CustomisationSyncTestGlobal = typeof globalThis & { app?: { commands?: { executeCommandById(commandId: string): boolean } }; @@ -131,6 +148,14 @@ type CustomisationScanResult = { files: string[]; }; +type AutomaticSyncSnapshot = { + syncInternalFiles: boolean; + pluginSyncExtendedSetting: Record; + hiddenFiles: { path: string; rev: string; deleted: boolean }[]; + processedFiles: [string, string | undefined][]; + processedDatabaseFiles: [string, string | undefined][]; +}; + async function writeVaultFile(vaultPath: string, path: string, content: string): Promise { const fullPath = join(vaultPath, path); await mkdir(dirname(fullPath), { recursive: true }); @@ -147,10 +172,11 @@ function selectedCase(): CustomisationSyncCase { if (args.length === 0) return "all"; const arg = args[0]; if (args.length !== 1 || !arg?.startsWith("--case=")) { - throw new Error("Usage: test:e2e:obsidian:customisation-sync [--case=all|visibility|mtime]"); + throw new Error("Usage: test:e2e:obsidian:customisation-sync [--case=all|visibility|mtime|automatic]"); } const selected = arg.slice("--case=".length); - if (selected === "all" || selected === "visibility" || selected === "mtime") return selected; + if (selected === "all" || selected === "visibility" || selected === "mtime" || selected === "automatic") + return selected; throw new Error(`Unknown Customisation Sync E2E case: ${selected}`); } @@ -244,6 +270,196 @@ async function inspectPluginFreshnessAndNewestSelection(session: ObsidianLiveSyn }); } +async function openAutomaticModeForPlugin(page: Page, pluginName: string): Promise { + const opened = await page.evaluate(() => + (globalThis as CustomisationSyncTestGlobal).app?.commands?.executeCommandById( + "obsidian-livesync:livesync-plugin-dialog-ex" + ) + ); + if (opened !== true) throw new Error("Could not open the Customisation Sync dialogue command."); + const dialogue = await waitForVisibleObsidianDialogue(page, "Customization Sync (Beta3)"); + const pluginRow = dialogue.locator(".labelrow").filter({ hasText: pluginName }).first(); + await pluginRow.waitFor({ state: "visible", timeout: 10000 }); + await pluginRow.locator("button.status").click(); + const automaticOption = page.locator(".menu-item").filter({ hasText: "Automatic" }).last(); + await automaticOption.waitFor({ state: "visible", timeout: 10000 }); + await automaticOption.click(); + return dialogue; +} + +async function selectAutomaticInitialAction(page: Page, label: "Overwrite Remote" | "Overwrite Local"): Promise { + const action = page.locator(".menu-item").filter({ hasText: label }).last(); + await action.waitFor({ state: "visible", timeout: 10000 }); + await action.click(); +} + +async function readAutomaticSyncSnapshot(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const hidden=core.getAddOn('HiddenFileSync');", + "const files=await hidden.getAllDatabaseFiles();", + "const processed=(cache)=>Array.from(cache.keys()).sort().map((path)=>[path,cache.get(path)]);", + "return JSON.stringify({", + "syncInternalFiles:core.settings.syncInternalFiles,", + "pluginSyncExtendedSetting:core.settings.pluginSyncExtendedSetting,", + "hiddenFiles:files.map((entry)=>{const path=hidden.getPath(entry);return {path:path.startsWith('i:')?path.slice(2):path,rev:entry._rev??'',deleted:!!(entry.deleted||entry._deleted)};}).sort((a,b)=>a.path.localeCompare(b.path)),", + "processedFiles:processed(hidden._fileInfoLastProcessed),", + "processedDatabaseFiles:processed(hidden._databaseInfoLastProcessed),", + "});", + "})()", + ].join(""), + env + ); +} + +async function setHiddenSyncEnabledForFixture( + cliBinary: string, + env: NodeJS.ProcessEnv, + enabled: boolean +): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + `core.settings.syncInternalFiles=${JSON.stringify(enabled)};`, + "return JSON.stringify({syncInternalFiles:core.settings.syncInternalFiles});", + "})()", + ].join(""), + env + ); +} + +async function inspectAutomaticSyncSelection( + context: RunnerContext, + session: ObsidianLiveSyncSession, + vault: TemporaryVault, + pluginName: string +): Promise { + const noticeText = + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files."; + const trackedFiles = [pluginStylesPath, pluginDataPath]; + const contentsBefore = await Promise.all(trackedFiles.map((path) => readVaultFile(vault.path, path))); + const disabledBefore = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + assertEqual( + JSON.stringify(disabledBefore.pluginSyncExtendedSetting[existingAutomaticKey]), + JSON.stringify(existingAutomaticSelections[existingAutomaticKey]), + "The fixture did not retain the saved Automatic selection while Hidden File Sync was disabled." + ); + + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await openAutomaticModeForPlugin(page, pluginDisplayName); + const notice = page.locator(".notice:visible").filter({ hasText: noticeText }).last(); + await notice.waitFor({ state: "visible", timeout: 10000 }); + const initialActionMenu = page.locator(".menu:visible").filter({ hasText: "Initial Action" }); + if ((await initialActionMenu.count()) !== 0) { + throw new Error("Automatic displayed an Initial Action menu while Hidden File Sync was disabled."); + } + await page.keyboard.press("Escape"); + await dialogue.waitFor({ state: "hidden", timeout: 10000 }); + await notice.click({ force: true }); + await notice.waitFor({ state: "hidden", timeout: 10000 }); + }); + + const disabledAfter = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + const contentsAfter = await Promise.all(trackedFiles.map((path) => readVaultFile(vault.path, path))); + if (JSON.stringify(disabledAfter) !== JSON.stringify(disabledBefore)) { + throw new Error( + `Automatic changed Hidden File Sync state while it was disabled: ${JSON.stringify({ + before: disabledBefore, + after: disabledAfter, + })}` + ); + } + assertEqual( + JSON.stringify(contentsAfter), + JSON.stringify(contentsBefore), + "Automatic changed plug-in files while Hidden File Sync was disabled." + ); + + // Exercise the second guard: the user may open Initial Action while enabled, + // then disable Hidden File Sync before choosing a direction. + await setHiddenSyncEnabledForFixture(context.cliBinary, session.cliEnv, true); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await openAutomaticModeForPlugin(page, pluginDisplayName); + const initialActionMenu = page.locator(".menu:visible").filter({ hasText: "Initial Action" }); + await initialActionMenu.waitFor({ state: "visible", timeout: 10000 }); + }); + await setHiddenSyncEnabledForFixture(context.cliBinary, session.cliEnv, false); + const beforeDisabledAction = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await selectAutomaticInitialAction(page, "Overwrite Remote"); + const notice = page.locator(".notice:visible").filter({ hasText: noticeText }).last(); + await notice.waitFor({ state: "visible", timeout: 10000 }); + await notice.click({ force: true }); + await notice.waitFor({ state: "hidden", timeout: 10000 }); + }); + const afterDisabledAction = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + if (JSON.stringify(afterDisabledAction) !== JSON.stringify(beforeDisabledAction)) { + throw new Error( + `Automatic changed Hidden File Sync state after it was disabled from Initial Action: ${JSON.stringify({ + before: beforeDisabledAction, + after: afterDisabledAction, + })}` + ); + } + assertEqual( + JSON.stringify(await Promise.all(trackedFiles.map((path) => readVaultFile(vault.path, path)))), + JSON.stringify(contentsBefore), + "Automatic changed plug-in files after Hidden File Sync was disabled from Initial Action." + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.keyboard.press("Escape"); + }); + + await setHiddenSyncEnabledForFixture(context.cliBinary, session.cliEnv, true); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await openAutomaticModeForPlugin(page, pluginDisplayName); + await selectAutomaticInitialAction(page, "Overwrite Remote"); + for (const label of ["MAIN", "DATA"]) { + const row = dialogue.locator(".filerow").filter({ hasText: label }).first(); + await row.locator(".statusnote").filter({ hasText: "Automatic" }).waitFor({ state: "visible" }); + } + await page.keyboard.press("Escape"); + await dialogue.waitFor({ state: "hidden", timeout: 10000 }); + }); + + for (const path of [pluginManifestPath, pluginMainPath, pluginStylesPath, pluginDataPath]) { + await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, path, { hidden: true }); + } + const enabled = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + const mainMode = enabled.pluginSyncExtendedSetting[`PLUGIN_MAIN/${pluginName}`]; + const dataMode = enabled.pluginSyncExtendedSetting[`PLUGIN_DATA/${pluginName}`]; + if (mainMode?.mode !== MODE_AUTOMATIC || dataMode?.mode !== MODE_AUTOMATIC) { + throw new Error( + `Automatic did not save both plug-in group child settings: ${JSON.stringify({ mainMode, dataMode })}` + ); + } + assertEqual( + JSON.stringify([...mainMode.files].sort()), + JSON.stringify([ + `plugins/${pluginName}/main.js`, + `plugins/${pluginName}/manifest.json`, + `plugins/${pluginName}/styles.css`, + ]), + "Automatic did not select the PLUGIN_MAIN files." + ); + assertEqual( + JSON.stringify([...dataMode.files].sort()), + JSON.stringify([`plugins/${pluginName}/data.json`]), + "Automatic did not select the PLUGIN_DATA file." + ); + assertEqual( + enabled.hiddenFiles.filter((entry) => entry.path.startsWith(pluginDir + "/")).length, + 4, + "Automatic did not initialise all files in the PLUGIN_ALL group." + ); + await pushLocalChanges(context.cliBinary, session.cliEnv); +} + async function removeVaultFile(vaultPath: string, path: string): Promise { await rm(join(vaultPath, path), { force: true }); } @@ -287,7 +503,8 @@ async function waitForPathContent( async function startConfiguredSession( context: RunnerContext, vault: TemporaryVault, - deviceName: string + deviceName: string, + settings: Partial> = {} ): Promise { const couchDbSettings = { uri: context.couchDb.uri, @@ -306,6 +523,7 @@ async function startConfiguredSession( autoSweepPlugins: false, autoSweepPluginsPeriodic: false, syncInternalFiles: false, + ...settings, }; const session = await startObsidianLiveSyncSession({ binary: context.binary, @@ -319,6 +537,18 @@ async function startConfiguredSession( }); await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); await assertE2eCompatibilityMarker(context.cliBinary, session.cliEnv); + if (Object.keys(settings.pluginSyncExtendedSetting ?? {}).length > 0) { + const restoredSelections = await evalObsidianJson( + context.cliBinary, + "JSON.stringify(app.plugins.plugins['obsidian-livesync'].core.settings.pluginSyncExtendedSetting)", + session.cliEnv + ); + assertEqual( + JSON.stringify(restoredSelections), + JSON.stringify(settings.pluginSyncExtendedSetting), + "The saved Automatic selection changed during start-up." + ); + } await configureCouchDb(context.cliBinary, session.cliEnv, couchDbSettings, customisationSettings); await evalObsidianJson( context.cliBinary, @@ -389,7 +619,7 @@ async function storeCustomisationFile(cliBinary: string, env: NodeJS.ProcessEnv, async function deleteCustomisationSyncEntry( cliBinary: string, env: NodeJS.ProcessEnv, - category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN", + category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN" | "PLUGIN_DATA", name: string, term?: string ): Promise { @@ -418,7 +648,7 @@ async function deleteCustomisationSyncEntry( async function waitForCustomisationEntry( cliBinary: string, env: NodeJS.ProcessEnv, - category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN", + category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN" | "PLUGIN_DATA", name: string, term?: string, timeoutMs = Number(process.env.E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS ?? 15000) @@ -430,7 +660,7 @@ async function waitForCustomisationEntry( async function waitForCustomisationEntries( cliBinary: string, env: NodeJS.ProcessEnv, - category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN", + category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN" | "PLUGIN_DATA", name: string, count: number, term?: string, @@ -574,6 +804,7 @@ async function main(): Promise { const testCase = selectedCase(); const testVisibility = testCase === "all" || testCase === "visibility"; const testMtime = testCase === "all" || testCase === "mtime"; + const testAutomatic = testCase === "automatic"; const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); if (!cli.binary) { @@ -604,6 +835,7 @@ async function main(): Promise { await writeVaultFile(vaultA.path, pluginManifestPath, pluginManifestContent); await writeVaultFile(vaultA.path, pluginMainPath, pluginMainContent); await writeVaultFile(vaultA.path, pluginStylesPath, pluginStylesContent); + if (testAutomatic) await writeVaultFile(vaultA.path, pluginDataPath, pluginDataContent); if (testVisibility) { for (const fixture of visibilityFixtures) { await writeVaultFile(vaultA.path, fixture.path, fixture.source); @@ -627,6 +859,12 @@ async function main(): Promise { await setVaultFileMtime(vaultB.path, fixture.path, targetPluginMtime); } } + if (testAutomatic) { + for (const fixture of pluginFixtures) { + await writeVaultFile(vaultB.path, fixture.path, fixture.target); + } + await writeVaultFile(vaultB.path, pluginDataPath, targetPluginDataContent); + } let session = await startConfiguredSession(context, vaultA, sourceDeviceName); const scanResult = await scanCustomisations(context.cliBinary, session.cliEnv); @@ -636,6 +874,9 @@ async function main(): Promise { for (const fixture of pluginFixtures) { await storeCustomisationFile(context.cliBinary, session.cliEnv, fixture.path); } + if (testAutomatic) { + await storeCustomisationFile(context.cliBinary, session.cliEnv, pluginDataPath); + } if (testVisibility) { for (const fixture of visibilityFixtures) { await storeCustomisationFile(context.cliBinary, session.cliEnv, fixture.path); @@ -650,15 +891,18 @@ async function main(): Promise { pluginName, 3 ); + const pluginDataEntry = testAutomatic + ? await waitForCustomisationEntry(context.cliBinary, session.cliEnv, "PLUGIN_DATA", pluginName) + : undefined; await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForCouchDbDocs(context.couchDb, context.dbName, (docs) => { const ids = new Set(docs.map((doc) => doc._id)); - const entries = [entry, configEntry, ...pluginEntries]; + const entries = [entry, configEntry, ...pluginEntries, ...(pluginDataEntry ? [pluginDataEntry] : [])]; return entries.every( (target) => ids.has(target.id) && target.children.every((childId) => ids.has(childId)) ); }); - for (const target of [entry, configEntry, ...pluginEntries]) { + for (const target of [entry, configEntry, ...pluginEntries, ...(pluginDataEntry ? [pluginDataEntry] : [])]) { const remoteEntry = await fetchCouchDbDocument(context.couchDb, context.dbName, target.id); if ( !remoteEntry.path?.startsWith("/\\:") || @@ -678,8 +922,15 @@ async function main(): Promise { throw new Error("The remote feature list does not declare encrypted internal Metadata."); } await session.app.stop(); + if (testAutomatic) { + for (const path of [pluginManifestPath, pluginMainPath, pluginStylesPath, pluginDataPath]) { + await removeVaultFile(vaultA.path, path); + } + } - session = await startConfiguredSession(context, vaultB, targetDeviceName); + session = await startConfiguredSession(context, vaultB, targetDeviceName, { + pluginSyncExtendedSetting: testAutomatic ? existingAutomaticSelections : {}, + }); if (testVisibility) { for (const fixture of visibilityFixtures) { await storeCustomisationFile(context.cliBinary, session.cliEnv, fixture.path); @@ -692,6 +943,15 @@ async function main(): Promise { } await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForCustomisationEntry(context.cliBinary, session.cliEnv, "SNIPPET", snippetName, sourceDeviceName); + if (testAutomatic) { + await waitForCustomisationEntry( + context.cliBinary, + session.cliEnv, + "PLUGIN_DATA", + pluginName, + sourceDeviceName + ); + } if (testVisibility) { for (const fixture of visibilityFixtures) { const name = fixture.path.split("/").pop() ?? fixture.path; @@ -721,10 +981,42 @@ async function main(): Promise { try { if (testVisibility) await inspectCustomisationVisibility(session); if (testMtime) await inspectPluginFreshnessAndNewestSelection(session); + if (testAutomatic) await inspectAutomaticSyncSelection(context, session, vaultB, pluginName); } catch (error) { await session.app.stop().catch(() => undefined); throw error; } + if (testAutomatic) { + await session.app.stop(); + session = await startConfiguredSession(context, vaultA, sourceDeviceName, { syncInternalFiles: true }); + try { + await pushLocalChanges(context.cliBinary, session.cliEnv); + const receivedStyles = await waitForPathContent( + vaultA.path, + pluginStylesPath, + (content) => content === targetPluginStylesContent + ); + const receivedData = await waitForPathContent( + vaultA.path, + pluginDataPath, + (content) => content === targetPluginDataContent + ); + assertEqual( + receivedStyles, + targetPluginStylesContent, + "Automatic plug-in group lost PLUGIN_MAIN files." + ); + assertEqual( + receivedData, + targetPluginDataContent, + "Automatic plug-in group lost the PLUGIN_DATA file." + ); + } finally { + await session.app.stop(); + } + console.log("Automatic Customisation Sync initialised and transferred both plug-in group children."); + return; + } if (testCase !== "all") { await session.app.stop(); console.log(`Customisation Sync ${testCase} regression case passed.`); diff --git a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts index 5079c021..9f9dba08 100644 --- a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts +++ b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts @@ -63,6 +63,21 @@ const manualMergeJsonPath = ".obsidian/livesync-e2e-manual-merge.json"; const targetPath = ".obsidian/livesync-targeted/only-a.json"; const hiddenFileCliTimeoutMs = Number(process.env.E2E_OBSIDIAN_HIDDEN_FILE_CLI_TIMEOUT_MS ?? 90000); const hiddenFileInitialisationStateKey = "__livesyncE2EHiddenFileInitialisation"; +const disabledInitialisationProbeKey = "__livesyncE2EDisabledHiddenInitialisation"; + +type DisabledInitialisationSnapshot = { + syncInternalFiles: boolean; + hiddenFiles: { id: string; path: string; rev: string; deleted: boolean }[]; + processedFiles: [string, string | undefined][]; + processedDatabaseFiles: [string, string | undefined][]; +}; + +type DisabledInitialisationObservation = { + result: boolean | null; + before: DisabledInitialisationSnapshot; + after: DisabledInitialisationSnapshot; + progressNotices: string[]; +}; type RunnerContext = { binary: string; @@ -132,6 +147,114 @@ async function waitForPathDeleted( throw new Error(`Timed out waiting for deleted file: ${join(vaultPath, path)}`); } +async function assertDisabledInitialisationIsNoOp( + context: RunnerContext, + session: ObsidianLiveSyncSession, + vault: TemporaryVault +): Promise { + const snippetExisted = await pathExists(vault.path, snippetPath); + const originalSnippet = snippetExisted ? await readVaultFile(vault.path, snippetPath) : undefined; + const originalJson = await readVaultFile(vault.path, mergeJsonPath); + const preservedSnippet = "body { --disabled-hidden-initialisation: preserved; }\n"; + const preservedJson = JSON.stringify({ disabledInitialisation: "preserved" }, null, 4) + "\n"; + await writeVaultFile(vault.path, snippetPath, preservedSnippet); + await writeVaultFile(vault.path, mergeJsonPath, preservedJson); + + try { + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.evaluate((stateKey) => { + const state = { progressNotices: [] as string[] }; + const observer = new MutationObserver(() => { + for (const notice of document.querySelectorAll(".notice")) { + const text = notice.textContent?.trim() ?? ""; + if (text.includes("[⚙") && !state.progressNotices.includes(text)) { + state.progressNotices.push(text); + } + } + }); + observer.observe(document.body, { childList: true, subtree: true, characterData: true }); + (globalThis as unknown as Record)[ + stateKey + ] = { ...state, observer }; + }, disabledInitialisationProbeKey); + }); + + const observation = await evalObsidianJson( + context.cliBinary, + [ + "(async()=>{", + `const targetFiles=${JSON.stringify([snippetPath, mergeJsonPath])};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const addOn=core.getAddOn('HiddenFileSync');", + "const snapshot=async()=>{", + "const files=await addOn.getAllDatabaseFiles();", + "const markers=(cache)=>Array.from(cache.keys()).sort().map((path)=>[path,cache.get(path)]);", + "return {", + "syncInternalFiles:core.settings.syncInternalFiles,", + "hiddenFiles:files.map((entry)=>({id:entry._id,path:addOn.getPath(entry),rev:entry._rev??'',deleted:!!(entry.deleted||entry._deleted)})).sort((a,b)=>a.path.localeCompare(b.path)),", + "processedFiles:markers(addOn._fileInfoLastProcessed),", + "processedDatabaseFiles:markers(addOn._databaseInfoLastProcessed),", + "};};", + "const before=await snapshot();", + "const result=await addOn.initialiseInternalFileSync('pullForce',true,targetFiles);", + "const after=await snapshot();", + "return JSON.stringify({result:result===undefined?null:result,before,after});", + "})()", + ].join(""), + session.cliEnv, + hiddenFileCliTimeoutMs + ); + const progressNotices = await withObsidianPage(session.remoteDebuggingPort, async (page) => { + return await page.evaluate((stateKey) => { + const state = ( + globalThis as unknown as Record< + string, + { progressNotices: string[]; observer: MutationObserver } | undefined + > + )[stateKey]; + state?.observer.disconnect(); + return state?.progressNotices ?? []; + }, disabledInitialisationProbeKey); + }); + + const failures: string[] = []; + if (observation.result !== false) + failures.push(`expected false result, received ${String(observation.result)}`); + if (observation.before.syncInternalFiles !== false || observation.after.syncInternalFiles !== false) { + failures.push("Hidden File Sync setting changed while initialisation was disabled"); + } + if (JSON.stringify(observation.before) !== JSON.stringify(observation.after)) { + failures.push("local hidden database state or processed markers changed"); + } + if ((await readVaultFile(vault.path, snippetPath)) !== preservedSnippet) { + failures.push("the existing local snippet was deleted or replaced"); + } + if ((await readVaultFile(vault.path, mergeJsonPath)) !== preservedJson) { + failures.push("the existing local JSON file was replaced"); + } + if (progressNotices.length > 0) + failures.push(`initialisation created progress Notices: ${JSON.stringify(progressNotices)}`); + if (failures.length > 0) { + throw new Error( + `Disabled Hidden File Sync initialisation was not a no-op: ${failures.join("; ")}\n${JSON.stringify({ + observation, + progressNotices, + })}` + ); + } + console.log( + "Disabled Hidden File Sync initialisation returned false without changing hidden files or cached state." + ); + } finally { + await writeVaultFile(vault.path, mergeJsonPath, originalJson); + if (snippetExisted && originalSnippet !== undefined) { + await writeVaultFile(vault.path, snippetPath, originalSnippet); + } else { + await removeVaultFile(vault.path, snippetPath); + } + } +} + function hasJsonValues(content: string, values: Record): boolean { try { const parsed = JSON.parse(content) as Record; @@ -633,6 +756,7 @@ async function runInitialisationNoticeGrouping(context: RunnerContext, vault: Te "Transient start-up Notices remained before the Hidden File Sync initialisation check." ); }); + await assertDisabledInitialisationIsNoOp(context, session, vault); await withObsidianPage(port, async (page) => { await page.evaluate((stateKey) => { const obsidianApp = (globalThis as typeof globalThis & { app: any }).app; diff --git a/updates.md b/updates.md index ca2e3497..cd3c3836 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We receive a clear Notice when **Automatic** requires Hidden File Sync to be enabled on this device. Our existing mode remains unchanged, and hidden-file initialisation cannot start while the feature is disabled. (#1270) +- We can initialise a whole plug-in's **MAIN** and **DATA** selections using the chosen Automatic initial action. + ## 1.0.39 20th October, 2026 From 9d30bed52ddd707a7af3ac9ed867b22a195ebe67 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 01:11:41 +0000 Subject: [PATCH 04/28] Clarify Hidden File Sync selectors and test normal allow-lists --- docs/settings.md | 6 +- docs/tips/hidden-file-sync.md | 4 + package.json | 1 + .../messages/LiveSyncProvisionalMessages.ts | 4 + .../CmdHiddenFileSync.selectors.unit.spec.ts | 127 +++++++++++ .../features/SettingDialogue/PaneSelector.ts | 6 +- test/e2e-obsidian/README.md | 2 + .../scripts/hidden-file-selectors.ts | 202 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + test/fixtures/optionalFileSyncDatabase.ts | 78 +++++++ updates.md | 7 + 11 files changed, 432 insertions(+), 6 deletions(-) create mode 100644 src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/hidden-file-selectors.ts create mode 100644 test/fixtures/optionalFileSyncDatabase.ts diff --git a/docs/settings.md b/docs/settings.md index 0885589c..f328db64 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -750,11 +750,13 @@ If enabled, the notification of hidden files change will be suppressed. #### Synchronising files -(RegExp) Empty to sync all files. Set filter as a regular expression to limit synchronising files. +Setting key: syncOnlyRegEx +(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns. #### Non-Synchronising files -(RegExp) If this is set, any changes to local and remote files that match this will be skipped. +Setting key: syncIgnoreRegEx +(RegExp) Skip changes to matching normal and hidden files, locally and remotely. #### Maximum file size diff --git a/docs/tips/hidden-file-sync.md b/docs/tips/hidden-file-sync.md index 6b270da2..d761bb73 100644 --- a/docs/tips/hidden-file-sync.md +++ b/docs/tips/hidden-file-sync.md @@ -32,6 +32,10 @@ Enabling Hidden File Sync requires an initialisation direction: Prefer a narrow target list. Device-specific workspace state and another plug-in's credentials are usually poor candidates for cross-device synchronisation. +`Normal Files` → `Synchronising files` limits ordinary files only. Hidden File Sync uses its own `Target patterns`. The `Non-Synchronising files` patterns under `Normal Files` also exclude matching hidden files, so review them alongside the hidden-file selectors. + +If an earlier version skipped hidden files because of the normal-file allow-list, repeat Hidden File Sync initialisation after updating. A routine scan can skip unchanged files which were already checked. Back up the Vault and choose the intended source direction as described above. + Target patterns also control directory traversal. A pattern must therefore match each parent directory as well as the intended files. For example, the following pattern admits the `.obsidian` parent and only its `snippets` subtree: ```text diff --git a/package.json b/package.json index a6ff13d4..4edfd29d 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "test:e2e:obsidian:two-vault-sync": "tsx test/e2e-obsidian/scripts/two-vault-sync.ts", "test:e2e:obsidian:security-seed-reconnect": "tsx test/e2e-obsidian/scripts/security-seed-reconnect.ts", "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", + "test:e2e:obsidian:hidden-file-selectors": "tsx test/e2e-obsidian/scripts/hidden-file-selectors.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", "test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts", "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 34ae100b..17ebe033 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,10 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns.": + "(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns.", + "(RegExp) Skip changes to matching normal and hidden files, locally and remotely.": + "(RegExp) Skip changes to matching normal and hidden files, locally and remotely.", "Preparing follow-up report…": "Preparing follow-up report…", "Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.", "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.": diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts new file mode 100644 index 00000000..4ec0f752 --- /dev/null +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts @@ -0,0 +1,127 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("./configureHiddenFileSyncMode.ts", () => ({ configureHiddenFileSyncMode: vi.fn() })); + +import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { readContent } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase"; +import { HiddenFileSync } from "./CmdHiddenFileSync"; + +const hiddenPath = ".obsidian/plugins/example/main.js" as FilePath; +const prefixedPath = `i:${hiddenPath}`; +const body = "module.exports = class ExamplePlugin {};"; +const allowList = "\\.(?:md|png|pdf)$"; +const fixtureOwners: Awaited>[] = []; + +async function hiddenFiles() { + const fixture = await createOptionalFileSyncDatabase(); + fixtureOwners.push(fixture); + fixture.settings.usePluginSync = false; + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { + settings: fixture.settings, + services: { + ...fixture.services, + vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) }, + }, + }, + cacheFileRegExps: new Map(), + cacheCustomisationSyncIgnoredFiles: new Map(), + }); + return { ...fixture, hiddenFileSync }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + for (const fixture of fixtureOwners.splice(0)) await fixture.dispose(); +}); + +describe("issue #1272: Hidden File Sync with normal-file selectors", () => { + it("stores a selected hidden file while the normal-file allow-list is set", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(true); + + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(prefixedPath, body))).resolves.toMatchObject({ + ok: true, + }); + const stored = await fixture.localDatabase.getDBEntry(prefixedPath as FilePath); + expect(stored).not.toBe(false); + if (stored) expect(await readContent(stored)).toBe(body); + }); + + it("reads existing hidden-file content while the normal-file allow-list is set", async () => { + const fixture = await hiddenFiles(); + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(prefixedPath, body))).resolves.toMatchObject({ + ok: true, + }); + const meta = await fixture.localDatabase.getDBEntryMeta(prefixedPath as FilePath); + if (!meta) throw new Error("The seeded hidden-file Metadata is unavailable"); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(true); + + const entry = await fixture.localDatabase.getDBEntryFromMeta(meta); + expect(entry).not.toBe(false); + if (entry) expect(await readContent(entry)).toBe(body); + }); + + it("stores and reads the same hidden file when the normal-file allow-list is empty", async () => { + const fixture = await hiddenFiles(); + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(prefixedPath, body))).resolves.toMatchObject({ + ok: true, + }); + const entry = await fixture.localDatabase.getDBEntry(prefixedPath as FilePath); + expect(entry).not.toBe(false); + if (entry) expect(await readContent(entry)).toBe(body); + }); + + it("continues to apply the normal-file allow-list to ordinary files", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + expect(fixture.localDatabase.isTargetFile("notes/example.md" as FilePath)).toBe(true); + expect(fixture.localDatabase.isTargetFile("notes/example.js" as FilePath)).toBe(false); + expect(fixture.localDatabase.isTargetFile("i:.obsidian/example.md" as FilePath)).toBe(true); + }); + + it("keeps Customisation Sync documents usable with the same normal-file allow-list", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + const path = "ix:device/PLUGIN_MAIN/example%main.js" as FilePath; + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(path, body))).resolves.toMatchObject({ + ok: true, + }); + const entry = await fixture.localDatabase.getDBEntry(path); + expect(entry).not.toBe(false); + if (entry) expect(await readContent(entry)).toBe(body); + }); + + it("keeps Hidden File Sync target and ignore patterns effective", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncInternalFilesTargetPatterns = + "^\\.obsidian/plugins/" as typeof fixture.settings.syncInternalFilesTargetPatterns; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(true); + await expect(fixture.hiddenFileSync.isTargetFile(".obsidian/snippets/example.css" as FilePath)).resolves.toBe( + false + ); + fixture.settings.syncInternalFilesIgnorePatterns = + "main\\.js$" as typeof fixture.settings.syncInternalFilesIgnorePatterns; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(false); + }); + + it("compatibility: applies the normal-file ignore-list to hidden documents", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + fixture.settings.syncIgnoreRegEx = "^\\.obsidian/" as typeof fixture.settings.syncIgnoreRegEx; + expect(fixture.localDatabase.isTargetFile(prefixedPath as FilePath)).toBe(false); + }); + + it("rejects hidden documents when Hidden File Sync is disabled", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncInternalFiles = false; + expect(fixture.localDatabase.isTargetFile(prefixedPath as FilePath)).toBe(false); + }); +}); diff --git a/src/modules/features/SettingDialogue/PaneSelector.ts b/src/modules/features/SettingDialogue/PaneSelector.ts index bfd7c0b1..783d6e99 100644 --- a/src/modules/features/SettingDialogue/PaneSelector.ts +++ b/src/modules/features/SettingDialogue/PaneSelector.ts @@ -13,7 +13,7 @@ export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme .setName($msg("Synchronising files")) .setDesc( $msg( - "(RegExp) Empty to sync all files. Set filter as a regular expression to limit synchronising files." + "(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns." ) ); const syncFilesControl = mount(MultipleRegExpControl, { @@ -32,9 +32,7 @@ export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme const nonSyncFilesSetting = new Setting(paneEl) .setName($msg("Non-Synchronising files")) - .setDesc( - $msg("(RegExp) If this is set, any changes to local and remote files that match this will be skipped.") - ); + .setDesc($msg("(RegExp) Skip changes to matching normal and hidden files, locally and remotely.")); const nonSyncFilesControl = mount(MultipleRegExpControl, { target: nonSyncFilesSetting.controlEl, diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..3e3093e7 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -248,6 +248,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. +`test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path. + `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. diff --git a/test/e2e-obsidian/scripts/hidden-file-selectors.ts b/test/e2e-obsidian/scripts/hidden-file-selectors.ts new file mode 100644 index 00000000..8686daa7 --- /dev/null +++ b/test/e2e-obsidian/scripts/hidden-file-selectors.ts @@ -0,0 +1,202 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { fileURLToPath } from "node:url"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +const hiddenRoot = ".obsidian/livesync-selector-example"; +const hiddenFiles = { + [`${hiddenRoot}/main.js`]: "console.log('hidden fixture');\n", + [`${hiddenRoot}/styles.css`]: ".hidden-fixture { color: blue; }\n", + [`${hiddenRoot}/settings.json`]: '{"fixture":true}\n', +}; +const controlRoot = ".obsidian/livesync-selector-control"; +const controlFiles = { + [`${controlRoot}/main.js`]: "console.log('control fixture');\n", + [`${controlRoot}/styles.css`]: ".control-fixture { color: red; }\n", + [`${controlRoot}/settings.json`]: '{"control":true}\n', +}; +const allowList = "\\.(?:md|png|pdf)$"; +const coreCode = "const core=app.plugins.plugins['obsidian-livesync'].core;"; +const hiddenCode = `${coreCode}const addOn=core.getAddOn('HiddenFileSync');`; + +type Observation = { name: string; expected: unknown; actual: unknown; passed: boolean }; + +async function writeFixtures(vaultPath: string, files: Record) { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(vaultPath, path)), { recursive: true }); + await writeFile(join(vaultPath, path), content); + } +} + +async function removeFixtures(vaultPath: string, files: Record) { + for (const path of Object.keys(files)) await rm(join(vaultPath, path), { force: true }); +} + +async function matchingFiles(vaultPath: string, files: Record) { + const matches: string[] = []; + for (const [path, content] of Object.entries(files)) { + try { + if ((await readFile(join(vaultPath, path), "utf8")) === content) matches.push(path); + } catch (error) { + if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error; + } + } + return matches.sort(); +} + +async function main() { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error("The Obsidian CLI is unavailable"); + const cliBinary = cli.binary; + const output = + process.env.E2E_HIDDEN_FILE_SELECTORS_OUTPUT ?? join(tmpdir(), "livesync-hidden-file-selectors.json"); + const observations: Observation[] = []; + const report: Record = { + revision: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), + sourceDiffSha256: createHash("sha256") + .update(new Uint8Array(execFileSync("git", ["diff", "--binary", "HEAD"]))) + .digest("hex"), + scenarioSha256: createHash("sha256") + .update(new Uint8Array(await readFile(fileURLToPath(import.meta.url)))) + .digest("hex"), + commonlibEntrySha256: createHash("sha256") + .update( + new Uint8Array( + await readFile( + "node_modules/@vrtmrz/livesync-commonlib/dist/managers/EntryManager/EntryManagerImpls.js" + ) + ) + ) + .digest("hex"), + bundleSha256: createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"), + commonlibVersion: JSON.parse(await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8")) + .version, + observations, + }; + const vault = await createTemporaryVault("livesync-hidden-file-selectors-"); + let session: ObsidianLiveSyncSession | undefined; + const record = (name: string, actual: unknown, expected: unknown) => { + const observation = { name, expected, actual, passed: isDeepStrictEqual(actual, expected) }; + observations.push(observation); + console.log(JSON.stringify(observation)); + }; + const evaluate = async (code: string): Promise => { + if (!session) throw new Error("The isolated Obsidian session has not started"); + return await evalObsidianJson(cliBinary, `(async()=>{${code}})()`, session.cliEnv, 60_000); + }; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + pluginData: createE2eCouchDbPluginData( + { uri: "http://127.0.0.1:9", username: "", password: "", dbName: "selector-local-only" }, + { periodicReplication: false, syncOnFileOpen: false, useOnlyLocalChunk: true } + ), + }); + report.pluginVersion = session.readiness.pluginVersion; + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + report.runtime = await evaluate(`${coreCode} + core.services.setting.setDeviceAndVaultName('selector-local'); + Object.assign(core.settings,{ + isConfigured:true,usePluginSync:false,usePluginSyncV2:true,syncInternalFiles:false, + autoSweepPlugins:false,autoSweepPluginsPeriodic:false,watchInternalFileChanges:false, + syncOnlyRegEx:${JSON.stringify(allowList)},syncIgnoreRegEx:'' + }); + return JSON.stringify({userAgent:navigator.userAgent,databaseReady:core.localDatabase.isReady}); + `); + + await writeFixtures(vault.path, hiddenFiles); + const uploadCode = (normalAllowList: string, files: Record) => `${hiddenCode} + Object.assign(core.settings,{ + syncInternalFiles:true,syncOnlyRegEx:${JSON.stringify(normalAllowList)}, + syncInternalFilesTargetPatterns:'^\\\\.obsidian(?:$|/livesync-selector-(?:example|control)(?:/|$))',syncInternalFilesIgnorePatterns:'' + }); + const candidates=await addOn.scanInternalFileNames(); + const selected=candidates.filter(path=>${JSON.stringify(Object.keys(files))}.includes(path)); + if(selected.length!==3)throw new Error('Hidden-file fixture selection failed: '+JSON.stringify({selected,target:core.settings.syncInternalFilesTargetPatterns})); + for(const path of selected)await addOn.updateLastProcessedAsActualFile(path); + await addOn.rebuildFromStorage(false,${JSON.stringify(Object.keys(files))},false); + const paths=[]; + for await(const doc of core.localDatabase.findEntries('i:','i;')){ + if(${JSON.stringify(Object.keys(files).map((path) => `i:${path}`))}.includes(doc.path)&&!doc.deleted&&!doc._deleted)paths.push(doc.path.substring(2)); + } + return JSON.stringify(paths.sort()); + `; + const hiddenUpload = await evaluate(uploadCode(allowList, hiddenFiles)); + record( + "#1272: a normal allow-list permits selected hidden uploads", + hiddenUpload, + Object.keys(hiddenFiles).sort() + ); + await writeFixtures(vault.path, controlFiles); + const uploadControl = await evaluate(uploadCode("", controlFiles)); + record( + "#1272 control: clearing the normal allow-list permits hidden uploads", + uploadControl, + Object.keys(controlFiles).sort() + ); + + const pullCode = (normalAllowList: string, files: Record) => `${hiddenCode} + core.settings.syncOnlyRegEx=${JSON.stringify(normalAllowList)}; + const results=[]; + for await(const doc of core.localDatabase.findEntries('i:','i;')){ + if(${JSON.stringify(Object.keys(files).map((path) => `i:${path}`))}.includes(doc.path)&&!doc.deleted&&!doc._deleted){ + results.push(await addOn.extractInternalFileFromDatabase(doc.path.substring(2),true,doc,false)); + } + } + return JSON.stringify(results); + `; + await removeFixtures(vault.path, hiddenFiles); + report.hiddenPullWithAllowList = await evaluate(pullCode(allowList, hiddenFiles)); + record( + "#1272: a normal allow-list permits selected hidden downloads", + await matchingFiles(vault.path, hiddenFiles), + Object.keys(hiddenFiles).sort() + ); + await removeFixtures(vault.path, controlFiles); + report.hiddenPullWithoutAllowList = await evaluate(pullCode("", controlFiles)); + record( + "#1272 control: clearing the normal allow-list permits hidden downloads", + await matchingFiles(vault.path, controlFiles), + Object.keys(controlFiles).sort() + ); + } catch (error) { + report.runtimeError = error instanceof Error ? error.stack : String(error); + if (session) report.processOutput = JSON.stringify(session.app.output()).slice(-4_000); + throw error; + } finally { + try { + await writeFile(output, JSON.stringify(report, null, 2) + "\n"); + console.log(`Verification report: ${output}`); + } finally { + try { + if (session) await session.app.stop(); + } finally { + await vault.dispose(); + } + } + } + if (observations.some((observation) => !observation.passed)) process.exitCode = 1; +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 7f766c67..d7da07c2 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -39,6 +39,7 @@ const focusedScenarios = new Set([ "two-vault-sync", "security-seed-reconnect", "hidden-file-snippet-sync", + "hidden-file-selectors", "customisation-sync", "received-change-readiness", "internal-metadata-doctor", diff --git a/test/fixtures/optionalFileSyncDatabase.ts b/test/fixtures/optionalFileSyncDatabase.ts new file mode 100644 index 00000000..dd3c6590 --- /dev/null +++ b/test/fixtures/optionalFileSyncDatabase.ts @@ -0,0 +1,78 @@ +import { randomUUID } from "node:crypto"; +import PouchDB from "pouchdb-core"; +import MemoryAdapter from "pouchdb-adapter-memory"; +import { vi } from "vitest"; +import { + DEFAULT_SETTINGS, + type EntryDoc, + type SavingEntry, + type FilePathWithPrefix, + type DocumentID, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { createTextBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { createLiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; +import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; + +PouchDB.plugin(MemoryAdapter); + +export function optionalFileEntry(path: string, content: string): SavingEntry { + return { + _id: path as DocumentID, + path: path as FilePathWithPrefix, + ctime: 1_000, + mtime: 2_000, + size: content.length, + type: "plain", + datatype: "plain", + data: createTextBlob(content), + children: [], + eden: {}, + }; +} + +export async function createOptionalFileSyncDatabase() { + const name = `optional-file-sync-${randomUUID()}`; + const database = new PouchDB(name, { adapter: "memory" }); + const settings = { + ...DEFAULT_SETTINGS, + syncInternalFiles: true, + useOnlyLocalChunk: true, + usePluginSync: true, + usePluginSyncV2: true, + pluginSyncExtendedSetting: {}, + }; + const services = { + API: { addLog: vi.fn(), getSystemConfigDir: () => ".obsidian" }, + setting: { currentSettings: () => settings, getDeviceAndVaultName: () => "device" }, + path: { + path2id: async (path: string) => path, + id2path: (id: string, entry?: { path?: string }) => entry?.path ?? id, + getPath: (entry: { path: FilePathWithPrefix }) => entry.path, + }, + context: { events: createLiveSyncEventHub() }, + database: { createPouchDBInstance: () => database }, + databaseEvents: { + onDatabaseInitialisation: async () => true, + onDatabaseHasReady: async () => true, + onCloseDatabase: async () => true, + onUnloadDatabase: async () => true, + }, + replicator: { onCloseActiveReplication: async () => true }, + }; + const localDatabase = new LiveSyncLocalDB(name, { services } as unknown as LiveSyncLocalDBEnv); + if (!(await localDatabase.initializeDatabase())) { + await database.destroy(); + throw new Error("The optional-file test database could not initialise"); + } + return { + database, + localDatabase, + services, + settings, + async dispose() { + localDatabase.offRemoteChunkFetchedHandler?.(); + await localDatabase.managers.teardownManagers(); + await database.destroy(); + }, + }; +} diff --git a/updates.md b/updates.md index ca2e3497..1700a53f 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Hidden File Sync + +#### Fixed + +- We can synchronise selected hidden files while the normal-file allow-list is set. (#1272) + - Normal-file exclusion patterns still apply to hidden files, and the Selector descriptions now explain their scope. + ## 1.0.39 20th October, 2026 From b7101fe9366839885d8180aac1fc9c7358931f09 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 01:11:41 +0000 Subject: [PATCH 05/28] Fix concurrent Customisation Sync catalogue updates --- package.json | 1 + .../CmdConfigSync.catalogue.unit.spec.ts | 298 ++++++++++++++++++ src/features/ConfigSync/CmdConfigSync.ts | 58 ++-- test/e2e-obsidian/README.md | 2 + .../scripts/customisation-catalogue-reload.ts | 220 +++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + test/fixtures/optionalFileSyncDatabase.ts | 78 +++++ updates.md | 7 + 8 files changed, 642 insertions(+), 23 deletions(-) create mode 100644 src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/customisation-catalogue-reload.ts create mode 100644 test/fixtures/optionalFileSyncDatabase.ts diff --git a/package.json b/package.json index a6ff13d4..224d8ef6 100644 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "test:e2e:obsidian:security-seed-reconnect": "tsx test/e2e-obsidian/scripts/security-seed-reconnect.ts", "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", + "test:e2e:obsidian:customisation-catalogue-reload": "tsx test/e2e-obsidian/scripts/customisation-catalogue-reload.ts", "test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts", "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", "test:e2e:obsidian:internal-metadata-migration": "tsx test/e2e-obsidian/scripts/internal-metadata-migration.ts", diff --git a/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts new file mode 100644 index 00000000..06b04919 --- /dev/null +++ b/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts @@ -0,0 +1,298 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + diff_match_patch: class DiffMatchPatch {}, + normalizePath: (path: string) => path, + parseYaml: vi.fn(), + Platform: {}, +})); +vi.mock("./PluginDialogModal.ts", () => ({ PluginDialogModal: class PluginDialogModal {} })); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({ + ConflictResolveModal: class ConflictResolveModal {}, +})); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("@/common/utils.ts", () => ({ + cancelTask: vi.fn(), + EVEN: Symbol("even"), + isCustomisationSyncMetadata: vi.fn(), + isPluginMetadata: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ PeriodicProcessor: class PeriodicProcessor {} })); +vi.mock("@/common/events.ts", () => ({ + EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync", + eventHub: { onEvent: vi.fn() }, +})); +vi.mock("@/common/translation", () => ({ $msg: (message: string) => message })); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ getObsidianCommunityPluginManager: vi.fn() })); + +import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase"; +import { ConfigSync, PluginDataExDisplayV2, pluginManifests } from "./CmdConfigSync"; + +const filenames = ["main.js", "manifest.json", "styles.css"]; +const content: Record = { + "main.js": "module.exports = class ExamplePlugin {};", + "manifest.json": JSON.stringify({ id: "example", name: "Example", version: "1.0.0" }), + "styles.css": ".example { color: red; }", +}; +const fixtureOwners: Awaited>[] = []; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function catalogue() { + const fixture = await createOptionalFileSyncDatabase(); + fixtureOwners.push(fixture); + fixture.settings.syncOnlyRegEx = "\\.(?:md|png|pdf)$" as typeof fixture.settings.syncOnlyRegEx; + const paths = filenames.map((name) => `ix:device/PLUGIN_MAIN/example%${name}` as FilePathWithPrefix); + for (const [index, path] of paths.entries()) { + const wireContent = `fixture-header\u200b\n\u200c${btoa(content[filenames[index]])}`; + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(path, wireContent))).resolves.toMatchObject({ + ok: true, + }); + } + const gates = new Map(paths.map((path) => [path, deferred()])); + const readEntry = fixture.localDatabase.getDBEntry.bind(fixture.localDatabase); + vi.spyOn(fixture.localDatabase, "getDBEntry").mockImplementation(async (path, ...args) => { + await gates.get(path)?.promise; + return readEntry(path, ...args); + }); + const written = new Map(); + const configSync = Object.create(ConfigSync.prototype) as ConfigSync; + Object.assign(configSync, { + core: { + localDatabase: fixture.localDatabase, + settings: fixture.settings, + services: fixture.services, + storageAccess: { + ensureDir: vi.fn(async () => true), + readHiddenFileText: vi.fn(async (path: string) => written.get(path) ?? ""), + writeHiddenFileAuto: vi.fn(async (path: string, data: string) => { + written.set(path, data); + }), + }, + }, + pluginList: [], + pluginListGeneration: 0, + loadedManifest_mTime: new Map(), + updatingV2Count: 0, + pluginScanProcessor: { clearQueue: vi.fn(), enqueue: vi.fn() }, + _log: vi.fn(), + storeCustomisationFileV2: vi.fn(async () => true), + }); + const updates: Promise[] = []; + const update = configSync.updatePluginListV2.bind(configSync); + vi.spyOn(configSync, "updatePluginListV2").mockImplementation((...args) => { + const promise = update(...args); + updates.push(promise); + return promise; + }); + const release = (order = filenames) => { + for (const name of order) gates.get(`ix:device/PLUGIN_MAIN/example%${name}` as FilePathWithPrefix)!.resolve(); + }; + return { ...fixture, configSync, updates, paths, release, written }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + pluginManifests.clear(); + for (const fixture of fixtureOwners.splice(0)) await fixture.dispose(); +}); + +describe("issue #1271: rebuilding a multi-file Customisation Sync item", () => { + it.each([ + { order: filenames }, + { order: [...filenames].reverse() }, + { order: ["manifest.json", "main.js", "styles.css"] }, + ])("retains all per-file documents when reads are released as $order", async ({ order }) => { + const fixture = await catalogue(); + await fixture.configSync.reloadPluginList(false); + fixture.release(order); + await Promise.all(fixture.updates); + + const documents = await fixture.database.allDocs({ keys: fixture.paths }); + expect(documents.rows).toHaveLength(3); + expect(fixture.configSync.pluginList).toHaveLength(1); + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual( + filenames.map((name) => `plugins/example/${name}`).sort() + ); + }); + + it("applies all three files after the first reload", async () => { + const fixture = await catalogue(); + await fixture.configSync.reloadPluginList(false); + fixture.release(); + await Promise.all(fixture.updates); + + await expect( + fixture.configSync.applyDataV2(fixture.configSync.pluginList[0] as PluginDataExDisplayV2) + ).resolves.toBe(true); + expect([...fixture.written.keys()].sort()).toEqual( + filenames.map((name) => `.obsidian/plugins/example/${name}`).sort() + ); + for (const name of filenames) + expect(fixture.written.get(`.obsidian/plugins/example/${name}`)).toBe(content[name]); + }); + + it("fills an existing row during a second enumeration", async () => { + const fixture = await catalogue(); + await fixture.configSync.reloadPluginList(false); + fixture.release(); + await Promise.all(fixture.updates); + await fixture.configSync.updatePluginList(false); + await Promise.all(fixture.updates); + + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual( + filenames.map((name) => `plugins/example/${name}`).sort() + ); + }); + + it("retains all files when item updates run sequentially", async () => { + const fixture = await catalogue(); + fixture.release(); + for (const path of fixture.paths) await fixture.configSync.updatePluginListV2(false, path); + + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual( + filenames.map((name) => `plugins/example/${name}`).sort() + ); + }); + + it("updates a different item while another item's reads are pending", async () => { + const fixture = await catalogue(); + const otherPath = "ix:device/PLUGIN_MAIN/other%main.js" as FilePathWithPrefix; + await fixture.localDatabase.putDBEntry( + optionalFileEntry(otherPath, `fixture-header\u200b\n\u200c${btoa(content["main.js"])}`) + ); + const pending = fixture.paths.map((path) => fixture.configSync.updatePluginListV2(false, path)); + try { + await fixture.configSync.updatePluginListV2(false, otherPath); + expect(fixture.configSync.pluginList.map((entry) => entry.name)).toEqual(["other"]); + expect(fixture.configSync.updatingV2Count).toBe(3); + } finally { + fixture.release(); + await Promise.all(pending); + } + expect(fixture.configSync.updatingV2Count).toBe(0); + }); + + it("continues updating an item after an earlier read fails", async () => { + const fixture = await catalogue(); + fixture.release(); + vi.spyOn(fixture.configSync, "createPluginDataExFileV2").mockRejectedValueOnce(new Error("Read failed")); + const failed = fixture.configSync.updatePluginListV2(false, fixture.paths[0]); + const next = fixture.configSync.updatePluginListV2(false, fixture.paths[1]); + + await expect(failed).rejects.toThrow("Read failed"); + await next; + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename)).toEqual([ + "plugins/example/manifest.json", + ]); + expect(fixture.configSync.updatingV2Count).toBe(0); + }); + + it.each(["read", "file update"])("reload discards cached files while an earlier %s is pending", async (phase) => { + const fixture = await catalogue(); + fixture.release(); + for (const path of fixture.paths) await fixture.configSync.updatePluginListV2(false, path); + const earlierRead = deferred(); + const freshRead = deferred(); + const entered = deferred(); + const createFile = fixture.configSync.createPluginDataExFileV2.bind(fixture.configSync); + let reads = 0; + vi.spyOn(fixture.configSync, "createPluginDataExFileV2").mockImplementation(async (path) => { + if (reads++ === 0) { + if (phase === "read") { + entered.resolve(); + await earlierRead.promise; + } + } else { + await freshRead.promise; + } + return createFile(path); + }); + if (phase === "file update") { + const entry = fixture.configSync.pluginList[0] as PluginDataExDisplayV2; + const setFile = entry.setFile.bind(entry); + vi.spyOn(entry, "setFile").mockImplementationOnce(async (file) => { + entered.resolve(); + await earlierRead.promise; + await setFile(file); + }); + } + const pending = fixture.configSync.updatePluginListV2(false, fixture.paths[0]); + try { + await entered.promise; + for (const path of fixture.paths.slice(1)) await fixture.database.remove(await fixture.database.get(path)); + await fixture.configSync.reloadPluginList(false); + earlierRead.resolve(); + await pending; + } finally { + earlierRead.resolve(); + freshRead.resolve(); + await Promise.all(fixture.updates); + } + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename)).toEqual([ + "plugins/example/main.js", + ]); + }); + + it("clearing the disabled feature prevents pending updates from repopulating the list", async () => { + const fixture = await catalogue(); + const pending = fixture.paths.map((path) => fixture.configSync.updatePluginListV2(false, path)); + fixture.settings.usePluginSync = false; + await fixture.configSync.updatePluginList(false); + fixture.release(); + await Promise.all(pending); + + expect(fixture.configSync.pluginList).toEqual([]); + expect(fixture.configSync.updatingV2Count).toBe(0); + }); + + it("a superseded enumeration cannot reintroduce a deleted item", async () => { + const fixture = await catalogue(); + const obsoletePath = "ix:device/PLUGIN_MAIN/obsolete%main.js" as FilePathWithPrefix; + await fixture.localDatabase.putDBEntry( + optionalFileEntry(obsoletePath, `fixture-header\u200b\n\u200c${btoa(content["main.js"])}`) + ); + const enumerate = fixture.localDatabase.findEntries.bind(fixture.localDatabase); + type EnumeratedEntry = ReturnType extends AsyncGenerator ? Entry : never; + const obsoleteDocs: EnumeratedEntry[] = []; + for await (const doc of enumerate(obsoletePath, `${obsoletePath}\u{10ffff}`, { include_docs: true })) { + obsoleteDocs.push(doc); + } + const entered = deferred(); + const release = deferred(); + let scans = 0; + vi.spyOn(fixture.localDatabase, "findEntries").mockImplementation((...args) => { + if (scans++ !== 0) return enumerate(...args); + return (async function* () { + entered.resolve(); + await release.promise; + yield* obsoleteDocs; + })(); + }); + const pending = fixture.configSync.updatePluginList(false); + try { + await entered.promise; + await fixture.database.remove(await fixture.database.get(obsoletePath)); + await fixture.configSync.reloadPluginList(false); + } finally { + release.resolve(); + fixture.release(); + await pending; + await Promise.all(fixture.updates); + } + + expect(fixture.configSync.pluginList.map((entry) => entry.name)).toEqual(["example"]); + expect(fixture.configSync.pluginList[0].files).toHaveLength(3); + expect(fixture.configSync.updatingV2Count).toBe(0); + }); +}); diff --git a/src/features/ConfigSync/CmdConfigSync.ts b/src/features/ConfigSync/CmdConfigSync.ts index 0e1412ee..4b07c99f 100644 --- a/src/features/ConfigSync/CmdConfigSync.ts +++ b/src/features/ConfigSync/CmdConfigSync.ts @@ -412,6 +412,7 @@ export class ConfigSync extends LiveSyncCommands { periodicPluginSweepProcessor = new PeriodicProcessor(this.core, async () => await this.scanAllConfigFiles(false)); pluginList: IPluginDataExDisplay[] = []; + private pluginListGeneration = 0; showPluginSyncModal() { if (!this.isThisModuleEnabled()) { return; @@ -532,6 +533,7 @@ export class ConfigSync extends LiveSyncCommands { return Promise.resolve(true); } async reloadPluginList(showMessage: boolean) { + this.pluginListGeneration++; this.pluginList = []; this.loadedManifest_mTime.clear(); pluginList.set(this.pluginList); @@ -761,39 +763,46 @@ export class ConfigSync extends LiveSyncCommands { updatingV2Count = 0; async updatePluginListV2(showMessage: boolean, unifiedFilenameWithKey: FilePathWithPrefix): Promise { + const generation = this.pluginListGeneration; + const isCurrent = () => generation === this.pluginListGeneration && this.isThisModuleEnabled(); try { this.updatingV2Count++; pluginV2Progress.set(this.updatingV2Count); // const unifiedFilenameWithKey = this.filenameWithUnifiedKey(updatedDocumentPath); const { pathV1 } = this.parseUnifiedPath(unifiedFilenameWithKey); - const oldEntry = this.pluginList.find((e) => e.documentPath == pathV1); - let entry: PluginDataExDisplayV2 | undefined = undefined; + await serialized(`config:catalogue:${pathV1}`, async () => { + if (!isCurrent()) return; + const oldEntry = this.pluginList.find((e) => e.documentPath == pathV1); + let entry: PluginDataExDisplayV2 | undefined = undefined; - if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) { - const newEntry = this.createPluginDataFromV2(unifiedFilenameWithKey); - if (newEntry) { - entry = newEntry; + if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) { + const newEntry = this.createPluginDataFromV2(unifiedFilenameWithKey); + if (newEntry) { + entry = newEntry; + } + } else if (oldEntry instanceof PluginDataExDisplayV2) { + entry = oldEntry; } - } else if (oldEntry instanceof PluginDataExDisplayV2) { - entry = oldEntry; - } - if (!entry) return; - const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey); - if (file) { - await entry.setFile(file); - } else { - entry.deleteFile(unifiedFilenameWithKey); - if (entry.files.length == 0) { - this.pluginList = this.pluginList.filter((e) => e.documentPath != pathV1); + if (!entry) return; + const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey); + if (!isCurrent()) return; + if (file) { + await entry.setFile(file); + } else { + entry.deleteFile(unifiedFilenameWithKey); + if (entry.files.length == 0) { + this.pluginList = this.pluginList.filter((e) => e.documentPath != pathV1); + } } - } - const newList = this.pluginList.filter((e) => e.documentPath != entry.documentPath); - newList.push(entry); - this.pluginList = newList; + if (!isCurrent()) return; + const newList = this.pluginList.filter((e) => e.documentPath != entry.documentPath); + newList.push(entry); + this.pluginList = newList; - scheduleTask("updatePluginListV2", 100, () => { - pluginList.set(this.pluginList); + scheduleTask("updatePluginListV2", 100, () => { + pluginList.set(this.pluginList); + }); }); } finally { this.updatingV2Count--; @@ -871,11 +880,13 @@ export class ConfigSync extends LiveSyncCommands { async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise { if (!this.isThisModuleEnabled()) { + this.pluginListGeneration++; this.pluginScanProcessor.clearQueue(); this.pluginList = []; pluginList.set(this.pluginList); return; } + const generation = this.pluginListGeneration; try { this.updatingV2Count++; pluginV2Progress.set(this.updatingV2Count); @@ -888,6 +899,7 @@ export class ConfigSync extends LiveSyncCommands { }) : this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true }); for await (const v of plugins) { + if (generation !== this.pluginListGeneration || !this.isThisModuleEnabled()) return; if (v.deleted || v._deleted) continue; if (v.path.indexOf("%") !== -1) { fireAndForget(() => this.updatePluginListV2(showMessage, v.path)); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..ec4c0f25 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -250,6 +250,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. +`test:e2e:obsidian:customisation-catalogue-reload` verifies that the first Reload lists all three files of a sample plug-in and that Apply restores their contents. It also checks a second enumeration and Apply in one temporary Vault with a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_CUSTOMISATION_CATALOGUE_OUTPUT` to choose the JSON report path. + The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`. diff --git a/test/e2e-obsidian/scripts/customisation-catalogue-reload.ts b/test/e2e-obsidian/scripts/customisation-catalogue-reload.ts new file mode 100644 index 00000000..ec74438d --- /dev/null +++ b/test/e2e-obsidian/scripts/customisation-catalogue-reload.ts @@ -0,0 +1,220 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { fileURLToPath } from "node:url"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +const pluginName = "livesync-catalogue-example"; +const sourceTerm = "catalogue-source"; +const pluginRoot = `.obsidian/plugins/${pluginName}`; +const pluginFiles = { + [`${pluginRoot}/main.js`]: "module.exports = class ExamplePlugin {};\n", + [`${pluginRoot}/manifest.json`]: JSON.stringify({ + id: pluginName, + name: "Catalogue Example", + version: "1.0.0", + minAppVersion: "1.0.0", + description: "An isolated test fixture", + author: "Self-hosted LiveSync", + }), + [`${pluginRoot}/styles.css`]: ".livesync-catalogue-example { color: red; }\n", +}; +const allowList = "\\.(?:md|png|pdf)$"; +const coreCode = "const core=app.plugins.plugins['obsidian-livesync'].core;"; +const configCode = `${coreCode}const addOn=core.getAddOn('ConfigSync');`; +const settleCode = ` + const settle=async()=>{ + await new Promise(resolve=>setTimeout(resolve,100)); + for(let attempt=0;attempt<200;attempt++){ + if(addOn.updatingV2Count===0)return; + await new Promise(resolve=>setTimeout(resolve,25)); + } + throw new Error('Customisation Sync did not settle'); + }; +`; +const sourceRowCode = ` + const row=addOn.pluginList.find(entry=>entry.category==='PLUGIN_MAIN'&&entry.name===${JSON.stringify(pluginName)}&&entry.term===${JSON.stringify(sourceTerm)}); + if(!row)throw new Error('The source catalogue row is missing'); +`; + +type Observation = { name: string; expected: unknown; actual: unknown; passed: boolean }; + +async function writeFixtures(vaultPath: string, files: Record) { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(vaultPath, path)), { recursive: true }); + await writeFile(join(vaultPath, path), content); + } +} + +async function removeFixtures(vaultPath: string, files: Record) { + for (const path of Object.keys(files)) await rm(join(vaultPath, path), { force: true }); +} + +async function matchingFiles(vaultPath: string, files: Record) { + const matches: string[] = []; + for (const [path, content] of Object.entries(files)) { + try { + if ((await readFile(join(vaultPath, path), "utf8")) === content) matches.push(path); + } catch (error) { + if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error; + } + } + return matches.sort(); +} + +async function main() { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error("The Obsidian CLI is unavailable"); + const cliBinary = cli.binary; + const output = + process.env.E2E_CUSTOMISATION_CATALOGUE_OUTPUT ?? join(tmpdir(), "livesync-customisation-catalogue.json"); + const observations: Observation[] = []; + const report: Record = { + revision: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), + sourceDiffSha256: createHash("sha256") + .update(new Uint8Array(execFileSync("git", ["diff", "--binary", "HEAD"]))) + .digest("hex"), + scenarioSha256: createHash("sha256") + .update(new Uint8Array(await readFile(fileURLToPath(import.meta.url)))) + .digest("hex"), + commonlibEntrySha256: createHash("sha256") + .update( + new Uint8Array( + await readFile( + "node_modules/@vrtmrz/livesync-commonlib/dist/managers/EntryManager/EntryManagerImpls.js" + ) + ) + ) + .digest("hex"), + bundleSha256: createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"), + commonlibVersion: JSON.parse(await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8")) + .version, + observations, + }; + const vault = await createTemporaryVault("livesync-customisation-catalogue-"); + let session: ObsidianLiveSyncSession | undefined; + const record = (name: string, actual: unknown, expected: unknown) => { + const observation = { name, expected, actual, passed: isDeepStrictEqual(actual, expected) }; + observations.push(observation); + console.log(JSON.stringify(observation)); + }; + const evaluate = async (code: string): Promise => { + if (!session) throw new Error("The isolated Obsidian session has not started"); + return await evalObsidianJson(cliBinary, `(async()=>{${code}})()`, session.cliEnv, 60_000); + }; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + pluginData: createE2eCouchDbPluginData( + { uri: "http://127.0.0.1:9", username: "", password: "", dbName: "catalogue-local-only" }, + { periodicReplication: false, syncOnFileOpen: false, useOnlyLocalChunk: true } + ), + }); + report.pluginVersion = session.readiness.pluginVersion; + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + report.runtime = await evaluate(`${coreCode} + core.services.setting.setDeviceAndVaultName('catalogue-local'); + Object.assign(core.settings,{ + isConfigured:true,usePluginSync:true,usePluginSyncV2:true,syncInternalFiles:false, + autoSweepPlugins:false,autoSweepPluginsPeriodic:false,watchInternalFileChanges:false, + syncOnlyRegEx:${JSON.stringify(allowList)},syncIgnoreRegEx:'' + }); + return JSON.stringify({userAgent:navigator.userAgent,databaseReady:core.localDatabase.isReady}); + `); + + await writeFixtures(vault.path, pluginFiles); + await evaluate(`${configCode}${settleCode} + for(const path of ${JSON.stringify(Object.keys(pluginFiles))}){ + const stored=await addOn.storeCustomisationFileV2(path,${JSON.stringify(sourceTerm)}); + if(stored===false)throw new Error('Could not seed a Customisation Sync document'); + } + await settle(); + return JSON.stringify({seeded:true}); + `); + const catalogue = await evaluate<{ files: string[]; documents: number }>(`${configCode}${settleCode} + await addOn.reloadPluginList(false); + await settle(); + ${sourceRowCode} + let documents=0; + for await(const doc of core.localDatabase.findEntries('ix:','ix;')){ + if(doc.path.startsWith(${JSON.stringify(`ix:${sourceTerm}/PLUGIN_MAIN/${pluginName}%`)})&&!doc.deleted&&!doc._deleted)documents++; + } + return JSON.stringify({files:row.files.map(file=>file.filename).sort(),documents}); + `); + record("#1271: all source documents remain in the database", catalogue.documents, 3); + record( + "#1271: one reload lists all files", + catalogue.files, + Object.keys(pluginFiles) + .map((path) => path.substring(".obsidian/".length)) + .sort() + ); + await removeFixtures(vault.path, pluginFiles); + const firstApply = await evaluate( + `${configCode}${sourceRowCode}return JSON.stringify({applied:await addOn.applyDataV2(row)});` + ); + report.firstApply = firstApply; + record( + "#1271: first Apply restores all file contents", + await matchingFiles(vault.path, pluginFiles), + Object.keys(pluginFiles).sort() + ); + + const secondPass = await evaluate(`${configCode}${settleCode} + await addOn.updatePluginList(false);await settle();${sourceRowCode} + return JSON.stringify(row.files.map(file=>file.filename).sort()); + `); + record( + "#1271 control: a second pass lists all files", + secondPass, + Object.keys(pluginFiles) + .map((path) => path.substring(".obsidian/".length)) + .sort() + ); + await removeFixtures(vault.path, pluginFiles); + await evaluate(`${configCode}${sourceRowCode}return JSON.stringify({applied:await addOn.applyDataV2(row)});`); + record( + "#1271 control: Apply after a second pass restores all contents", + await matchingFiles(vault.path, pluginFiles), + Object.keys(pluginFiles).sort() + ); + } catch (error) { + report.runtimeError = error instanceof Error ? error.stack : String(error); + if (session) report.processOutput = JSON.stringify(session.app.output()).slice(-4_000); + throw error; + } finally { + try { + await writeFile(output, JSON.stringify(report, null, 2) + "\n"); + console.log(`Verification report: ${output}`); + } finally { + try { + if (session) await session.app.stop(); + } finally { + await vault.dispose(); + } + } + } + if (observations.some((observation) => !observation.passed)) process.exitCode = 1; +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 7f766c67..54534944 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -40,6 +40,7 @@ const focusedScenarios = new Set([ "security-seed-reconnect", "hidden-file-snippet-sync", "customisation-sync", + "customisation-catalogue-reload", "received-change-readiness", "internal-metadata-doctor", "setting-markdown-export", diff --git a/test/fixtures/optionalFileSyncDatabase.ts b/test/fixtures/optionalFileSyncDatabase.ts new file mode 100644 index 00000000..dd3c6590 --- /dev/null +++ b/test/fixtures/optionalFileSyncDatabase.ts @@ -0,0 +1,78 @@ +import { randomUUID } from "node:crypto"; +import PouchDB from "pouchdb-core"; +import MemoryAdapter from "pouchdb-adapter-memory"; +import { vi } from "vitest"; +import { + DEFAULT_SETTINGS, + type EntryDoc, + type SavingEntry, + type FilePathWithPrefix, + type DocumentID, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { createTextBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { createLiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; +import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; + +PouchDB.plugin(MemoryAdapter); + +export function optionalFileEntry(path: string, content: string): SavingEntry { + return { + _id: path as DocumentID, + path: path as FilePathWithPrefix, + ctime: 1_000, + mtime: 2_000, + size: content.length, + type: "plain", + datatype: "plain", + data: createTextBlob(content), + children: [], + eden: {}, + }; +} + +export async function createOptionalFileSyncDatabase() { + const name = `optional-file-sync-${randomUUID()}`; + const database = new PouchDB(name, { adapter: "memory" }); + const settings = { + ...DEFAULT_SETTINGS, + syncInternalFiles: true, + useOnlyLocalChunk: true, + usePluginSync: true, + usePluginSyncV2: true, + pluginSyncExtendedSetting: {}, + }; + const services = { + API: { addLog: vi.fn(), getSystemConfigDir: () => ".obsidian" }, + setting: { currentSettings: () => settings, getDeviceAndVaultName: () => "device" }, + path: { + path2id: async (path: string) => path, + id2path: (id: string, entry?: { path?: string }) => entry?.path ?? id, + getPath: (entry: { path: FilePathWithPrefix }) => entry.path, + }, + context: { events: createLiveSyncEventHub() }, + database: { createPouchDBInstance: () => database }, + databaseEvents: { + onDatabaseInitialisation: async () => true, + onDatabaseHasReady: async () => true, + onCloseDatabase: async () => true, + onUnloadDatabase: async () => true, + }, + replicator: { onCloseActiveReplication: async () => true }, + }; + const localDatabase = new LiveSyncLocalDB(name, { services } as unknown as LiveSyncLocalDBEnv); + if (!(await localDatabase.initializeDatabase())) { + await database.destroy(); + throw new Error("The optional-file test database could not initialise"); + } + return { + database, + localDatabase, + services, + settings, + async dispose() { + localDatabase.offRemoteChunkFetchedHandler?.(); + await localDatabase.managers.teardownManagers(); + await database.destroy(); + }, + }; +} diff --git a/updates.md b/updates.md index ca2e3497..9a2da6ee 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Customisation Sync + +#### Fixed + +- We can reload the customisation list and apply every file in a multi-file item on the first attempt. (#1271) + - Reloading or clearing the list also prevents earlier pending updates from restoring removed files and items. + ## 1.0.39 20th October, 2026 From d1e4c340543b936c2508fdfe4ab8fdba79a7f5a3 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 09:12:02 +0000 Subject: [PATCH 06/28] Prepare consistent Chunk sizes and fresh Fast Fetch integration --- docs/settings.md | 27 +- src/common/replicatorProviders.ts | 5 + src/common/replicatorProviders.unit.spec.ts | 16 +- src/common/replicatorResources.unit.spec.ts | 30 + src/common/replicatorResources/index.ts | 1 + .../initialFetchFastForward.ts | 17 + .../features/SetupManager.unit.spec.ts | 19 +- .../scripts/couchdb-manual-setup-workflow.ts | 72 ++- updates.md | 16 + utils/chunk-size-benchmark/README.md | 72 +++ .../cloudant-json-boundary.mjs | 108 ++++ utils/chunk-size-benchmark/cloudant-limit.mjs | 50 ++ utils/chunk-size-benchmark/compose.yml | 30 + utils/chunk-size-benchmark/corpus.mjs | 310 +++++++++ .../datachannel-metrics.cjs | 23 + utils/chunk-size-benchmark/local.mjs | 596 ++++++++++++++++++ utils/chunk-size-benchmark/p2p.mjs | 159 +++++ .../rabin-window.test.mjs | 20 + utils/chunk-size-benchmark/remote.mjs | 210 ++++++ utils/chunk-size-benchmark/report.mjs | 85 +++ utils/chunk-size-benchmark/summarise.mjs | 83 +++ utils/chunk-size-benchmark/verify-results.mjs | 82 +++ utils/flyio/generate_setupuri.test.ts | 18 +- utils/setup/generate_setup_uri.test.ts | 7 +- 24 files changed, 2042 insertions(+), 14 deletions(-) create mode 100644 src/common/replicatorResources/initialFetchFastForward.ts create mode 100644 utils/chunk-size-benchmark/README.md create mode 100644 utils/chunk-size-benchmark/cloudant-json-boundary.mjs create mode 100644 utils/chunk-size-benchmark/cloudant-limit.mjs create mode 100644 utils/chunk-size-benchmark/compose.yml create mode 100644 utils/chunk-size-benchmark/corpus.mjs create mode 100644 utils/chunk-size-benchmark/datachannel-metrics.cjs create mode 100644 utils/chunk-size-benchmark/local.mjs create mode 100644 utils/chunk-size-benchmark/p2p.mjs create mode 100644 utils/chunk-size-benchmark/rabin-window.test.mjs create mode 100644 utils/chunk-size-benchmark/remote.mjs create mode 100644 utils/chunk-size-benchmark/report.mjs create mode 100644 utils/chunk-size-benchmark/summarise.mjs create mode 100644 utils/chunk-size-benchmark/verify-results.mjs diff --git a/docs/settings.md b/docs/settings.md index 0885589c..96684b21 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -936,6 +936,18 @@ Setting key: hashCacheMaxAmount Setting key: customChunkSize +New setup uses 0 for every remote type. With V3 Rabin–Karp, this gives binary +Chunks a 100 KiB maximum, a 25 KiB minimum, and an approximate 25 KiB candidate +gap. The candidate gap is not the mean Chunk length. Text below 4 MiB retains +adaptive smaller boundaries; large text uses the binary thresholds with +UTF-8-safe cuts. + +Larger values can reduce document counts for mostly immutable binary files, +but can increase the new content stored after an edit. Existing explicit and +agreed values are retained. Updating the plug-in does not re-chunk the Vault; +editing, forced writes, Chunk recreation, or a full Rebuild can create new +boundaries while existing content remains readable. + #### Chunk Splitter Setting key: chunkSplitterVersion @@ -957,7 +969,10 @@ If this enabled, chunks will be split into semantically meaningful segments. Not #### Fetch chunks on demand Setting key: readChunksOnline -(ex. Read chunks online) If this option is enabled, LiveSync reads chunks online directly instead of replicating them locally. Increasing Custom chunk size is recommended. +If enabled, this plug-in obtains missing Chunks when they are needed instead +of including all Chunks in ordinary pull replication. New setup retains the +same Chunk-size recommendation of 0. This setting does not re-chunk content +already stored by another device. #### Batch size of on-demand fetching @@ -1010,12 +1025,18 @@ The setting remains disabled by default because the measured storage and transfe #### Batch size Setting key: batch_size -Number of changes to sync at a time. Defaults to 50. Minimum is 2. +Number of changes in an ordinary CouchDB replication batch. The schema default +is 25, and the minimum is 2. Changing this value alone preserves the +replication checkpoint. Larger counts can reduce round trips, but do not bound +the final request size or memory use. #### Batch limit Setting key: batches_limit -Number of batches to process at a time. Defaults to 40. Minimum is 2. This along with batch size controls how many docs are kept in memory at a time. +Number of changes batches which PouchDB can read ahead. The schema default is +25, and the minimum is 2. This does not create parallel body transfers; +PouchDB processes one current replication batch at a time. It does not control +the separate P2P or Object Storage transfer units. #### Use timeouts instead of heartbeats diff --git a/src/common/replicatorProviders.ts b/src/common/replicatorProviders.ts index 845d8c2e..1e62c25d 100644 --- a/src/common/replicatorProviders.ts +++ b/src/common/replicatorProviders.ts @@ -29,6 +29,7 @@ import { } from "./replicatorConfigurationIdentity"; import { createCouchDBConnectionProbeFactory, + createCouchDBInitialFetchFastForwardResourceFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, createCouchDBSynchronisationInformationResourceFactory, @@ -125,6 +126,9 @@ export function createCentralReplicatorProviderDefinitions( [REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION]: supportedCapability( createCouchDBSynchronisationInformationResourceFactory(host) ), + [REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]: supportedCapability( + createCouchDBInitialFetchFastForwardResourceFactory(host) + ), }, centralRemoteAdministration: COUCHDB_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY, userInitiatedOneShot: supportedCapability(couchDBUserInitiatedOneShot), @@ -151,6 +155,7 @@ export function createCentralReplicatorProviderDefinitions( createObjectStorageSecuritySeedResourceFactory(host) ), [REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION]: CAPABILITY_NOT_APPLICABLE, + [REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]: CAPABILITY_NOT_APPLICABLE, }, centralRemoteAdministration: OBJECT_STORAGE_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY, userInitiatedOneShot: supportedCapability(objectStorageUserInitiatedOneShot), diff --git a/src/common/replicatorProviders.unit.spec.ts b/src/common/replicatorProviders.unit.spec.ts index 54a2c441..01a7d6be 100644 --- a/src/common/replicatorProviders.unit.spec.ts +++ b/src/common/replicatorProviders.unit.spec.ts @@ -44,7 +44,15 @@ describe("central Replicator provider definitions", () => { it("keeps the retained remote-resource catalogue bounded", () => { expect .soft(Object.values(REMOTE_RESOURCE_KINDS).sort()) - .toEqual(["connection", "preferred-tweak", "security-seed", "synchronisation-information"].sort()); + .toEqual( + [ + "connection", + "preferred-tweak", + "security-seed", + "synchronisation-information", + "initial-fetch-fast-forward", + ].sort() + ); }); it("lets Journal prepare its own fresh Security Seed while CouchDB uses central preparation", () => { @@ -114,10 +122,16 @@ describe("central Replicator provider definitions", () => { expect(couchResources?.[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION].kind).toBe( CAPABILITY_SUPPORT_KINDS.SUPPORTED ); + expect(couchResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]?.kind).toBe( + CAPABILITY_SUPPORT_KINDS.SUPPORTED + ); expect(objectResources?.[REMOTE_RESOURCE_KINDS.SECURITY_SEED].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(objectResources?.[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION].kind).toBe( CAPABILITY_SUPPORT_KINDS.NOT_APPLICABLE ); + expect(objectResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]?.kind).toBe( + CAPABILITY_SUPPORT_KINDS.NOT_APPLICABLE + ); expect(couchAdministration?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(objectAdministration?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect("activeRemoteReads" in definitions.get(REMOTE_COUCHDB)!).toBe(false); diff --git a/src/common/replicatorResources.unit.spec.ts b/src/common/replicatorResources.unit.spec.ts index 2215a48a..07550057 100644 --- a/src/common/replicatorResources.unit.spec.ts +++ b/src/common/replicatorResources.unit.spec.ts @@ -5,6 +5,7 @@ import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings"; const mocks = vi.hoisted(() => ({ logger: vi.fn(), + initialFetchFastForwardResource: undefined as unknown, couchDB: [] as Array<{ host: unknown; isMobile: ReturnType; @@ -12,6 +13,7 @@ const mocks = vi.hoisted(() => ({ getRemoteStatus: ReturnType; getRemotePreferredTweakValues: ReturnType; getReplicationPBKDF2Salt: ReturnType; + createInitialFetchFastForwardResource: ReturnType; closeReplication: ReturnType; }>, objectStorage: [] as Array<{ @@ -42,6 +44,7 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicato getRemoteStatus = vi.fn(); getRemotePreferredTweakValues = vi.fn(); getReplicationPBKDF2Salt = vi.fn(); + createInitialFetchFastForwardResource = vi.fn(() => mocks.initialFetchFastForwardResource); closeReplication = vi.fn(); constructor(host: unknown) { @@ -69,6 +72,7 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe import { createCouchDBConnectionProbeFactory, + createCouchDBInitialFetchFastForwardResourceFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, createCouchDBSynchronisationInformationResourceFactory, @@ -92,6 +96,7 @@ describe("replicator probe factories", () => { beforeEach(() => { mocks.couchDB.length = 0; mocks.objectStorage.length = 0; + mocks.initialFetchFastForwardResource = undefined; mocks.checkSyncInfo.mockReset().mockResolvedValue(true); mocks.logger.mockClear(); }); @@ -370,4 +375,29 @@ describe("replicator probe factories", () => { await resource.dispose(); expect(replicator.closeReplication).toHaveBeenCalledOnce(); }); + + it("creates an Initial Fetch Fast Forward resource from a settings snapshot and leaves ownership to the resource", async () => { + const host = { name: "host" }; + const settings = createSettings(); + const snapshot = { ...settings }; + const prepare = vi.fn(async () => undefined); + const dispose = vi.fn(async () => undefined); + const resource = { prepare, dispose }; + const factory = createCouchDBInitialFetchFastForwardResourceFactory(host as never); + mocks.initialFetchFastForwardResource = resource; + + const result = await factory(settings); + const replicator = mocks.couchDB[0]; + + settings.couchDB_URI = "https://changed.example.test"; + expect(replicator.createInitialFetchFastForwardResource).toHaveBeenCalledWith(snapshot); + expect(replicator.createInitialFetchFastForwardResource.mock.calls[0][0]).not.toBe(settings); + expect(replicator.host).toBe(host); + expect(result).toBe(resource); + + await expect(result.prepare()).resolves.toBeUndefined(); + await result.dispose(); + expect(prepare).toHaveBeenCalledOnce(); + expect(dispose).toHaveBeenCalledOnce(); + }); }); diff --git a/src/common/replicatorResources/index.ts b/src/common/replicatorResources/index.ts index 9888e85c..36a55545 100644 --- a/src/common/replicatorResources/index.ts +++ b/src/common/replicatorResources/index.ts @@ -14,3 +14,4 @@ export { type SecuritySeedResourceHost, } from "./securitySeed"; export { createCouchDBSynchronisationInformationResourceFactory } from "./synchronisationInformation"; +export { createCouchDBInitialFetchFastForwardResourceFactory } from "./initialFetchFastForward"; diff --git a/src/common/replicatorResources/initialFetchFastForward.ts b/src/common/replicatorResources/initialFetchFastForward.ts new file mode 100644 index 00000000..601f2d1d --- /dev/null +++ b/src/common/replicatorResources/initialFetchFastForward.ts @@ -0,0 +1,17 @@ +import type { InitialFetchFastForwardResourceFactory } from "@vrtmrz/livesync-commonlib/replication"; +import { + LiveSyncCouchDBReplicator, + type LiveSyncCouchDBReplicatorEnv, +} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; +import { snapshotRemoteSettings } from "./shared"; + +/** Create a private CouchDB Replicator for a fresh Fast Fetch checkpoint handoff. */ +export function createCouchDBInitialFetchFastForwardResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): InitialFetchFastForwardResourceFactory { + return (setting) => { + const snapshot = snapshotRemoteSettings(setting); + const replicator = new LiveSyncCouchDBReplicator(host); + return Promise.resolve(replicator.createInitialFetchFastForwardResource(snapshot)); + }; +} diff --git a/src/modules/features/SetupManager.unit.spec.ts b/src/modules/features/SetupManager.unit.spec.ts index c8dfe4ec..713b75f9 100644 --- a/src/modules/features/SetupManager.unit.spec.ts +++ b/src/modules/features/SetupManager.unit.spec.ts @@ -172,10 +172,12 @@ describe("SetupManager", () => { const { manager, dialogManager } = createSetupManager(); dialogManager.openWithExplicitCancel.mockResolvedValueOnce("configure-manually"); const configureManually = vi.spyOn(manager, "onConfigureManually").mockResolvedValue(true); + const newVaultSettings = createNewVaultSettings(); await manager.onOnboard(UserMode.NewUser); - expect(configureManually).toHaveBeenCalledWith(createNewVaultSettings(), UserMode.NewUser); + expect(newVaultSettings.customChunkSize).toBe(0); + expect(configureManually).toHaveBeenCalledWith(newVaultSettings, UserMode.NewUser); }); it("compatibility: normalises imported flat remote settings from a Setup URI before applying", async () => { @@ -487,8 +489,15 @@ describe("SetupManager", () => { expect(applySettings).not.toHaveBeenCalled(); }); - it("preserves modern profiles, display names, and the active selection from a Setup URI", async () => { + it.each([ + { importedSize: 0, expectedSize: 0 }, + { importedSize: undefined, expectedSize: 60 }, + ])("preserves modern Setup URI profiles and adopts the supplied chunk size $importedSize", async ({ importedSize, expectedSize }) => { const { manager, setting, dialogManager } = createSetupManager(); + setting.settings = { + ...setting.currentSettings(), + customChunkSize: 60, + }; const imported = { ...DEFAULT_SETTINGS, remoteConfigurations: { @@ -507,6 +516,8 @@ describe("SetupManager", () => { }, activeConfigurationId: "archive", } as ObsidianLiveSyncSettings; + if (importedSize === undefined) delete (imported as Partial).customChunkSize; + else imported.customChunkSize = importedSize; dialogManager.openWithExplicitCancel .mockResolvedValueOnce(imported) .mockResolvedValueOnce("compatible-existing-user"); @@ -514,6 +525,7 @@ describe("SetupManager", () => { await manager.onUseSetupURI(UserMode.Unknown, "mock-config://modern-settings"); const current = setting.currentSettings(); + expect(current.customChunkSize).toBe(expectedSize); expect(current.remoteConfigurations).toEqual(imported.remoteConfigurations); expect(current.activeConfigurationId).toBe("archive"); expect(Object.keys(current.remoteConfigurations).some((id) => id.startsWith("legacy-"))).toBe(false); @@ -523,6 +535,7 @@ describe("SetupManager", () => { const { manager, setting, dialogManager } = createSetupManager(); setting.settings = { ...setting.currentSettings(), + customChunkSize: 60, isConfigured: true, remoteConfigurations: { existing: { @@ -554,6 +567,7 @@ describe("SetupManager", () => { await manager.onCouchDBManualSetup(UserMode.ExistingUser, setting.currentSettings()); const current = setting.currentSettings(); + expect(current.customChunkSize).toBe(60); expect(current.remoteConfigurations.existing).toBeDefined(); expect(Object.keys(current.remoteConfigurations)).toHaveLength(2); expect(current.activeConfigurationId).not.toBe("existing"); @@ -662,6 +676,7 @@ describe("SetupManager", () => { await manager.onP2PManualSetup(UserMode.NewUser, setting.currentSettings()); const current = setting.currentSettings(); + expect(current.customChunkSize).toBe(0); expect(Object.keys(current.remoteConfigurations)).toHaveLength(1); expect(current.activeConfigurationId).not.toBe(""); expect(current.P2P_ActiveRemoteConfigurationId).toBe(current.activeConfigurationId); diff --git a/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts b/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts index 37e5e22f..34aa2e5e 100644 --- a/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts +++ b/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts @@ -47,6 +47,16 @@ const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_SETUP_URI_TIMEOUT_MS ?? 3000 const e2eePassphrase = `%${randomBytes(24).toString("base64url")}`; const notePath = "E2E/manual-couchdb/from-first-device.md"; const noteContent = "# Manual CouchDB setup\n\nThis note was sent by the manually configured first device.\n"; +const binaryPath = "E2E/manual-couchdb/content-boundaries.png"; +const binaryBlockSize = 32 * 1024; +function binaryFixture(): Uint8Array { + const bytes = new Uint8Array(binaryBlockSize * 8); + for (let index = 0; index < 8; index++) { + bytes[index * binaryBlockSize] = index + 2; + bytes[(index + 1) * binaryBlockSize - 1] = 1; + } + return bytes; +} const returnNotePath = "E2E/manual-couchdb/from-second-device.md"; const returnNoteContent = "# Manual CouchDB return journey\n\nThis note returned through a Setup URI generated by the first device.\n"; @@ -395,6 +405,32 @@ async function waitForVaultFile( throw new Error(`Timed out waiting for ${path}. Last content:\n${lastContent}`); } +async function waitForBinaryFile(vault: TemporaryVault): Promise { + const expected = Buffer.from(binaryFixture()); + const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ?? 30000); + while (Date.now() < deadline) { + try { + if ((await readFile(join(vault.path, binaryPath))).equals(expected)) return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error("The fetched binary did not match its exact original bytes."); +} + +async function readChunkInventory(context: RunnerContext, session: ObsidianLiveSyncSession): Promise { + return await evalObsidianJson( + context.cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + const rows=(await core.localDatabase.allDocsRaw({include_docs:true})).rows; + return JSON.stringify(rows.filter(row=>row.doc?.type==='leaf').map(row=>row.id).sort()); + })()`, + session.cliEnv + ); +} + async function waitForRemoteEntry(context: RunnerContext, entry: { id: string; children: string[] }): Promise { await waitForCouchDbDocs(context.couchDb, context.dbName, (docs) => { const ids = new Set(docs.map((doc) => doc._id)); @@ -681,6 +717,8 @@ async function main(): Promise { process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true" ? randomBytes(32).toString("base64url") : undefined; let firstEntryId: string | undefined; let returnEntryId: string | undefined; + let binaryRevision: string | undefined; + let chunkInventory: string[] = []; try { await assertCouchDbReachable(couchDb); @@ -712,10 +750,34 @@ async function main(): Promise { } await writeNoteViaObsidian(context.cliBinary, session.cliEnv, notePath, noteContent); + const size = await evalObsidianJson( + context.cliBinary, + `JSON.stringify(app.plugins.plugins['obsidian-livesync'].core.settings.customChunkSize)`, + session.cliEnv + ); + assertEqual(size, 0, "New manual setup did not retain the common Chunk-size recommendation."); + await evalObsidianJson( + context.cliBinary, + `(async()=>{ + const bytes=new Uint8Array(${binaryBlockSize}*8); + for(let index=0;index<8;index++){ + bytes[index*${binaryBlockSize}]=index+2; + bytes[(index+1)*${binaryBlockSize}-1]=1; + } + await app.vault.createBinary(${JSON.stringify(binaryPath)},bytes.buffer); + return JSON.stringify(true); + })()`, + session.cliEnv + ); + const binaryEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, binaryPath); + assertEqual(binaryEntry.children.length, 8, "V3 size 0 did not use the expected content boundaries."); + binaryRevision = binaryEntry.rev; const entry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, notePath); firstEntryId = entry.id; await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForRemoteEntry(context, entry); + await waitForRemoteEntry(context, binaryEntry); + chunkInventory = await readChunkInventory(context, session); } catch (error) { await captureFailure(session, "first-device"); throw error; @@ -729,6 +791,13 @@ async function main(): Promise { session = await startUnconfiguredSession(context, vaultA); try { await assertRestoredE2EEPassphrase(session, context.cliBinary); + const unchanged = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, binaryPath); + assertEqual(unchanged.rev, binaryRevision, "Restart rewrote unchanged binary Metadata."); + assertEqual( + JSON.stringify(await readChunkInventory(context, session)), + JSON.stringify(chunkInventory), + "Restart created Chunks for unchanged Vault files." + ); await scheduleRemoteOverwrite(session.remoteDebuggingPort); screenshots.push(await confirmRebuild(session.remoteDebuggingPort, e2eeRebuildCaptures)); screenshots.push( @@ -775,6 +844,7 @@ async function main(): Promise { await assertPersistedE2EE(vaultB, independentIdSource); await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForVaultFile(vaultB, notePath, noteContent); + await waitForBinaryFile(vaultB); if (independentIdSource) { assertEqual( await readDocumentId(context.cliBinary, session.cliEnv, notePath), @@ -815,7 +885,7 @@ async function main(): Promise { } console.log( - `Manual CouchDB onboarding created and tested its database, generated a second-device Setup URI, and completed a bidirectional note round-trip. Screenshots: ${screenshots.join(", ")}` + `Manual CouchDB onboarding retained size 0, used binary content boundaries, preserved unchanged revisions and Chunks across restart, fetched exact binary bytes, and completed a bidirectional note round-trip. Screenshots: ${screenshots.join(", ")}` ); } finally { await stopTrackedSessions(context).catch((error: unknown) => { diff --git a/updates.md b/updates.md index ca2e3497..2bf14f77 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,22 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We can reuse unchanged content more reliably after edits with the V3 Rabin–Karp chunk splitter. + - Corrected rolling-window processing lets boundary detection recover after an insertion. Existing content remains readable, and installing this update does not trigger a full re-chunk of our Vaults. + - Chunk-size values 0 and 1 now use content-defined binary boundaries below their maximum lengths, improving reuse after an insertion instead of producing fixed-length pieces. + - Editing or explicitly rewriting a file, recreating its chunks, or rebuilding from our files can produce different chunk boundaries and additional chunks. Older and updated devices can retain different chunk layouts; this affects reuse and storage, not the file contents. +- We can preserve all conflict branches, including deleted branches, during Fast Setup. + - A fresh, complete Fast Fetch can also hand its verified progress to ordinary synchronisation before file watching resumes, reducing the first subsequent identifier scan. Existing resume cursors and On Demand chunk retrieval retain their existing behaviour. + +#### Changed + +- New setup uses the same Chunk-size recommendation of 0 across remote types, and Setting Doctor no longer recommends increasing every self-hosted V3 configuration to 60. + - Existing explicit and agreed values are retained. Smaller Chunks favour reuse during editing, but increase document counts and can make first uploads and large rebuilds slower. Mostly immutable binary Vaults can still benefit from a larger explicit value. + ## 1.0.39 20th October, 2026 diff --git a/utils/chunk-size-benchmark/README.md b/utils/chunk-size-benchmark/README.md new file mode 100644 index 00000000..f3db9ebd --- /dev/null +++ b/utils/chunk-size-benchmark/README.md @@ -0,0 +1,72 @@ +# Chunk-size research harness + +These scripts support the October 2026 Chunk-size investigation. The decision, +measurements, and their limits are recorded in Commonlib's +`docs/adr/2026_10_chunking_and_transport_batching.md` and its evidence companion. +They are research tools with experiment-specific assertions, separate from +the maintained unit and integration suites. + +The original experiments used Commonlib 0.1.44 and unpublished candidates with +the same version label. `local.mjs` checks that version explicitly. Record the +installed package's source revision and tarball digest when comparing results; +the version label alone does not identify the tested implementation. Scripts +which import internal `dist` paths also depend on that package layout. + +## Local measurements + +Install the exact Commonlib candidate to assess, then run from the repository +root with Node.js 24: + +```sh +NODE_OPTIONS=--max-old-space-size=3072 node utils/chunk-size-benchmark/local.mjs --output .bench/chunk-size --sizes 0,1,4 --repeats 1 +node --test utils/chunk-size-benchmark/rabin-window.test.mjs +``` + +The local run generates a deterministic corpus from repository Markdown, +images, and seeded binary data. It records initial, overwrite, and insertion +stages, Chunk identifiers, reconstructed content checks, and timings. Keep +the generated corpus when comparing revisions: source Markdown may change. +Generated Vaults, logs, and results belong under `.bench/` and are not source +files for the normal test runner. + +## Transport measurements + +Build the CLI and start the disposable fixtures required for the chosen +transport. These use loopback ports and public benchmark-only credentials. +The scripts create and delete their own test databases and buckets. +GNU `time`, Docker Compose, and a Linux CLI runtime are required. + +```sh +NODE_OPTIONS=--max-old-space-size=3072 npm run build --workspace self-hosted-livesync-cli +docker compose -p livesync-chunk-benchmark -f utils/chunk-size-benchmark/compose.yml up -d couchdb object-storage +node utils/chunk-size-benchmark/remote.mjs --output .bench/chunk-size --sizes 0,1,4 --repeats 1 --rtt-ms 0 +docker compose -p livesync-chunk-benchmark -f utils/chunk-size-benchmark/compose.yml down --volumes +``` + +`remote.mjs` accepts `--providers couchdb`, `--providers s3`, or both, plus +`--corpus`, `--source-cli`, and `--target-cli`. Its HTTP proxy adds the requested +RTT across the request and response legs. This models per-request delay rather +than network bandwidth, packet loss, or a production service. + +For P2P, start the `relay` fixture instead and run `p2p.mjs` with the same +output, corpus, size, repeat, and CLI options, omitting `--rtt-ms`. Its +DataChannel counters measure application payloads and record the selected +connection path. An archived CLI entry point needs all of its generated +CommonJS chunks beside it. Run transports sequentially and stop the named +Compose project afterwards. + +## Historical analysis helpers + +- `summarise.mjs`, `report.mjs`, and `verify-results.mjs` consume the original + result layouts. The verifier requires that experiment's exact variants, + configuration counts, and files; it is not a general mixed-version verifier. +- `cloudant-limit.mjs` temporarily changes the disposable CouchDB document + limit and restores it afterwards. `cloudant-json-boundary.mjs` probes final + serialised document sizes against the same fixture. These scripts model + published Cloudant constraints; they do not connect to IBM Cloudant or + certify its acceptance of a Vault. + +Some helpers retain historical output-directory and size defaults. Supply +explicit paths and inspect their assertions before reusing them for a new +experiment. The recorded implementation matrix checks file content in both +writer directions; revision-conflict retention has separate Commonlib tests. diff --git a/utils/chunk-size-benchmark/cloudant-json-boundary.mjs b/utils/chunk-size-benchmark/cloudant-json-boundary.mjs new file mode 100644 index 00000000..e7c8ad99 --- /dev/null +++ b/utils/chunk-size-benchmark/cloudant-json-boundary.mjs @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const packageRoot = dirname(fileURLToPath(import.meta.resolve('@vrtmrz/livesync-commonlib/package.json'))); +const { splitPiecesRabinKarp } = await import(pathToFileURL(join(packageRoot, 'dist/string_and_binary/chunks.js')).href); +const { getConfiguredFunctionsForEncryption } = await import(pathToFileURL(join(packageRoot, 'dist/pouchdb/encryption.js')).href); +const { compressDoc } = await import(pathToFileURL(join(packageRoot, 'dist/pouchdb/compress.js')).href); +const output = resolve(process.argv[2] || '.bench/rabin-window-fixed/cloudant-json-boundary'); +await fs.mkdir(output, { recursive: true }); +const uri = 'http://127.0.0.1:15994'; +const headers = { Authorization: `Basic ${Buffer.from('bench:benchmark-only-password').toString('base64')}`, 'Content-Type': 'application/json' }; +const configPath = '/_node/_local/_config/couchdb/max_document_size'; +const previousResponse = await fetch(uri + configPath, { headers }); +assert(previousResponse.ok || previousResponse.status === 404); +const previous = previousResponse.ok ? await previousResponse.json() : undefined; +const requestConfigPath = '/_node/_local/_config/chttpd/max_http_request_size'; +const previousRequestResponse = await fetch(uri + requestConfigPath, { headers }); +assert(previousRequestResponse.ok || previousRequestResponse.status === 404); +const previousRequestLimit = previousRequestResponse.ok ? await previousRequestResponse.json() : undefined; +const database = `chunkbench-json-budget-${Date.now()}`; +const inputs = { + binary: new Blob([new Uint8Array(4 * 1024 * 1024)], { type: 'application/octet-stream' }), + nulText: new Blob(['\0'.repeat(4 * 1024 * 1024)], { type: 'text/plain' }), + escapedText: new Blob(['\\'.repeat(4 * 1024 * 1024)], { type: 'text/plain' }), +}; +const cases = [ + { kind: 'binary', customChunkSize: 4, encryption: 'v1', compression: false, expectedAccepted: true }, + { kind: 'binary', customChunkSize: 4, encryption: 'v2', compression: false, expectedAccepted: true }, + { kind: 'binary', customChunkSize: 5, encryption: 'v2', compression: false, expectedAccepted: false }, + { kind: 'nulText', customChunkSize: 0, encryption: 'none', compression: false, expectedAccepted: true }, + { kind: 'nulText', customChunkSize: 4, encryption: 'none', compression: false }, + { kind: 'escapedText', customChunkSize: 4, encryption: 'none', compression: false }, + { kind: 'nulText', customChunkSize: 4, encryption: 'v2', compression: false, expectedAccepted: true }, + { kind: 'nulText', customChunkSize: 4, encryption: 'none', compression: true, expectedAccepted: true }, +]; +const records = []; +const record = { scope: 'Synthetic leaf-document probes using the installed splitter, compression, encryption, and JSON serialisation, sent to the owned CouchDB fixture with a 1,000,000-byte document limit and 64,000,000-byte request limit. This is not an IBM Cloudant service measurement or a whole-file synchronisation test. CouchDB 3.5.0 estimates document size from parsed strings and does not count JSON escapes; Cloudant internal size accounting has not been verified.', source: 'https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-limits', sizeCalculationSource: 'https://github.com/apache/couchdb/blob/3.5.0/src/couch/src/couch_ejson_size.erl', documentLimitBytes: 1000000, requestLimitBytes: 64000000, previousDocumentLimit: previous ?? null, previousRequestLimit: previousRequestLimit ?? null, cases: records, batches: [] }; +try { + assert((await fetch(uri + configPath, { method: 'PUT', headers, body: JSON.stringify('1000000') })).ok); + assert((await fetch(uri + requestConfigPath, { method: 'PUT', headers, body: JSON.stringify('64000000') })).ok); + assert((await fetch(`${uri}/${database}`, { method: 'PUT', headers })).ok); + for (const condition of cases) { + const input = inputs[condition.kind]; + const chunks = await splitPiecesRabinKarp(input, 102400 * (condition.customChunkSize + 1), condition.kind !== 'binary', 20); + let largest = ''; + for await (const chunk of chunks()) if (chunk.length > largest.length) largest = chunk; + const id = `h:+${createHash('sha256').update(JSON.stringify(condition)).digest('hex')}`; + let doc = { _id: id, type: 'leaf', data: largest }; + if (condition.compression) doc = await compressDoc(doc); + if (condition.encryption !== 'none') { + const { incoming, outgoing } = getConfiguredFunctionsForEncryption('public-json-budget-probe-passphrase', false, false, async () => new Uint8Array(16), condition.encryption); + const encrypted = await incoming(doc); + assert.equal(encrypted.e_, true); + const decoded = await outgoing(structuredClone(encrypted)); + assert.equal(decoded.data, doc.data); + doc = encrypted; + } + doc = { ...doc, _rev: `1-${'0'.repeat(32)}`, _revisions: { start: 1, ids: ['0'.repeat(32)] } }; + const jsonBytes = Buffer.byteLength(JSON.stringify(doc)); + const body = JSON.stringify({ docs: [doc], new_edits: false }); + const response = await fetch(`${uri}/${database}/_bulk_docs`, { method: 'POST', headers, body }); + const result = await response.json(); + const accepted = response.ok && Array.isArray(result) && result.every(entry => !entry.error); + const measurement = { ...condition, largestPieceUtf8Bytes: Buffer.byteLength(largest), jsonBytes, jsonBytesUnderLimit: jsonBytes < record.documentLimitBytes, requestBodyBytes: Buffer.byteLength(body), status: response.status, accepted, response: result }; + records.push(measurement); + await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2)); + if (condition.expectedAccepted !== undefined) assert.equal(accepted, condition.expectedAccepted, JSON.stringify(measurement)); + assert(Buffer.byteLength(body) < record.requestLimitBytes); + console.log(JSON.stringify({ ...measurement, response: undefined })); + } + const batchInput = new Uint8Array(512000 * 25); + for (let index = 0; index < 25; index++) batchInput[(index + 1) * 512000 - 1] = index + 1; + const splitBatch = await splitPiecesRabinKarp(new Blob([batchInput], { type: 'text/plain' }), 512000, true, 20); + const batchPieces = []; + for await (const piece of splitBatch()) batchPieces.push(piece); + assert.equal(batchPieces.length, 25); + assert(batchPieces.every(piece => Buffer.byteLength(piece) === 512000)); + assert.equal(new Set(batchPieces).size, 25); + for (const count of [25, 20]) { + const docs = batchPieces.slice(0, count).map((piece, index) => ({ + _id: `h:+${createHash('sha256').update(`escaped-batch-${count}-${index}`).digest('hex')}`, + type: 'leaf', data: piece, + _rev: `1-${'0'.repeat(32)}`, _revisions: { start: 1, ids: ['0'.repeat(32)] }, + })); + const body = JSON.stringify({ docs, new_edits: false }); + const response = await fetch(`${uri}/${database}/_bulk_docs`, { method: 'POST', headers, body }); + const result = await response.json(); + const accepted = response.ok && Array.isArray(result) && result.every(entry => !entry.error); + const measurement = { count, rawBytesPerLeaf: 512000, requestBodyBytes: Buffer.byteLength(body), largestDocumentJsonBytes: Math.max(...docs.map(doc => Buffer.byteLength(JSON.stringify(doc)))), status: response.status, accepted, response: result }; + record.batches.push(measurement); + await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2)); + assert.equal(accepted, count === 20, JSON.stringify(measurement)); + if (count === 25) assert.equal(response.status, 413); + console.log(JSON.stringify({ ...measurement, response: undefined })); + } +} finally { + await fetch(`${uri}/${database}`, { method: 'DELETE', headers }); + const restored = await fetch(uri + configPath, previous === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previous) }); + record.configurationRestored = restored.ok; + const requestRestored = await fetch(uri + requestConfigPath, previousRequestLimit === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previousRequestLimit) }); + record.requestConfigurationRestored = requestRestored.ok; + await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2)); + assert(record.configurationRestored); + assert(record.requestConfigurationRestored); +} diff --git a/utils/chunk-size-benchmark/cloudant-limit.mjs b/utils/chunk-size-benchmark/cloudant-limit.mjs new file mode 100644 index 00000000..b88d8fed --- /dev/null +++ b/utils/chunk-size-benchmark/cloudant-limit.mjs @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const options = new Map(); +for (let index = 2; index < process.argv.length; index += 2) { + assert(['--output', '--corpus'].includes(process.argv[index]), `Unknown option: ${process.argv[index]}`); + assert(process.argv[index + 1], `Missing value for ${process.argv[index]}`); + options.set(process.argv[index], process.argv[index + 1]); +} +const output = resolve(options.get('--output') || '.bench/chunk-size-cloudant-model'); +const corpusPath = resolve(options.get('--corpus') || '.bench/chunk-size/corpus/manifest.json'); +const configUri = 'http://127.0.0.1:15994/_node/_local/_config/couchdb/max_document_size'; +const headers = { Authorization: `Basic ${Buffer.from('bench:benchmark-only-password').toString('base64')}`, 'Content-Type': 'application/json' }; +const previousResponse = await fetch(configUri, { headers }); +assert(previousResponse.ok || previousResponse.status === 404); +const previous = previousResponse.ok ? await previousResponse.json() : undefined; +const manifest = JSON.parse(await fs.readFile(corpusPath, 'utf8')); +manifest.stages = manifest.stages.filter((stage) => stage.name === 'initial'); +for (const stage of manifest.stages) stage.directory = resolve(dirname(corpusPath), stage.directory); +await fs.mkdir(output, { recursive: true }); +const initialManifest = join(output, 'initial-only-manifest.json'); +await fs.writeFile(initialManifest, JSON.stringify(manifest, null, 2)); +const record = { scope: 'CouchDB with a 1,000,000-byte document limit, modelling the published Cloudant document-size constraint. The IBM Cloudant service itself is not measured.', source: 'https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-limits', limitBytes: 1000000, publishedRequestLimitBytes: 64000000, variant: process.env.LIVESYNC_BENCH_VARIANT || 'unlabelled', previousConfiguredValue: previous ?? null }; +try { + const configured = await fetch(configUri, { method: 'PUT', headers, body: JSON.stringify(String(record.limitBytes)) }); + assert(configured.ok, `Could not set fixture document limit: ${configured.status}`); + const child = spawn(process.execPath, [join(root, 'utils/chunk-size-benchmark/remote.mjs'), '--output', output, '--corpus', initialManifest, '--providers', 'couchdb', '--sizes', '4,10', '--repeats', '1'], { + cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=3072' }, + }); + let log = ''; child.stdout.on('data', (data) => { log += data; process.stdout.write(data); }); child.stderr.on('data', (data) => { log += data; }); + record.exitCode = await new Promise((accept, reject) => { child.once('error', reject); child.once('close', accept); }); + await fs.writeFile(join(output, 'run.log'), log); + const measured = JSON.parse(await fs.readFile(join(output, 'remote/results-rtt0.json'), 'utf8')); + record.cases = measured.results.map((entry) => ({ name: entry.name, customChunkSize: entry.customChunkSize, complete: entry.complete, error: entry.error, stages: entry.stages.map((stage) => ({ name: stage.name, largestDocumentJsonBytes: stage.remote.largestDocumentJsonBytes, maxUploadRequestBodyBytes: stage.upload.http.maxRequestBodyBytes, verifiedFiles: stage.verifiedFiles })) })); + assert(record.cases.some((entry) => entry.customChunkSize === 4 && entry.complete), 'The 500 KiB candidate must complete under the modelled limit'); + assert(record.cases.some((entry) => entry.customChunkSize === 10 && !entry.complete), 'The 1,100 KiB candidate must hit the modelled document limit'); + const rejectionLog = await fs.readFile(join(output, 'remote/couchdb-c10-r1/logs/initial-upload.txt'), 'utf8'); + record.sizeRejectionObserved = /413|document_too_large|document.*too large|too_large|request entity too large/iu.test(rejectionLog); + assert(record.sizeRejectionObserved, 'The rejection log must identify the document-size limit'); +} finally { + const restored = await fetch(configUri, previous === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previous) }); + record.configurationRestored = restored.ok; + await fs.writeFile(join(output, 'constraint-result.json'), JSON.stringify(record, null, 2)); + assert(record.configurationRestored, 'Fixture document limit restoration failed'); +} +console.log(JSON.stringify(record)); diff --git a/utils/chunk-size-benchmark/compose.yml b/utils/chunk-size-benchmark/compose.yml new file mode 100644 index 00000000..275fa271 --- /dev/null +++ b/utils/chunk-size-benchmark/compose.yml @@ -0,0 +1,30 @@ +services: + relay: + image: ghcr.io/hoytech/strfry@sha256:36f1886d185a88ca57c66ebe52e6e9e8428dac2486eea0a5d50ff934f18b60c3 + command: ["--config", "/etc/strfry.conf", "relay"] + volumes: + - ../../test/fixtures/p2p-relay/strfry.conf:/etc/strfry.conf:ro + tmpfs: + - /app/strfry-db:rw,size=128m,mode=1777 + ports: + - "127.0.0.1:14094:7777" + mem_limit: 256m + couchdb: + image: couchdb:3.5.0@sha256:1c0a30a54535377bc9ae23f42efb1ca4e92abe796f78ec98b768ac1d5874cc1b + environment: + COUCHDB_USER: bench + COUCHDB_PASSWORD: benchmark-only-password + COUCHDB_SINGLE_NODE: "true" + ports: + - "127.0.0.1:15994:5984" + mem_limit: 768m + object-storage: + image: rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035 + environment: + RUSTFS_ACCESS_KEY: bench-access-key + RUSTFS_SECRET_KEY: benchmark-only-secret-key + RUSTFS_CONSOLE_ENABLE: "false" + command: ["/data"] + ports: + - "127.0.0.1:19094:9000" + mem_limit: 1g diff --git a/utils/chunk-size-benchmark/corpus.mjs b/utils/chunk-size-benchmark/corpus.mjs new file mode 100644 index 00000000..8c7232f6 --- /dev/null +++ b/utils/chunk-size-benchmark/corpus.mjs @@ -0,0 +1,310 @@ +import { createHash } from "node:crypto"; +import { copyFile, mkdir, readFile, readdir, stat, utimes, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const DEFAULT_REPO_ROOT = path.resolve(MODULE_DIR, "../.."); +const SEED = 0x4c53594e; +const KIB = 1024; +const MIB = 1024 * KIB; +const BINARY_BYTES = 8 * MIB; +const SMALL_MARKDOWN_BYTES = 512 * KIB; +const LARGE_MARKDOWN_BYTES = 5 * MIB; +const BINARY_EDIT_BYTES = 4 * KIB; +const INITIAL_MTIME = new Date("2020-01-01T00:00:00.000Z"); +const STAGE_MTIMES = { + overwrite: new Date("2020-01-02T00:00:00.000Z"), + insert: new Date("2020-01-03T00:00:00.000Z"), +}; + +const MARKDOWN_FILES = [ + "README.md", + "devs.md", + "docs/settings.md", + "docs/troubleshooting.md", + "docs/quick_setup.md", + "docs/datastructure.md", + "docs/p2p.md", + "docs/tips/fast-setup.md", + "docs/design_docs/intention_of_chunks.md", + "docs/specs_data_compression.md", +]; + +const SNIPPET_FILES = [ + "docs/terms.md", + "docs/glossary.md", + "docs/quick_setup_ja.md", + "docs/design_docs/unicode_path_identity.md", + "docs/settings.md", +]; + +const CORPUS_PATHS = { + markdownSmall: "benchmark/generated-small.md", + markdownLarge: "benchmark/generated-large.md", + binary: "benchmark/random-8m.bin", + cloudantImage: "assets/cloudant-setup.png", + setupImage: "assets/quick-setup-first-setup-uri.png", +}; + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function createPrng(seed) { + let state = seed >>> 0; + return () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return state >>> 0; + }; +} + +function makeRandomBytes(length, seed) { + const next = createPrng(seed); + const output = Buffer.allocUnsafe(length); + for (let offset = 0; offset < length; offset += 4) { + const value = next(); + output[offset] = value & 0xff; + if (offset + 1 < length) output[offset + 1] = (value >>> 8) & 0xff; + if (offset + 2 < length) output[offset + 2] = (value >>> 16) & 0xff; + if (offset + 3 < length) output[offset + 3] = (value >>> 24) & 0xff; + } + return output; +} + +function extractParagraphs(source) { + const withoutFrontmatter = source.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/u, ""); + return withoutFrontmatter + .replace(/\r\n/gu, "\n") + .split(/\n{2,}/u) + .map((paragraph) => paragraph.trim()) + .filter((paragraph) => { + if (paragraph.length < 80) return false; + if (/^(?:#{1,6}\s|!\[|\|)/u.test(paragraph)) return false; + return true; + }); +} + +async function readSnippetParagraphs(repoRoot) { + const excerpts = []; + for (const file of SNIPPET_FILES) { + const source = await readFile(path.join(repoRoot, file), "utf8"); + for (const paragraph of extractParagraphs(source)) { + const excerpt = Array.from(paragraph).slice(0, 1000).join(""); + excerpts.push({ source: file, text: excerpt }); + } + } + if (excerpts.length === 0) throw new Error("No repository Markdown snippets were available."); + return excerpts; +} + +function buildGeneratedMarkdown({ targetBytes, title, variant, excerpts }) { + const lines = [ + `# ${title}`, + "", + "A deterministic UTF-8 fixture contains varied passages from the project documentation.", + "", + ]; + let byteLength = Buffer.byteLength(lines.join("\n"), "utf8"); + const profiles = ["replication", "metadata", "recovery", "integrity", "configuration", "history", "incremental", "transport"]; + const unicodeLabels = ["résumé", "café", "日本語", "中文", "Δοκιμή", "ångström", "naïve", "雪", "λ", "mañana"]; + let record = 0; + while (byteLength < targetBytes) { + const excerpt = excerpts[(record * 7 + variant.length) % excerpts.length]; + const profile = profiles[(record * 3 + variant.length) % profiles.length]; + const unicode = unicodeLabels[(record * 5 + variant.length) % unicodeLabels.length]; + const tag = ((SEED ^ (record * 0x9e3779b1)) >>> 0).toString(16).padStart(8, "0"); + const paragraph = [ + `## Record ${String(record + 1).padStart(6, "0")} · ${profile} · ${variant}`, + "", + excerpt.text, + "", + `Fixture label ${tag} uses ${unicode}; source excerpt: ${excerpt.source}; revision ${record % 97}.`, + "", + ].join("\n"); + lines.push(paragraph); + byteLength += Buffer.byteLength(paragraph, "utf8"); + record += 1; + } + return lines.join("\n"); +} + +function insertNearStart(markdown, paragraph) { + const firstGap = markdown.indexOf("\n\n"); + const introGap = markdown.indexOf("\n\n", firstGap + 2); + if (firstGap < 0 || introGap < 0) throw new Error("Generated Markdown fixture has no introductory paragraph."); + const insertionPoint = introGap + 2; + return `${markdown.slice(0, insertionPoint)}${paragraph}\n\n${markdown.slice(insertionPoint)}`; +} + +async function writeFileWithMtime(directory, relativePath, contents, mtime) { + const outputPath = path.join(directory, relativePath); + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, contents); + await utimes(outputPath, mtime, mtime); +} + +async function copyStage(sourceDirectory, targetDirectory, relativePaths) { + for (const relativePath of relativePaths) { + const sourcePath = path.join(sourceDirectory, relativePath); + const targetPath = path.join(targetDirectory, relativePath); + await mkdir(path.dirname(targetPath), { recursive: true }); + const sourceStat = await stat(sourcePath); + await copyFile(sourcePath, targetPath); + await utimes(targetPath, sourceStat.atime, sourceStat.mtime); + } +} + +async function setMtime(directory, relativePath, mtime) { + const outputPath = path.join(directory, relativePath); + await utimes(outputPath, mtime, mtime); +} + +async function listFiles(directory) { + const files = []; + async function visit(currentDirectory) { + const entries = await readdir(currentDirectory, { withFileTypes: true }); + entries.sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + const fullPath = path.join(currentDirectory, entry.name); + if (entry.isDirectory()) { + await visit(fullPath); + } else if (entry.isFile()) { + files.push(path.relative(directory, fullPath).split(path.sep).join("/")); + } + } + } + await visit(directory); + return files.sort(); +} + +async function describeStage(stageName, stageDirectory, paths) { + const files = []; + for (const relativePath of paths) { + const bytes = await readFile(path.join(stageDirectory, relativePath)); + files.push({ path: relativePath, bytes: bytes.byteLength, sha256: sha256(bytes) }); + } + return { name: stageName, directory: stageName, files }; +} + +export async function createCorpus(outDir, repoRoot = DEFAULT_REPO_ROOT) { + const outputDirectory = path.resolve(outDir); + const sourceRoot = path.resolve(repoRoot); + const initialDirectory = path.join(outputDirectory, "initial"); + const overwriteDirectory = path.join(outputDirectory, "overwrite"); + const insertDirectory = path.join(outputDirectory, "insert"); + + await mkdir(outputDirectory, { recursive: true }); + + const initialContents = new Map(); + for (const relativePath of MARKDOWN_FILES) { + initialContents.set(relativePath, await readFile(path.join(sourceRoot, relativePath))); + } + + const excerpts = await readSnippetParagraphs(sourceRoot); + initialContents.set( + CORPUS_PATHS.markdownSmall, + Buffer.from( + buildGeneratedMarkdown({ + targetBytes: SMALL_MARKDOWN_BYTES, + title: "Generated Markdown · 512 KiB", + variant: "small", + excerpts, + }), + "utf8", + ), + ); + initialContents.set( + CORPUS_PATHS.markdownLarge, + Buffer.from( + buildGeneratedMarkdown({ + targetBytes: LARGE_MARKDOWN_BYTES, + title: "Generated Markdown · 5 MiB", + variant: "large", + excerpts, + }), + "utf8", + ), + ); + initialContents.set(CORPUS_PATHS.binary, makeRandomBytes(BINARY_BYTES, SEED)); + initialContents.set(CORPUS_PATHS.cloudantImage, await readFile(path.join(sourceRoot, "instruction_images/cloudant_1.png"))); + initialContents.set(CORPUS_PATHS.setupImage, await readFile(path.join(sourceRoot, "images/quick-setup/guide-quick-setup-first-setup-uri.png"))); + + const relativePaths = [...initialContents.keys()].sort(); + for (const [relativePath, contents] of initialContents) { + await writeFileWithMtime(initialDirectory, relativePath, contents, INITIAL_MTIME); + } + + await copyStage(initialDirectory, overwriteDirectory, relativePaths); + const smallMarkdownPath = path.join(overwriteDirectory, CORPUS_PATHS.markdownSmall); + const smallMarkdownBefore = await readFile(smallMarkdownPath, "utf8"); + const originalIntro = "A deterministic UTF-8 fixture contains varied passages from the project documentation."; + const changedIntro = "A deterministic UTF-8 fixture records a short content update using varied project passages."; + if (!smallMarkdownBefore.includes(originalIntro)) throw new Error("Could not locate the generated Markdown paragraph to overwrite."); + await writeFile(smallMarkdownPath, smallMarkdownBefore.replace(originalIntro, changedIntro)); + const binaryPath = path.join(overwriteDirectory, CORPUS_PATHS.binary); + const initialBinary = await readFile(binaryPath); + const binaryOverwriteOffset = Math.floor(initialBinary.byteLength * 0.25); + const overwrittenBinary = Buffer.from(initialBinary); + makeRandomBytes(BINARY_EDIT_BYTES, (SEED ^ 0x00a55a5a) >>> 0).copy(overwrittenBinary, binaryOverwriteOffset); + await writeFile(binaryPath, overwrittenBinary); + const alternatePng = await readFile(path.join(sourceRoot, "instruction_images/cloudant_2.png")); + await writeFile(path.join(overwriteDirectory, CORPUS_PATHS.cloudantImage), alternatePng); + for (const relativePath of [CORPUS_PATHS.markdownSmall, CORPUS_PATHS.binary, CORPUS_PATHS.cloudantImage]) { + await setMtime(overwriteDirectory, relativePath, STAGE_MTIMES.overwrite); + } + + await copyStage(overwriteDirectory, insertDirectory, relativePaths); + const insertedBinaryPath = path.join(insertDirectory, CORPUS_PATHS.binary); + const overwrittenBinaryForInsert = await readFile(insertedBinaryPath); + const binaryInsertOffset = Math.floor(overwrittenBinaryForInsert.byteLength * 0.25); + const insertedBinary = Buffer.concat([ + overwrittenBinaryForInsert.subarray(0, binaryInsertOffset), + makeRandomBytes(BINARY_EDIT_BYTES, (SEED ^ 0x005aa5a5) >>> 0), + overwrittenBinaryForInsert.subarray(binaryInsertOffset), + ]); + await writeFile(insertedBinaryPath, insertedBinary); + + const insertedParagraphs = [ + [CORPUS_PATHS.markdownSmall, "The insert stage adds record 0047, profile delta-small, while retaining Unicode labels such as 日本語 and café."], + [CORPUS_PATHS.markdownLarge, "The insert stage adds record 0913, profile delta-large, while retaining Unicode labels such as 中文 and Δοκιμή."], + ]; + for (const [relativePath, paragraph] of insertedParagraphs) { + const markdownPath = path.join(insertDirectory, relativePath); + const markdown = await readFile(markdownPath, "utf8"); + await writeFile(markdownPath, insertNearStart(markdown, paragraph)); + } + for (const relativePath of [CORPUS_PATHS.binary, ...insertedParagraphs.map(([relativePath]) => relativePath)]) { + await setMtime(insertDirectory, relativePath, STAGE_MTIMES.insert); + } + + const actualPaths = await listFiles(initialDirectory); + for (const directory of [overwriteDirectory, insertDirectory]) { + const stagePaths = await listFiles(directory); + if (JSON.stringify(stagePaths) !== JSON.stringify(actualPaths)) { + throw new Error(`Corpus stage ${path.basename(directory)} has different relative paths.`); + } + } + + const sourceForPath = new Map(MARKDOWN_FILES.map((file) => [file, `repo:${file}`])); + sourceForPath.set(CORPUS_PATHS.markdownSmall, "generated:utf8-markdown-512KiB-from-repository-snippets"); + sourceForPath.set(CORPUS_PATHS.markdownLarge, "generated:utf8-markdown-5MiB-from-repository-snippets"); + sourceForPath.set(CORPUS_PATHS.binary, `generated:seeded-high-entropy-binary-seed-${SEED.toString(16)}`); + sourceForPath.set(CORPUS_PATHS.cloudantImage, "repo:instruction_images/cloudant_1.png"); + sourceForPath.set(CORPUS_PATHS.setupImage, "repo:images/quick-setup/guide-quick-setup-first-setup-uri.png"); + + const files = actualPaths.map((relativePath) => { + const kind = relativePath.endsWith(".md") ? "markdown" : "binary"; + return { path: relativePath, kind, source: sourceForPath.get(relativePath) }; + }); + const stages = [ + await describeStage("initial", initialDirectory, actualPaths), + await describeStage("overwrite", overwriteDirectory, actualPaths), + await describeStage("insert", insertDirectory, actualPaths), + ]; + const manifest = { seed: SEED, files, stages }; + await writeFile(path.join(outputDirectory, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + return manifest; +} diff --git a/utils/chunk-size-benchmark/datachannel-metrics.cjs b/utils/chunk-size-benchmark/datachannel-metrics.cjs new file mode 100644 index 00000000..fe13ed31 --- /dev/null +++ b/utils/chunk-size-benchmark/datachannel-metrics.cjs @@ -0,0 +1,23 @@ +const { writeFileSync } = require('node:fs'); +const { RTCDataChannel } = require('werift'); + +const output = process.env.LIVESYNC_BENCH_DATACHANNEL_JSON; +if (!output) throw new Error('LIVESYNC_BENCH_DATACHANNEL_JSON is required'); +const channels = new Set(); +const originalSend = RTCDataChannel.prototype.send; +RTCDataChannel.prototype.send = function (...args) { + channels.add(this); + return originalSend.apply(this, args); +}; +process.once('exit', () => { + const rows = [...channels].map((channel) => ({ label: channel.label, payloadBytesSent: channel.bytesSent, messagesSent: channel.messagesSent })); + const cpu = process.cpuUsage(); + writeFileSync(output, JSON.stringify({ + source: 'werift 0.24.4 RTCDataChannel counters; observed channels through a benchmark-only preload', + scope: 'Application DataChannel payload; includes application retransmissions, excludes SCTP/DTLS/UDP/IP overhead, transport retransmissions, and relay signalling', + payloadBytesSent: rows.reduce((sum, row) => sum + row.payloadBytesSent, 0), + messagesSent: rows.reduce((sum, row) => sum + row.messagesSent, 0), + processUsage: { userCpuMs: cpu.user / 1000, systemCpuMs: cpu.system / 1000, maxResidentSetKiB: process.resourceUsage().maxRSS }, + channels: rows, + }, null, 2)); +}); diff --git a/utils/chunk-size-benchmark/local.mjs b/utils/chunk-size-benchmark/local.mjs new file mode 100644 index 00000000..6c5eeb98 --- /dev/null +++ b/utils/chunk-size-benchmark/local.mjs @@ -0,0 +1,596 @@ +import { createHash } from "node:crypto"; +import { readFile, mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { HashManager } from "@vrtmrz/livesync-commonlib/hashing"; +import { createCorpus } from "./corpus.mjs"; + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const PROJECT_ROOT = path.resolve(MODULE_DIR, "../.."); +const DEFAULT_SIZES = [0, 1, 4, 10, 20, 40, 60]; +const DEFAULT_REPEATS = 3; +const ID_DERIVATION_KEY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const LEGACY_PASSPHRASE = "public-local-chunk-size-benchmark-fixture"; +const CHUNK_MANAGER_WITHOUT_CACHE = { + getCachedChunk: () => undefined, + getChunkIDFromCache: () => false, +}; + +const COMMONLIB_PACKAGE_JSON = fileURLToPath(import.meta.resolve("@vrtmrz/livesync-commonlib/package.json")); +const COMMONLIB_ROOT = path.dirname(COMMONLIB_PACKAGE_JSON); +const COMMONLIB_PACKAGE = JSON.parse(await readFile(COMMONLIB_PACKAGE_JSON, "utf8")); +const { ContentSplitterRabinKarp } = await import( + pathToFileURL(path.join(COMMONLIB_ROOT, "dist/ContentSplitter/ContentSplitterRabinKarp.js")).href +); +const { prepareChunk } = await import( + pathToFileURL(path.join(COMMONLIB_ROOT, "dist/managers/EntryManager/EntryManagerImpls.js")).href +); + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function parseArguments(args) { + const parsed = { output: undefined, sizes: [...DEFAULT_SIZES], repeats: DEFAULT_REPEATS, help: false }; + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + if (argument === "--help" || argument === "-h") { + parsed.help = true; + continue; + } + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${argument}.`); + if (argument === "--output") { + parsed.output = value; + } else if (argument === "--sizes") { + parsed.sizes = value.split(",").map((item) => { + const trimmed = item.trim(); + if (!/^\d+$/u.test(trimmed)) throw new Error(`Invalid customChunkSize value: ${item}`); + const size = Number(trimmed); + if (!Number.isSafeInteger(size) || size > 10000) throw new Error(`customChunkSize is out of range: ${item}`); + return size; + }); + if (parsed.sizes.length === 0 || new Set(parsed.sizes).size !== parsed.sizes.length) { + throw new Error("--sizes must contain one or more distinct integers."); + } + } else if (argument === "--repeats") { + if (!/^\d+$/u.test(value)) throw new Error(`Invalid repeat count: ${value}`); + parsed.repeats = Number(value); + if (!Number.isSafeInteger(parsed.repeats) || parsed.repeats < 1 || parsed.repeats > 100) { + throw new Error("--repeats must be an integer from 1 to 100."); + } + } else { + throw new Error(`Unknown argument: ${argument}`); + } + index += 1; + } + if (!parsed.help && !parsed.output) throw new Error("--output is required."); + return parsed; +} + +function usage() { + return "Usage: node utils/chunk-size-benchmark/local.mjs --output [--sizes 0,1,4,10,20,40,60] [--repeats 3]"; +} + +function settingsFor(customChunkSize, mode) { + return { + chunkSplitterVersion: "v3-rabin-karp", + customChunkSize, + minimumChunkSize: 20, + disableWorkerForGeneratingChunks: true, + processSmallFilesInUIThread: false, + encrypt: true, + E2EEAlgorithm: "v2", + hashAlg: "xxhash64", + passphrase: mode === "legacy" ? LEGACY_PASSPHRASE : "", + idDerivationVersion: mode === "primary" ? 1 : 0, + idDerivationKey: mode === "primary" ? ID_DERIVATION_KEY : "", + enableCompression: false, + useEden: false, + }; +} + +async function createCondition(customChunkSize) { + const primarySettings = settingsFor(customChunkSize, "primary"); + const legacySettings = settingsFor(customChunkSize, "legacy"); + const primarySettingService = { currentSettings: () => primarySettings }; + const legacySettingService = { currentSettings: () => legacySettings }; + const splitter = new ContentSplitterRabinKarp({ settingService: primarySettingService }); + const primaryHashManager = new HashManager({ settingService: primarySettingService }); + const legacyHashManager = new HashManager({ settingService: legacySettingService }); + await Promise.all([splitter.initialised, primaryHashManager.initialise(), legacyHashManager.initialise()]); + return { customChunkSize, splitter, primaryHashManager, legacyHashManager }; +} + +async function warmCondition(condition, file) { + await splitAndHashFile(condition, file, new Map(), new Map()); +} + +async function loadCorpus(manifest, corpusDirectory) { + const filesByPath = new Map(manifest.files.map((file) => [file.path, file])); + const loadedStages = []; + for (const stage of manifest.stages) { + const stageDirectory = path.join(corpusDirectory, stage.directory); + const loadedFiles = []; + for (const expected of stage.files) { + const metadata = filesByPath.get(expected.path); + if (!metadata) throw new Error(`Corpus manifest has no file metadata for ${expected.path}.`); + const data = await readFile(path.join(stageDirectory, expected.path)); + if (data.byteLength !== expected.bytes || sha256(data) !== expected.sha256) { + throw new Error(`Corpus file does not match its manifest: ${stage.name}/${expected.path}`); + } + loadedFiles.push({ ...metadata, expectedSha256: expected.sha256, data }); + } + loadedStages.push({ name: stage.name, files: loadedFiles }); + } + return loadedStages; +} + +function decodeChunk(kind, piece) { + return kind === "markdown" ? Buffer.from(piece, "utf8") : Buffer.from(piece, "base64"); +} + +function assertSameContentForId(idsToPieces, id, piece, label) { + const existing = idsToPieces.get(id); + if (existing !== undefined && existing !== piece) { + throw new Error(`${label} collision: chunk ID ${id} was produced for different content.`); + } + if (existing === undefined) idsToPieces.set(id, piece); +} + +async function splitAndHashFile(condition, file, primaryIdContents, legacyIdContents) { + const blobType = file.kind === "markdown" ? "text/plain" : "application/octet-stream"; + const blob = new Blob([file.data], { type: blobType }); + const entry = { path: file.path, data: blob }; + const primaryChunks = []; + const primaryCpuStart = process.cpuUsage(); + const primaryStart = process.hrtime.bigint(); + const generator = await condition.splitter.splitContent(entry); + for await (const piece of generator) { + const prepared = await prepareChunk( + { hashManager: condition.primaryHashManager, chunkManager: CHUNK_MANAGER_WITHOUT_CACHE }, + piece, + ); + assertSameContentForId(primaryIdContents, prepared.id, piece, "Primary keyed hash"); + primaryChunks.push({ piece, id: prepared.id }); + } + const primaryElapsedMs = Number(process.hrtime.bigint() - primaryStart) / 1e6; + const primaryCpu = process.cpuUsage(primaryCpuStart); + const primaryCpuMs = (primaryCpu.user + primaryCpu.system) / 1000; + + const legacyCpuStart = process.cpuUsage(); + const legacyStart = process.hrtime.bigint(); + for (const chunk of primaryChunks) { + const prepared = await prepareChunk( + { hashManager: condition.legacyHashManager, chunkManager: CHUNK_MANAGER_WITHOUT_CACHE }, + chunk.piece, + ); + assertSameContentForId(legacyIdContents, prepared.id, chunk.piece, "Legacy encrypted xxhash64"); + chunk.legacyId = prepared.id; + } + const legacyIdElapsedMs = Number(process.hrtime.bigint() - legacyStart) / 1e6; + const legacyCpu = process.cpuUsage(legacyCpuStart); + const legacyIdCpuMs = (legacyCpu.user + legacyCpu.system) / 1000; + + const decoded = primaryChunks.map((chunk) => decodeChunk(file.kind, chunk.piece)); + const reconstructed = Buffer.concat(decoded); + if (!reconstructed.equals(file.data) || sha256(reconstructed) !== file.expectedSha256) { + throw new Error(`Splitter output did not reconstruct ${file.path} in the ${file.kind} encoding.`); + } + + const chunks = primaryChunks.map((chunk, index) => { + const raw = decoded[index]; + return { + id: chunk.id, + legacyId: chunk.legacyId, + rawBytes: raw.byteLength, + storedUtf8Bytes: Buffer.byteLength(chunk.piece, "utf8"), + dataSha256: sha256(raw), + idUtf8Bytes: Buffer.byteLength(chunk.id, "utf8"), + legacyIdUtf8Bytes: Buffer.byteLength(chunk.legacyId, "utf8"), + piece: chunk.piece, + }; + }); + const fingerprintInput = chunks.map(({ id, legacyId, rawBytes, dataSha256 }) => [id, legacyId, rawBytes, dataSha256]); + const chunkFingerprintSha256 = sha256(Buffer.from(JSON.stringify(fingerprintInput), "utf8")); + return { + chunks, + primaryElapsedMs, + primaryCpuMs, + legacyIdElapsedMs, + legacyIdCpuMs, + chunkFingerprintSha256, + }; +} + +function percentile(sortedValues, fraction) { + if (sortedValues.length === 0) return 0; + return sortedValues[Math.max(0, Math.ceil(sortedValues.length * fraction) - 1)]; +} + +function summarizeRawChunkBytes(values) { + const sorted = [...values].sort((left, right) => left - right); + return { + count: sorted.length, + p50: percentile(sorted, 0.5), + p90: percentile(sorted, 0.9), + p95: percentile(sorted, 0.95), + p99: percentile(sorted, 0.99), + max: sorted.at(-1) ?? 0, + }; +} + +function summariseIdLengths(chunks, key) { + const lengths = chunks.map((chunk) => Buffer.byteLength(chunk[key], "utf8")).sort((left, right) => left - right); + const uniqueLengths = new Map(); + for (const chunk of chunks) uniqueLengths.set(chunk[key], Buffer.byteLength(chunk[key], "utf8")); + const uniqueValues = [...uniqueLengths.values()].sort((left, right) => left - right); + return { + occurrences: lengths.length, + totalUtf8Bytes: lengths.reduce((sum, value) => sum + value, 0), + minLengthBytes: lengths[0] ?? 0, + medianLengthBytes: percentile(lengths, 0.5), + maxLengthBytes: lengths.at(-1) ?? 0, + uniqueIds: uniqueValues.length, + uniqueTotalUtf8Bytes: uniqueValues.reduce((sum, value) => sum + value, 0), + }; +} + +function uniqueById(chunks, idKey = "id") { + const unique = new Map(); + for (const chunk of chunks) { + if (!unique.has(chunk[idKey])) unique.set(chunk[idKey], chunk); + } + return unique; +} + +function dataBytes(chunks, key) { + return chunks.reduce((sum, chunk) => sum + chunk[key], 0); +} + +function summarizeFile(file, measured, includeChunkArray) { + const { chunks } = measured; + const unique = uniqueById(chunks); + const fileMetric = { + path: file.path, + kind: file.kind, + sourceBytes: file.data.byteLength, + chunkCount: chunks.length, + uniqueChunkCount: unique.size, + storedChunkDataUtf8Bytes: dataBytes(chunks, "storedUtf8Bytes"), + uniqueChunkDataUtf8Bytes: dataBytes([...unique.values()], "storedUtf8Bytes"), + uniqueChunkRawBytes: dataBytes([...unique.values()], "rawBytes"), + rawChunkBytes: summarizeRawChunkBytes(chunks.map((chunk) => chunk.rawBytes)), + primaryId: summariseIdLengths(chunks, "id"), + legacyId: summariseIdLengths(chunks, "legacyId"), + chunkFingerprintSha256: measured.chunkFingerprintSha256, + timing: { + primarySplitAndIdElapsedMs: measured.primaryElapsedMs, + primarySplitAndIdCpuMs: measured.primaryCpuMs, + legacyIdElapsedMs: measured.legacyIdElapsedMs, + legacyIdCpuMs: measured.legacyIdCpuMs, + }, + }; + if (includeChunkArray) { + fileMetric.chunks = chunks.map(({ id, legacyId, rawBytes, storedUtf8Bytes, dataSha256, idUtf8Bytes, legacyIdUtf8Bytes }) => ({ + id, + legacyId, + rawBytes, + storedUtf8Bytes, + dataSha256, + idUtf8Bytes, + legacyIdUtf8Bytes, + })); + } + return fileMetric; +} + +function aggregateKind(stage, kind, fileResults, historyIds, previousIds) { + const files = fileResults.filter((item) => item.file.kind === kind); + const chunks = files.flatMap((item) => item.measured.chunks); + const unique = uniqueById(chunks); + const uniqueChunks = [...unique.values()]; + const newVsHistory = uniqueChunks.filter((chunk) => !historyIds.has(chunk.id)); + const newVsPrevious = previousIds === null ? null : uniqueChunks.filter((chunk) => !previousIds.has(chunk.id)); + const storedBytes = dataBytes(chunks, "storedUtf8Bytes"); + const primaryIds = summariseIdLengths(chunks, "id"); + const legacyIds = summariseIdLengths(chunks, "legacyId"); + + return { + kind, + fileCount: files.length, + sourceBytes: files.reduce((sum, item) => sum + item.file.data.byteLength, 0), + chunkCount: chunks.length, + uniqueChunkCount: unique.size, + storedChunkDataUtf8Bytes: storedBytes, + uniqueChunkDataUtf8Bytes: dataBytes(uniqueChunks, "storedUtf8Bytes"), + uniqueChunkRawBytes: dataBytes(uniqueChunks, "rawBytes"), + rawChunkBytes: summarizeRawChunkBytes(chunks.map((chunk) => chunk.rawBytes)), + primaryId: primaryIds, + legacyId: legacyIds, + newUniqueChunksVsHistory: newVsHistory.length, + newUniqueChunkDataUtf8BytesVsHistory: dataBytes(newVsHistory, "storedUtf8Bytes"), + newUniqueChunkRawBytesVsHistory: dataBytes(newVsHistory, "rawBytes"), + newUniqueChunksVsPreviousStage: newVsPrevious === null ? null : newVsPrevious.length, + newUniqueChunkDataUtf8BytesVsPreviousStage: newVsPrevious === null ? null : dataBytes(newVsPrevious, "storedUtf8Bytes"), + newUniqueChunkRawBytesVsPreviousStage: newVsPrevious === null ? null : dataBytes(newVsPrevious, "rawBytes"), + primaryElapsedMs: files.reduce((sum, item) => sum + item.measured.primaryElapsedMs, 0), + primaryCpuMs: files.reduce((sum, item) => sum + item.measured.primaryCpuMs, 0), + legacyIdElapsedMs: files.reduce((sum, item) => sum + item.measured.legacyIdElapsedMs, 0), + legacyIdCpuMs: files.reduce((sum, item) => sum + item.measured.legacyIdCpuMs, 0), + }; +} + +async function runCondition(condition, stages, repeat, order, includeChunkArray) { + const primaryIdContents = new Map(); + const legacyIdContents = new Map(); + const historyIds = new Set(); + let previousIds = null; + const stageResults = []; + + for (const stage of stages) { + const fileResults = []; + for (const file of stage.files) { + const measured = await splitAndHashFile(condition, file, primaryIdContents, legacyIdContents); + fileResults.push({ file, measured }); + } + + const currentIds = new Set(); + for (const item of fileResults) { + for (const chunk of item.measured.chunks) currentIds.add(chunk.id); + } + const kinds = [...new Set(fileResults.map((item) => item.file.kind))].sort(); + const byKind = kinds.map((kind) => aggregateKind(stage, kind, fileResults, historyIds, previousIds)); + const stageFingerprint = fileResults + .map((item) => `${item.file.path}\0${item.measured.chunkFingerprintSha256}`) + .sort() + .join("\n"); + stageResults.push({ + name: stage.name, + fingerprintSha256: sha256(Buffer.from(stageFingerprint, "utf8")), + byKind, + files: fileResults.map((item) => summarizeFile(item.file, item.measured, includeChunkArray)), + }); + + for (const id of currentIds) historyIds.add(id); + previousIds = currentIds; + } + + return { repeat, conditionOrder: order, stages: stageResults }; +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + if (sorted.length === 0) return 0; + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; +} + +function orderForRepeat(sizes, repeat) { + const offset = repeat % sizes.length; + return [...sizes.slice(offset), ...sizes.slice(0, offset)]; +} + +function timingsForRuns(runs, stageName, kind) { + const samples = runs.map((run) => { + const stage = run.stages.find((item) => item.name === stageName); + const group = stage.byKind.find((item) => item.kind === kind); + return { + repeat: run.repeat, + primaryElapsedMs: group.primaryElapsedMs, + primaryCpuMs: group.primaryCpuMs, + legacyIdElapsedMs: group.legacyIdElapsedMs, + legacyIdCpuMs: group.legacyIdCpuMs, + }; + }); + return { + primarySplitAndIdElapsedMsMedian: median(samples.map((sample) => sample.primaryElapsedMs)), + primarySplitAndIdCpuMsMedian: median(samples.map((sample) => sample.primaryCpuMs)), + legacyIdElapsedMsMedian: median(samples.map((sample) => sample.legacyIdElapsedMs)), + legacyIdCpuMsMedian: median(samples.map((sample) => sample.legacyIdCpuMs)), + repeats: samples, + }; +} + +function timingsForFile(runs, stageName, filePath) { + const samples = runs.map((run) => { + const stage = run.stages.find((item) => item.name === stageName); + const file = stage.files.find((item) => item.path === filePath); + return { repeat: run.repeat, ...file.timing }; + }); + return { + primarySplitAndIdElapsedMsMedian: median(samples.map((sample) => sample.primarySplitAndIdElapsedMs)), + primarySplitAndIdCpuMsMedian: median(samples.map((sample) => sample.primarySplitAndIdCpuMs)), + legacyIdElapsedMsMedian: median(samples.map((sample) => sample.legacyIdElapsedMs)), + legacyIdCpuMsMedian: median(samples.map((sample) => sample.legacyIdCpuMs)), + repeats: samples, + }; +} + +function assertRepeatStability(runs, customChunkSize) { + const first = runs[0]; + for (const run of runs.slice(1)) { + for (const stage of first.stages) { + const currentStage = run.stages.find((item) => item.name === stage.name); + if (currentStage.fingerprintSha256 !== stage.fingerprintSha256) { + throw new Error(`Chunk boundaries or IDs changed across repeats for customChunkSize=${customChunkSize}, stage=${stage.name}.`); + } + for (const file of stage.files) { + const currentFile = currentStage.files.find((item) => item.path === file.path); + if (currentFile.chunkFingerprintSha256 !== file.chunkFingerprintSha256) { + throw new Error(`Chunk IDs changed across repeats for ${stage.name}/${file.path} at customChunkSize=${customChunkSize}.`); + } + } + } + } +} + +function compareCandidateBoundaries(candidates, sizes) { + const candidate40 = candidates.get(40); + const candidate60 = candidates.get(60); + if (!sizes.includes(40) || !sizes.includes(60)) return null; + for (const stage of candidate40.stages) { + const otherStage = candidate60.stages.find((item) => item.name === stage.name); + for (const file of stage.files) { + const other = otherStage.files.find((item) => item.path === file.path); + if (file.chunkFingerprintSha256 !== other.chunkFingerprintSha256) { + throw new Error(`customChunkSize=40 and 60 differ for ${stage.name}/${file.path}.`); + } + } + } + return { + customChunkSize40And60SamePerFileBoundariesAndIds: true, + comparedStages: candidate40.stages.length, + comparedFilesPerStage: candidate40.stages[0]?.files.length ?? 0, + }; +} + +async function main() { + const args = parseArguments(process.argv.slice(2)); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return; + } + + const outputDirectory = path.resolve(args.output); + const corpusDirectory = path.join(outputDirectory, "corpus"); + const resultPath = path.join(outputDirectory, "results.json"); + await mkdir(outputDirectory, { recursive: true }); + const startedAt = new Date().toISOString(); + if (COMMONLIB_PACKAGE.version !== "0.1.44") { + throw new Error(`Expected @vrtmrz/livesync-commonlib 0.1.44, found ${COMMONLIB_PACKAGE.version}.`); + } + const manifest = await createCorpus(corpusDirectory, PROJECT_ROOT); + const manifestJson = JSON.stringify(manifest); + const corpusManifestSha256 = sha256(Buffer.from(manifestJson, "utf8")); + const stages = await loadCorpus(manifest, corpusDirectory); + const packageJson = JSON.parse(await readFile(path.join(PROJECT_ROOT, "package.json"), "utf8")); + + if (ID_DERIVATION_KEY.length !== 64 || !/^[0-9a-f]{64}$/u.test(ID_DERIVATION_KEY)) { + throw new Error("The benchmark ID derivation fixture must be 64 lower-case hex characters."); + } + + const conditions = new Map(); + for (const size of args.sizes) { + const condition = await createCondition(size); + const warmupFile = stages[0].files.find((file) => file.kind === "markdown" && file.source.startsWith("repo:")); + if (!warmupFile) throw new Error("Corpus has no repository Markdown file for warm-up."); + await warmCondition(condition, warmupFile); + conditions.set(size, condition); + } + + const runsBySize = new Map(args.sizes.map((size) => [size, []])); + for (let repeat = 0; repeat < args.repeats; repeat += 1) { + const order = orderForRepeat(args.sizes, repeat); + for (const size of order) { + const includeChunkArray = repeat === 0 && (size === 40 || size === 60); + const run = await runCondition(conditions.get(size), stages, repeat + 1, order, includeChunkArray); + runsBySize.get(size).push(run); + } + } + + const candidates = new Map(); + for (const size of args.sizes) { + const runs = runsBySize.get(size); + assertRepeatStability(runs, size); + const firstRun = runs[0]; + const detailedStages = firstRun.stages.map((stage) => ({ + name: stage.name, + fingerprintSha256: stage.fingerprintSha256, + byKind: stage.byKind.map((group) => { + const { primaryElapsedMs, primaryCpuMs, legacyIdElapsedMs, legacyIdCpuMs, ...metrics } = group; + return { + ...metrics, + timing: timingsForRuns(runs, stage.name, group.kind), + }; + }), + files: stage.files.map(({ timing, ...file }) => ({ + ...file, + timing: timingsForFile(runs, stage.name, file.path), + })), + })); + const candidate = { + customChunkSize: size, + chunkSizeCeilingBytes: Math.floor(102400 * (size + 1)), + stages: detailedStages, + runs: runs.map((run) => ({ + repeat: run.repeat, + conditionOrder: run.conditionOrder, + stages: run.stages.map((stage) => ({ name: stage.name, fingerprintSha256: stage.fingerprintSha256 })), + })), + }; + candidates.set(size, candidate); + } + + const comparisons = compareCandidateBoundaries(candidates, args.sizes); + const results = { + schemaVersion: 1, + benchmark: "local Rabin-Karp splitting and chunk-ID measurement", + startedAt, + completedAt: new Date().toISOString(), + runtime: { + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + osRelease: os.release(), + cpuModel: os.cpus()[0]?.model ?? "unknown", + logicalCpuCount: os.cpus().length, + projectVersion: packageJson.version, + commonlibVersion: COMMONLIB_PACKAGE.version, + variant: process.env.LIVESYNC_BENCH_VARIANT || "unlabelled", + commonlibChunksSha256: sha256(await readFile(path.join(COMMONLIB_ROOT, "dist/string_and_binary/chunks.js"))), + }, + settings: { + splitter: "v3-rabin-karp", + minimumChunkSize: 20, + workerDisabled: true, + E2EEAlgorithm: "v2", + encrypt: true, + hashAlg: "xxhash64", + primaryIdDerivation: { + version: 1, + keyFixture: "fixed public benchmark fixture; 64 lower-case hex characters omitted", + }, + legacyIdComparison: { + idDerivationVersion: 0, + algorithm: "encrypted xxhash64 with the fixed public legacy passphrase fixture", + passphrase: "omitted", + }, + enableCompression: false, + useEden: false, + customChunkSizeValues: args.sizes, + warmup: "one actual repository Markdown file per condition; excluded from timings", + timings: "primary includes splitting and actual EntryManager prepareChunk hashing; legacy timing hashes the same emitted chunks separately", + }, + corpus: { + seed: manifest.seed, + manifestSha256: corpusManifestSha256, + fileCount: manifest.files.length, + stages: manifest.stages.map((stage) => ({ + name: stage.name, + directory: stage.directory, + fileCount: stage.files.length, + sourceBytes: stage.files.reduce((sum, file) => sum + file.bytes, 0), + })), + }, + assertions: { + everyFileReconstructedFromSplitterOutput: true, + duplicateIdsHadIdenticalChunkContent: true, + sameBoundariesAndIdsAcrossRepeats: true, + ...(comparisons ?? {}), + }, + candidates: [...candidates.values()], + scope: "Local splitting and hashing only; no network or request timing is measured.", + }; + + await writeFile(resultPath, `${JSON.stringify(results, null, 2)}\n`); + process.stdout.write(`${JSON.stringify({ resultPath, corpusManifestSha256, candidateCount: args.sizes.length, repeats: args.repeats })}\n`); +} + +main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/utils/chunk-size-benchmark/p2p.mjs b/utils/chunk-size-benchmark/p2p.mjs new file mode 100644 index 00000000..b98655f9 --- /dev/null +++ b/utils/chunk-size-benchmark/p2p.mjs @@ -0,0 +1,159 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import { createNewVaultSettings, REMOTE_P2P, SALT_OF_PASSPHRASE } from '@vrtmrz/livesync-commonlib/compat/common/types'; +import { encryptString } from '@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const cli = join(root, 'src/apps/cli/dist/index.cjs'); +const preload = join(root, 'utils/chunk-size-benchmark/datachannel-metrics.cjs'); +const args = process.argv.slice(2); +const option = (key, fallback) => { const at = args.indexOf(`--${key}`); return at < 0 ? fallback : args[at + 1]; }; +const sourceCli = resolve(option('source-cli', cli)); +const targetCli = resolve(option('target-cli', cli)); +const output = resolve(option('output', join(root, '.bench/chunk-size'))); +const corpusPath = resolve(option('corpus', join(output, 'corpus/manifest.json'))); +const sizes = option('sizes', '0,4,10,40').split(',').map(Number); +const repeats = Number(option('repeats', '3')); +const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex'); +const cliSha256 = sha256(await fs.readFile(cli)); +const sourceCliSha256 = sha256(await fs.readFile(sourceCli)); +const targetCliSha256 = sha256(await fs.readFile(targetCli)); +const environment = { ...process.env, NODE_OPTIONS: '--max-old-space-size=2048' }; + +async function runCli(vault, settings, command, commandArgs, logPath, extraEnvironment = {}, entrypoint = cli) { + const marker = '__BENCH_TIME__'; + const started = performance.now(); + const nodeArgs = extraEnvironment.LIVESYNC_BENCH_DATACHANNEL_JSON ? ['--require', preload] : []; + const child = spawn('/usr/bin/time', ['-f', `${marker}%U\t%S\t%M`, process.execPath, ...nodeArgs, entrypoint, vault, '--settings', settings, command, ...commandArgs], { + cwd: root, detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: { ...environment, ...extraEnvironment }, + }); + let stdout = ''; let stderr = ''; + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + const timeout = setTimeout(() => { try { process.kill(-child.pid, 'SIGTERM'); } catch (error) { if (error.code !== 'ESRCH') throw error; } }, 240000); + const code = await new Promise((accept, reject) => { child.once('error', reject); child.once('close', accept); }); + clearTimeout(timeout); + await fs.writeFile(logPath, `${stdout}\n${stderr}`); + if (code !== 0) throw new Error(`CLI ${command} exited ${code}; ${logPath}\n${stderr.slice(-2000)}`); + const line = stderr.split(/\r?\n/).find((row) => row.startsWith(marker)); + assert(line, 'GNU time output is missing'); + const [userSeconds, systemSeconds, maxResidentSetKiB] = line.slice(marker.length).split('\t').map(Number); + return { elapsedMs: performance.now() - started, userCpuMs: userSeconds * 1000, systemCpuMs: systemSeconds * 1000, maxResidentSetKiB }; +} + +async function startHost(vault, settings, logPath, metricsPath) { + const child = spawn(process.execPath, ['--require', preload, sourceCli, vault, '--settings', settings, 'p2p-host'], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: { ...environment, LIVESYNC_BENCH_DATACHANNEL_JSON: metricsPath } }); + let combined = ''; let ended = false; let failure; + const closed = new Promise((accept) => { child.once('error', (error) => { failure = error; ended = true; accept(); }); child.once('close', () => { ended = true; accept(); }); }); + child.stdout.on('data', (data) => { combined += data; }); child.stderr.on('data', (data) => { combined += data; }); + const stop = async () => { + if (!ended) child.kill('SIGTERM'); + const timeout = setTimeout(() => { if (!ended) child.kill('SIGKILL'); }, 10000); + await closed; clearTimeout(timeout); await fs.writeFile(logPath, combined); + }; + const started = performance.now(); + try { + while (!combined.includes('P2P host is running')) { + if (ended || performance.now() - started > 25000) throw failure || new Error(`P2P host did not start: ${combined.slice(-2500)}`); + await delay(100); + } + } catch (error) { await stop(); throw error; } + return { stop, readyMs: performance.now() - started }; +} + +async function settingsFor(size, room, name) { + return { + ...createNewVaultSettings(), isConfigured: true, remoteType: REMOTE_P2P, + encrypt: true, passphrase: 'chunk-size-benchmark-only-passphrase', usePathObfuscation: true, encryptInternalMetadata: true, + idDerivationVersion: 1, idDerivationKey: '', encryptedIdDerivationKey: await encryptString('0123456789abcdef'.repeat(4), '*' + SALT_OF_PASSPHRASE), + E2EEAlgorithm: 'v2', hashAlg: 'xxhash64', chunkSplitterVersion: 'v3-rabin-karp', minimumChunkSize: 20, customChunkSize: size, + enableCompression: false, useEden: false, useIndexedDBAdapter: false, handleFilenameCaseSensitive: false, pathNormalizationVersion: 0, + autoAcceptCompatibleTweak: false, readChunksOnline: false, liveSync: false, syncOnStart: false, syncOnSave: false, periodicReplication: false, + syncOnFileOpen: false, syncMaxSizeInMB: 50, + P2P_Enabled: true, P2P_AutoStart: false, P2P_AutoBroadcast: false, P2P_IsHeadless: true, + P2P_AppID: 'livesync-chunk-size-benchmark', P2P_roomID: room, P2P_passphrase: 'chunk-size-local-relay-fixture', + P2P_relays: 'ws://127.0.0.1:14094/', P2P_AutoAcceptingPeers: '~.*', P2P_AutoDenyingPeers: '', + P2P_turnServers: '', P2P_turnUsername: '', P2P_turnCredential: '', P2P_DevicePeerName: name, + }; +} + +async function copyStage(stage, vault, previous) { + const old = new Map((previous?.files || []).map((file) => [file.path, file.sha256])); + const changed = []; + for (const file of stage.files) { + if (old.get(file.path) === file.sha256) continue; + const from = join(stage.directory, file.path); const to = join(vault, file.path); + await fs.mkdir(dirname(to), { recursive: true }); await fs.copyFile(from, to); + const stat = await fs.stat(from); await fs.utimes(to, stat.atime, stat.mtime); changed.push(file.path); + } + return changed; +} + +await fs.mkdir(join(output, 'p2p'), { recursive: true }); +const manifest = JSON.parse(await fs.readFile(corpusPath, 'utf8')); +for (const stage of manifest.stages) stage.directory = resolve(dirname(corpusPath), stage.directory); +const corpusDigest = sha256(JSON.stringify(manifest.stages.map((stage) => ({ name: stage.name, files: stage.files })))); +const resultPath = join(output, 'p2p/results.json'); +let results = []; +try { const saved = JSON.parse(await fs.readFile(resultPath, 'utf8')); assert.equal(saved.corpusDigest, corpusDigest); assert.equal(saved.sourceCliSha256 ?? saved.cliSha256, sourceCliSha256); assert.equal(saved.targetCliSha256 ?? saved.cliSha256, targetCliSha256); results = saved.results; } catch (error) { if (error.code !== 'ENOENT') throw error; } +const save = () => fs.writeFile(resultPath, JSON.stringify({ version: '1.0.39', commonlib: '0.1.44', variant: process.env.LIVESYNC_BENCH_VARIANT || 'unlabelled', cliSha256, sourceCliSha256, targetCliSha256, corpusDigest, + scope: 'Local WebRTC direct path; timed target command includes process start-up, peer discovery, connection establishment, pull and reverse synchronisation. DataChannel payload bytes are measured using werift counters observed through a preload on both endpoints; they exclude transport overhead, transport retransmissions, and relay signalling. ICE byte counters are retained as diagnostics and report zero in this runtime. GNU time covers the target process; the preload also records CPU/RSS on each endpoint. Source mirroring and host start-up are recorded separately. E2EE V2 settings with independent IDs; compression disabled.', results }, null, 2)); +for (let repeat = 0; repeat < repeats; repeat++) { + const rotation = repeat % sizes.length; const order = [...sizes.slice(rotation), ...sizes.slice(0, rotation)]; + for (const size of order) { + const name = `p2p-c${size}-r${repeat + 1}`; + if (results.some((result) => result.name === name && result.complete)) continue; + const work = join(output, 'p2p', name); await fs.mkdir(join(work, 'logs'), { recursive: true }); + const a = join(work, 'source'); const b = join(work, 'target'); + await fs.rm(a, { recursive: true, force: true }); await fs.rm(b, { recursive: true, force: true }); + await fs.mkdir(a, { recursive: true }); await fs.mkdir(b, { recursive: true }); + const room = `chunk-size-${Date.now().toString(36)}-${name}`; + const hostName = `host-${name}`; + const settingsA = join(work, 'source-settings.json'); const settingsB = join(work, 'target-settings.json'); + await fs.writeFile(settingsA, JSON.stringify(await settingsFor(size, room, hostName))); + await fs.writeFile(settingsB, JSON.stringify(await settingsFor(size, room, `target-${name}`))); + const result = { name, customChunkSize: size, repeat: repeat + 1, stages: [], complete: false }; + results.push(result); await save(); + let previous; + try { + for (const stage of manifest.stages) { + const log = (label) => join(work, 'logs', `${stage.name}-${label}.txt`); + const changedFiles = await copyStage(stage, a, previous); + const mirror = await runCli(a, settingsA, 'mirror', [], log('mirror'), {}, sourceCli); + const hostMetricsPath = join(work, `${stage.name}-host-datachannel.json`); + const targetMetricsPath = join(work, `${stage.name}-target-datachannel.json`); + const host = await startHost(a, settingsA, log('host'), hostMetricsPath); + const statsPath = join(work, `${stage.name}-stats.jsonl`); await fs.rm(statsPath, { force: true }); + let sync; + try { sync = await runCli(b, settingsB, 'p2p-sync', [hostName, '120'], log('sync'), { LIVESYNC_P2P_STATS_JSONL: statsPath, LIVESYNC_BENCH_DATACHANNEL_JSON: targetMetricsPath }, targetCli); } + finally { await host.stop(); } + const rows = (await fs.readFile(statsPath, 'utf8')).trim().split(/\r?\n/); + const stats = JSON.parse(rows.at(-1)); + assert(stats.candidatePathCollected, 'ICE statistics were not collected'); + assert(!stats.selectedPath.includes('relay'), `Unexpected TURN path: ${stats.selectedPath}`); + const dataChannel = { host: JSON.parse(await fs.readFile(hostMetricsPath, 'utf8')), target: JSON.parse(await fs.readFile(targetMetricsPath, 'utf8')) }; + assert(dataChannel.host.payloadBytesSent > 0 && dataChannel.target.payloadBytesSent > 0, 'DataChannel counters must record both directions'); + const materialisedByPull = []; + for (const file of stage.files) { + let bytes; try { bytes = await fs.readFile(join(b, file.path)); } catch (error) { if (error.code !== 'ENOENT') throw error; } + if (!bytes || sha256(bytes) !== file.sha256) { + const to = join(work, 'verification', file.path); await fs.mkdir(dirname(to), { recursive: true }); + const measurement = await runCli(b, settingsB, 'pull', [file.path, to], log(`verify-${file.path.replace(/[^a-zA-Z0-9]/g, '-')}`), {}, targetCli); + bytes = await fs.readFile(to); materialisedByPull.push({ path: file.path, measurement }); + } + assert.equal(sha256(bytes), file.sha256, `${name} ${stage.name} ${file.path}`); + } + result.stages.push({ name: stage.name, changedFiles, mirror, hostReadyMs: host.readyMs, sync, stats, dataChannel, verifiedFiles: stage.files.length, materialisedByPull }); + await save(); console.log(JSON.stringify({ case: name, stage: stage.name, syncMs: Math.round(sync.elapsedMs), selectedPath: stats.selectedPath, hostPayloadBytesSent: dataChannel.host.payloadBytesSent, targetPayloadBytesSent: dataChannel.target.payloadBytesSent })); + previous = stage; + } + result.complete = true; await save(); + } catch (error) { result.error = String(error); await save(); throw error; } + } +} +console.log(`Completed P2P measurements: ${resultPath}`); diff --git a/utils/chunk-size-benchmark/rabin-window.test.mjs b/utils/chunk-size-benchmark/rabin-window.test.mjs new file mode 100644 index 00000000..78cf170a --- /dev/null +++ b/utils/chunk-size-benchmark/rabin-window.test.mjs @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import test from 'node:test'; + +const packageRoot = dirname(fileURLToPath(import.meta.resolve('@vrtmrz/livesync-commonlib/package.json'))); +const { splitPiecesRabinKarp } = await import(pathToFileURL(join(packageRoot, 'dist/string_and_binary/chunks.js')).href); + +test('zero windows lose the contribution of a marker in the preceding chunk', async () => { + const blockBytes = 300 * 1024; + const input = new Uint8Array(blockBytes * 3); + for (let end = blockBytes; end <= input.length; end += blockBytes) input[end - 1] = 1; + const pieces = await splitPiecesRabinKarp(new Blob([input], { type: 'application/octet-stream' }), 1100 * 1024, false, 20, 'markers.bin', false); + const chunks = []; + for await (const encoded of pieces()) chunks.push(Buffer.from(encoded, 'base64')); + assert.deepEqual(Buffer.concat(chunks), Buffer.from(input), 'all input bytes must survive splitting'); + // A 48-byte zero window hashes to zero; each isolated marker hashes to one. + // Markers are further apart than the 256 KiB minimum and below the maximum. + assert.deepEqual(chunks.map((chunk) => chunk.length), [blockBytes, blockBytes, blockBytes]); +}); diff --git a/utils/chunk-size-benchmark/remote.mjs b/utils/chunk-size-benchmark/remote.mjs new file mode 100644 index 00000000..5dc7683e --- /dev/null +++ b/utils/chunk-size-benchmark/remote.mjs @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import http from 'node:http'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import { createNewVaultSettings, REMOTE_COUCHDB, REMOTE_MINIO, SALT_OF_PASSPHRASE } from '@vrtmrz/livesync-commonlib/compat/common/types'; +import { encryptString } from '@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption'; +import { S3Client, CreateBucketCommand, ListObjectsV2Command, DeleteObjectsCommand, DeleteBucketCommand } from '@aws-sdk/client-s3'; + +const here = dirname(fileURLToPath(import.meta.url)); +const root = resolve(here, '../..'); +const cli = join(root, 'src/apps/cli/dist/index.cjs'); +const args = process.argv.slice(2); +const option = (key, fallback) => { const at = args.indexOf(`--${key}`); return at < 0 ? fallback : args[at + 1]; }; +const sourceCli = resolve(option('source-cli', cli)); +const targetCli = resolve(option('target-cli', cli)); +const output = resolve(option('output', join(root, '.bench/chunk-size'))); +const corpusPath = resolve(option('corpus', join(output, 'corpus/manifest.json'))); +const sizes = option('sizes', '0,4,10,20,40').split(',').map(Number); +const providers = option('providers', 'couchdb,s3').split(','); +const repeats = Number(option('repeats', '3')); +const rttMs = Number(option('rtt-ms', '0')); +const user = 'bench'; +const password = 'benchmark-only-password'; +const accessKeyId = 'bench-access-key'; +const secretAccessKey = 'benchmark-only-secret-key'; +const couchUri = 'http://127.0.0.1:15994'; +const s3Uri = 'http://127.0.0.1:19094'; +const auth = `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; +const key = '0123456789abcdef'.repeat(4); +const sha256 = (value) => createHash('sha256').update(value).digest('hex'); +const sourceCliSha256 = sha256(await fs.readFile(sourceCli)); +const targetCliSha256 = sha256(await fs.readFile(targetCli)); +const credentials = { accessKeyId, secretAccessKey }; +const s3 = new S3Client({ endpoint: s3Uri, region: 'us-east-1', forcePathStyle: true, credentials }); + +function emptyCounters() { return { requests: 0, requestBodyBytes: 0, responseBodyBytes: 0, maxRequestBodyBytes: 0, byMethod: {}, byOperation: {} }; } + +async function startProxy(backend) { + let counters = emptyCounters(); + const server = http.createServer(async (request, response) => { + counters.requests++; + counters.byMethod[request.method] = (counters.byMethod[request.method] || 0) + 1; + const url = new URL(request.url, backend); + const operation = url.pathname.split('/').find((part) => part.startsWith('_')) || (url.searchParams.has('list-type') ? 'list' : 'object'); + counters.byOperation[operation] = (counters.byOperation[operation] || 0) + 1; + if (rttMs) await delay(rttMs / 2); + const target = new URL(request.url, backend); + const upstream = http.request(target, { method: request.method, headers: request.headers }, async (incoming) => { + if (rttMs) await delay(rttMs / 2); + response.writeHead(incoming.statusCode, incoming.headers); + incoming.on('data', (chunk) => { counters.responseBodyBytes += chunk.length; }); + incoming.pipe(response); + }); + upstream.on('error', (error) => { response.writeHead(502); response.end(String(error)); }); + let requestBytes = 0; + request.on('data', (chunk) => { counters.requestBodyBytes += chunk.length; requestBytes += chunk.length; }); + request.on('end', () => { counters.maxRequestBodyBytes = Math.max(counters.maxRequestBodyBytes, requestBytes); }); + request.pipe(upstream); + }); + await new Promise((accept, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', accept); }); + return { uri: `http://127.0.0.1:${server.address().port}`, reset: () => { counters = emptyCounters(); }, snapshot: () => structuredClone(counters), close: () => new Promise((accept) => server.close(accept)) }; +} + +async function couch(path, method = 'GET', body) { + const response = await fetch(`${couchUri}${path}`, { method, headers: { Authorization: auth, 'Content-Type': 'application/json' }, body: body === undefined ? undefined : JSON.stringify(body) }); + if (!response.ok) throw new Error(`CouchDB ${method} ${path}: ${response.status} ${await response.text()}`); + return response.json(); +} + +async function ready() { + for (let attempt = 0; attempt < 30; attempt++) { + try { await couch('/_up'); await s3.send(new ListObjectsV2Command({ Bucket: 'missing-benchmark-bucket' })).catch((error) => { if (error.$metadata?.httpStatusCode !== 404) throw error; }); return; } + catch (error) { if (attempt === 29) throw error; await delay(1000); } + } +} + +async function runCli(directory, settings, command, commandArgs, logPath, entrypoint = cli) { + const marker = '__BENCH_TIME__'; + const started = performance.now(); + const child = spawn('/usr/bin/time', ['-f', `${marker}%U\t%S\t%M`, process.execPath, entrypoint, directory, '--settings', settings, command, ...commandArgs], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=2048' } }); + let stdout = ''; let stderr = ''; + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + const timeout = setTimeout(() => child.kill('SIGTERM'), 180000); + const code = await new Promise((accept, reject) => { child.once('error', reject); child.once('exit', accept); }); + clearTimeout(timeout); + const elapsedMs = performance.now() - started; + await fs.writeFile(logPath, `${stdout}\n${stderr}`); + if (code !== 0) throw new Error(`CLI ${command} exited ${code}; ${logPath}\n${stderr.slice(-2200)}`); + const line = stderr.split(/\r?\n/).find((row) => row.startsWith(marker)); + assert(line, 'GNU time output is missing'); + const [userSeconds, systemSeconds, maxResidentSetKiB] = line.slice(marker.length).split('\t').map(Number); + return { elapsedMs, userCpuMs: userSeconds * 1000, systemCpuMs: systemSeconds * 1000, maxResidentSetKiB }; +} + +async function settingsFor(provider, endpoint, resource, size) { + return { + ...createNewVaultSettings(), isConfigured: true, remoteType: provider === 'couchdb' ? REMOTE_COUCHDB : REMOTE_MINIO, + couchDB_URI: endpoint, couchDB_DBNAME: resource, couchDB_USER: user, couchDB_PASSWORD: password, + endpoint, bucket: resource, accessKey: accessKeyId, secretKey: secretAccessKey, region: 'us-east-1', forcePathStyle: true, + encrypt: true, passphrase: 'chunk-size-benchmark-only-passphrase', usePathObfuscation: true, encryptInternalMetadata: true, + idDerivationVersion: 1, idDerivationKey: '', encryptedIdDerivationKey: await encryptString(key, '*' + SALT_OF_PASSPHRASE), E2EEAlgorithm: 'v2', hashAlg: 'xxhash64', + chunkSplitterVersion: 'v3-rabin-karp', minimumChunkSize: 20, customChunkSize: size, autoAcceptCompatibleTweak: false, + enableCompression: false, useEden: false, useIndexedDBAdapter: false, handleFilenameCaseSensitive: false, pathNormalizationVersion: 0, + readChunksOnline: false, liveSync: false, syncOnStart: false, syncOnSave: false, periodicReplication: false, + syncOnFileOpen: false, syncMaxSizeInMB: 50, P2P_Enabled: false, + }; +} + +async function copyStage(stage, vault, previous) { + const old = new Map((previous?.files || []).map((file) => [file.path, file.sha256])); + const changed = []; + for (const file of stage.files) { + if (old.get(file.path) === file.sha256) continue; + const from = join(stage.directory, file.path); const to = join(vault, file.path); + await fs.mkdir(dirname(to), { recursive: true }); await fs.copyFile(from, to); + const stat = await fs.stat(from); await fs.utimes(to, stat.atime, stat.mtime); changed.push(file.path); + } + return changed; +} + +async function inspect(provider, resource, previousIds) { + if (provider === 'couchdb') { + const info = await couch(`/${resource}`); + const all = await couch(`/${resource}/_all_docs?include_docs=true`); + const chunks = all.rows.filter(({ doc }) => doc?.type === 'leaf'); + const added = chunks.filter(({ id }) => !previousIds.has(id)); + const ids = chunks.map(({ id }) => id); + const jsonBytes = ({ doc }) => Buffer.byteLength(JSON.stringify(doc)); + return { ids, metrics: { documentCount: info.doc_count, chunkCount: chunks.length, newChunkCount: added.length, chunkJsonBytes: chunks.reduce((sum, row) => sum + jsonBytes(row), 0), newChunkJsonBytes: added.reduce((sum, row) => sum + jsonBytes(row), 0), largestChunkJsonBytes: Math.max(0, ...chunks.map(jsonBytes)), largestDocumentJsonBytes: Math.max(0, ...all.rows.map(jsonBytes)), storedChunkDataBytes: chunks.reduce((sum, { doc }) => sum + Buffer.byteLength(doc.data || ''), 0), sizes: info.sizes, idLengthMin: Math.min(...ids.map((id) => id.length)), idLengthMax: Math.max(...ids.map((id) => id.length)) } }; + } + const objects = []; let token; + do { const result = await s3.send(new ListObjectsV2Command({ Bucket: resource, ContinuationToken: token })); objects.push(...result.Contents || []); token = result.NextContinuationToken; } while (token); + return { ids: objects.map(({ Key }) => Key), metrics: { objectCount: objects.length, storedObjectBytes: objects.reduce((sum, { Size }) => sum + Size, 0), newObjectCount: objects.filter(({ Key }) => !previousIds.has(Key)).length } }; +} + +async function removeResource(provider, resource) { + if (provider === 'couchdb') { await couch(`/${resource}`, 'DELETE'); return; } + let token; + do { + const listing = await s3.send(new ListObjectsV2Command({ Bucket: resource, ContinuationToken: token })); + if (listing.Contents?.length) await s3.send(new DeleteObjectsCommand({ Bucket: resource, Delete: { Objects: listing.Contents.map(({ Key }) => ({ Key })) } })); + token = listing.NextContinuationToken; + } while (token); + await s3.send(new DeleteBucketCommand({ Bucket: resource })); +} + +await fs.mkdir(join(output, 'remote'), { recursive: true }); +const manifest = JSON.parse(await fs.readFile(corpusPath, 'utf8')); +for (const stage of manifest.stages) stage.directory = resolve(dirname(corpusPath), stage.directory); +const digest = sha256(JSON.stringify(manifest.stages.map((stage) => ({ name: stage.name, files: stage.files })))); +const resultPath = join(output, 'remote', `results-rtt${rttMs}.json`); +let results = []; +try { const saved = JSON.parse(await fs.readFile(resultPath, 'utf8')); assert.equal(saved.corpusDigest, digest); assert.equal(saved.sourceCliSha256, sourceCliSha256); assert.equal(saved.targetCliSha256, targetCliSha256); results = saved.results; } catch (error) { if (error.code !== 'ENOENT') throw error; } +const proxies = { couchdb: await startProxy(couchUri), s3: await startProxy(s3Uri) }; +const save = async () => fs.writeFile(resultPath, JSON.stringify({ version: '1.0.39', commonlib: '0.1.44', variant: process.env.LIVESYNC_BENCH_VARIANT || 'unlabelled', sourceCliSha256, targetCliSha256, corpusDigest: digest, rttMs, mode: 'local services; HTTP body bytes exclude headers; full chunk download; E2EE V2 with independent IDs; compression disabled', results }, null, 2)); +try { + await ready(); + for (let repeat = 0; repeat < repeats; repeat++) { + const rotation = repeat % sizes.length; const order = [...sizes.slice(rotation), ...sizes.slice(0, rotation)]; + for (const provider of (repeat % 2 ? [...providers].reverse() : providers)) for (const size of order) { + const name = `${provider}-c${size}-r${repeat + 1}`; + if (results.some((result) => result.name === name && result.complete)) continue; + const resource = `chunks-${Date.now().toString(36)}-${name}`; + const work = join(output, 'remote', name); await fs.mkdir(join(work, 'logs'), { recursive: true }); + const a = join(work, 'source'); const b = join(work, 'target'); + await fs.rm(a, { recursive: true, force: true }); await fs.rm(b, { recursive: true, force: true }); + await fs.mkdir(a, { recursive: true }); await fs.mkdir(b, { recursive: true }); + const settingsA = join(work, 'source-settings.json'); const settingsB = join(work, 'target-settings.json'); + const settings = await settingsFor(provider, proxies[provider].uri, resource, size); + await fs.writeFile(settingsA, JSON.stringify(settings)); await fs.writeFile(settingsB, JSON.stringify(settings)); + if (provider === 'couchdb') await couch(`/${resource}`, 'PUT'); else await s3.send(new CreateBucketCommand({ Bucket: resource })); + const result = { name, provider, customChunkSize: size, repeat: repeat + 1, stages: [], complete: false }; + results.push(result); await save(); + try { + let previous; let ids = new Set(); + for (const stage of manifest.stages) { + const changedFiles = await copyStage(stage, a, previous); + const log = (label) => join(work, 'logs', `${stage.name}-${label}.txt`); + const mirror = await runCli(a, settingsA, 'mirror', [], log('mirror'), sourceCli); + proxies[provider].reset(); + const upload = await runCli(a, settingsA, 'sync', [], log('upload'), sourceCli); upload.http = proxies[provider].snapshot(); + const remote = await inspect(provider, resource, ids); ids = new Set(remote.ids); + proxies[provider].reset(); + const download = await runCli(b, settingsB, 'sync', [], log('download'), targetCli); download.http = proxies[provider].snapshot(); + const verified = []; const materialisedByPull = []; + for (const file of stage.files) { + let bytes; try { bytes = await fs.readFile(join(b, file.path)); } catch (error) { if (error.code !== 'ENOENT') throw error; } + if (!bytes || sha256(bytes) !== file.sha256) { + const target = join(work, 'verification', file.path); await fs.mkdir(dirname(target), { recursive: true }); + const measurement = await runCli(b, settingsB, 'pull', [file.path, target], log(`verify-${file.path.replace(/[^a-zA-Z0-9]/g, '-')}`), targetCli); + bytes = await fs.readFile(target); materialisedByPull.push({ path: file.path, measurement }); + } + assert.equal(sha256(bytes), file.sha256, `${name} ${stage.name} ${file.path}`); verified.push(file.path); + } + result.stages.push({ name: stage.name, changedFiles, mirror, upload, download, remote: remote.metrics, verifiedFiles: verified.length, materialisedByPull }); + await save(); console.log(JSON.stringify({ case: name, stage: stage.name, uploadMs: Math.round(upload.elapsedMs), downloadMs: Math.round(download.elapsedMs), requestBytes: upload.http.requestBodyBytes, responseBytes: download.http.responseBodyBytes, remote: remote.metrics })); + previous = stage; + } + result.complete = true; await save(); + } catch (error) { result.error = String(error); await save(); throw error; } + finally { await removeResource(provider, resource); } + } + } +} finally { await Promise.all(Object.values(proxies).map((proxy) => proxy.close())); s3.destroy(); } +console.log(`Completed remote measurements: ${resultPath}`); diff --git a/utils/chunk-size-benchmark/report.mjs b/utils/chunk-size-benchmark/report.mjs new file mode 100644 index 00000000..3c0d5c87 --- /dev/null +++ b/utils/chunk-size-benchmark/report.mjs @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import { join, resolve } from 'node:path'; + +const directory = resolve(process.argv[2] || '.bench/rabin-window-fixed'); +const baselineDirectory = resolve(process.argv[3] || '.bench/chunk-size'); +const load = async (file, base = directory) => JSON.parse(await fs.readFile(join(base, file), 'utf8')); +const summary = await load('summary.json'); +const baseline = await load('summary.json', baselineDirectory); +const comparison = await load('comparison.json'); +const validation = await load('validation.json'); +const boundary = await load('cloudant-json-boundary/results.json'); +const consumer = await load('context/validated-consumer.json'); +const recovery = await load('context/cli-bundle-recovery.json'); +assert.equal(validation.completedRemoteCases, 30); +assert.equal(validation.completedP2PCases, 12); +assert.equal(summary.corpus.manifestSha256, baseline.corpus.manifestSha256); +const mib = bytes => (bytes / 1048576).toFixed(3); +const second = milliseconds => (milliseconds / 1000).toFixed(2); +const getRemote = (data, provider, size, stage) => data.remote.find(row => row.provider === provider && row.customChunkSize === size && row.stage === stage); +const getP2P = (data, size, stage) => data.p2p.find(row => row.customChunkSize === size && row.stage === stage); +const getLocal = (data, size, stage) => data.local.find(row => row.customChunkSize === size && row.stage === stage); +const decrease = (before, after) => ((1 - after / before) * 100).toFixed(1); +const lines = []; +const paragraph = text => lines.push(text, ''); +paragraph('Rabin–Karp窓処理修正後の実測（Self-hosted LiveSync 1.0.39 / Commonlib 0.1.44の未公開修正パッケージ）'); +paragraph('窓処理は、今回の挿入後のチャンク再利用を妨げていた原因の一つだった。最小限の修正をCommonlibへ加え、前回と完全に同じ入力・設定・候補を再測定した。今回の混在コーパスでは、通常の新規推奨値を4（上限500 KiB）へ揃える案が有力になった。Cloudant対応は接続先ごとの推奨値ではなく、将来の同期先も含むvault共通のデータ制約として扱う必要がある。ただし数値4だけで全設定のCloudant互換性を保証する結論ではない。初期値、推奨値、UI、および移行処理はまだ変更していない。'); +paragraph(`LiveSyncの基点は ${consumer.liveSyncBaseRevision}、Commonlibの基点は ${consumer.commonlibBaseRevision}。Commonlib所有リポジトリーの作業ツリーで修正し、単体・境界・リリースツール・パッケージ検証を通したtarballをLiveSyncへfile依存として入れた。公開版0.1.44を置換して公開したものではない。候補の実体は ${consumer.packageIntegrity}、インストール済みchunks.jsのSHA-256は ${consumer.installedChunksSha256}。`); +paragraph('修正はsrc/string_and_binary/chunks.tsの窓更新条件を pos >= start + windowSize から pos >= windowSize へ変える1行と、その目的を示すコメントである。境界でstartが進んでも、hashの48バイト窓を連続して更新する。形式、ID導出方法、設定キー、最小・平均・最大チャンク目標は変更していない。'); +paragraph('先に3件の振る舞いの回帰テストを追加し、未修正の公開0.1.44相当ソースで3件とも期待した境界assertionが失敗することを確認した。バイナリ300/300/300 KiBの目印列は従来300/256/256/88 KiBとなり、挿入後の同一末尾チャンクを再利用できなかった。テキストの目印列でも過去のチャンクの影響が残った。修正後は3件とも成功し、再構成した全バイトも一致した。Commonlib全123ファイル・2,416テスト、型、boundary、release tools、packed consumerが成功した。LiveSyncはnpm 10クリーンインストール、通常npm ci、check、単体1,297成功・1 skipped、およびObsidian bundle/CLI/WebApp/WebPeerビルドが成功した。Community lintの既存51 warningsは残っている。'); +paragraph(`入力は前回と同じ15ファイル、初期合計 ${mib(summary.corpus.stages[0].sourceBytes)} MiB。通常のプロジェクトMarkdown10本、約512 KiBと約5 MiBの生成Markdown、PNG2枚、および固定seedの8 MiB高エントロピーバイナリである。manifest SHA-256は ${summary.corpus.manifestSha256}。段階は初回、4 KiBのバイナリ上書き等、続いてバイナリ途中4 KiB挿入と生成Markdownへの段落挿入。設定はRabin–Karp、minimumChunkSize=20、E2EE V2、独立ID鍵version 1、パス・内部Metadata暗号化、チャンク圧縮なし、Edenなし、全chunk取得で固定した。`); +paragraph('ローカル分割は0,1,4,10,20,40,60を各3回。HTTPはCouchDB 3.5.0とRustFSを0,4,10,20,40各3回、P2Pは0,4,10,40各3回。同じホスト上の生成テストvaultとタスク専用コンテナを使用し、WAN遅延やパケットロスは加えていない。CPU負荷をかける検証・測定は同時に実行していない。Node v24.18.0 / Linux arm64 / Neoverse-N1 / 論理CPU3である。'); +paragraph('以下はHTTPの実送信本文の3回中央値。ヘッダー・TLSは含めない。同期秒数はCLI起動を含み、事前のmirrorと転送後のファイル復元・SHA照合を含まない。OSキャッシュやサービスは各回に再起動していない。S3のJournalパック圧縮はenableCompressionによるチャンク圧縮とは別である。'); +lines.push('remote | value | 初期送信MiB | 上書き送信MiB | 挿入送信MiB | 初期送信秒 | 初期受信秒 | 初期HTTP要求数'); +for (const provider of ['couchdb', 's3']) for (const size of [0, 4, 10, 20, 40]) { + const initial = getRemote(summary, provider, size, 'initial'); + const overwrite = getRemote(summary, provider, size, 'overwrite'); + const insert = getRemote(summary, provider, size, 'insert'); + lines.push([provider, size, mib(initial.upload_requestBodyBytes.median), mib(overwrite.upload_requestBodyBytes.median), mib(insert.upload_requestBodyBytes.median), second(initial.upload_elapsedMs.median), second(initial.download_elapsedMs.median), initial.upload_requests.median].join(' | ')); +} +lines.push(''); +paragraph('P2Pは全回host<->host経路。werift 0.24.4のcandidate-pair bytesカウンターは0のため採用せず、RTCDataChannelの実送信payloadを両端で観測した。下表は送信元host側で、RPC制御とアプリ層再送を含み、SCTP/DTLS/UDP/IPのヘッダー、下位層再送、Nostr signallingは含まない。HTTP本文量と完全に同じ層ではない。同期時間は探索・接続・双方向同期を含む。'); +lines.push('value | 初期送信MiB | 上書き送信MiB | 挿入送信MiB 中央値 [min..max] | 初期同期秒 | 上書き同期秒 | 挿入同期秒'); +for (const size of [0, 4, 10, 40]) { + const initial = getP2P(summary, size, 'initial'); const overwrite = getP2P(summary, size, 'overwrite'); const insert = getP2P(summary, size, 'insert'); + lines.push([size, mib(initial.host_payloadBytesSent.median), mib(overwrite.host_payloadBytesSent.median), `${mib(insert.host_payloadBytesSent.median)} [${mib(insert.host_payloadBytesSent.min)}..${mib(insert.host_payloadBytesSent.max)}]`, second(initial.elapsedMs.median), second(overwrite.elapsedMs.median), second(insert.elapsedMs.median)].join(' | ')); +} +lines.push(''); +paragraph('値4の途中挿入における修正前→修正後の比較は次のとおり。'); +lines.push('対象 | 修正前MiB | 修正後MiB | 減少率'); +const oldRaw = getLocal(baseline, 4, 'insert').binaryNewRawBytes; const newRaw = getLocal(summary, 4, 'insert').binaryNewRawBytes; +lines.push(`新しく必要なバイナリの生データ | ${mib(oldRaw)} | ${mib(newRaw)} | ${decrease(oldRaw, newRaw)}%`); +for (const provider of ['couchdb', 's3']) { + const oldValue = getRemote(baseline, provider, 4, 'insert').upload_requestBodyBytes.median; const newValue = getRemote(summary, provider, 4, 'insert').upload_requestBodyBytes.median; + lines.push(`${provider}送信本文 | ${mib(oldValue)} | ${mib(newValue)} | ${decrease(oldValue, newValue)}%`); +} +const oldPeer = getP2P(baseline, 4, 'insert').host_payloadBytesSent.median; const newPeer = getP2P(summary, 4, 'insert').host_payloadBytesSent.median; +lines.push(`P2P host payload | ${mib(oldPeer)} | ${mib(newPeer)} | ${decrease(oldPeer, newPeer)}%`, ''); +paragraph('通常の4 MiB未満のMarkdown11本×3段階は、修正後も7候補の全てで分割境界とIDが一致した。customChunkSizeを上げてもこの領域のID数は減らない。新しい独立ID鍵方式のIDは67文字で、旧暗号化xxhash64比較は14〜16文字、中央値16文字だった。ID命名の変更がIDを短くしたという前提は置けない。'); +paragraph('0/1ではバイナリ・大きいテキストの最小値が最大値へクランプされ、100/200 KiBの固定分割になるため、窓修正後も挿入時の再同期の利点は出ない。小さい局所上書きだけなら0が有利だが、今回の挿入を含む編集では4の方が大幅に少なく済む。40と60は実装上の4 MiB上限に達し、全15ファイル×3段階で分割境界とIDが完全一致した。60の重複するネットワーク測定は省いた。'); +for (const provider of ['couchdb', 's3']) { + const four = getRemote(summary, provider, 4, 'initial').upload_requestBodyBytes.median; + const forty = getRemote(summary, provider, 40, 'initial').upload_requestBodyBytes.median; + paragraph(`${provider}で4から40へ増やした初回送信量の削減は ${((1 - forty / four) * 100).toFixed(3)}%程度だった。上書き・挿入では40の送信量が大きく、今回の条件から大きいチャンクを維持する性能上の根拠は得られなかった。`); +} +paragraph('S3ではどの候補も初回7オブジェクト・14 HTTP要求だった。4と40は編集時もそれぞれ1オブジェクト追加・6 HTTP要求で、chunk数の削減がそのままPUT削減になる構成ではない。AWS等の実サービスの課金やWAN性能を測った値ではない。P2Pでは回ごとの再送等の変動があるため、中央値だけでなくsummary.jsonのmin/maxも参照する。150msの断片再要求との因果は計測しておらず、この処理は変更していない。'); +paragraph('Cloudantの現行IBM公式limits（2026-10-23確認）は、単一JSON文書1 MB、リクエスト全体64MB、添付10 MB。https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-limits 。chunksは添付ではなくJSON文書として送る。モデルには保守的に1,000,000 / 64,000,000 bytesを使った。通常の暗号化条件ではCouchDBに文書上限だけをかけ、4は15ファイルの復元まで成功、10は413となった。4の最大保存文書は910,461 bytes、通常HTTP測定の最大リクエストは約10.27 MBで、両方の上限内だった。'); +paragraph('別の境界probeでは、実際の分割器・圧縮・暗号化関数を用い、レプリケーションのrevisionフィールドを含む合成leafを送信した。結果は次のとおり。'); +lines.push('入力 | value | 暗号化 | 圧縮 | JSON文書bytes | CouchDB HTTP status'); +for (const row of boundary.cases) lines.push([row.kind, row.customChunkSize, row.encryption, row.compression, row.jsonBytes, row.status].join(' | ')); +lines.push(''); +paragraph('値4のバイナリはraw 512,000 bytes → base64 682,668 bytes → E2EE V1/V2で約910.5 KBとなった。値5のE2EE V2は1,092,573 bytesとなり、文書上限で413が返った。したがって暗号化バイナリに対して4は整数の候補として上限に近く、余裕を残す候補である。'); +paragraph('ただし、CouchDB 3.5.0のmax_document_sizeはparsed document bodyをcouch_ejson_size:encoded_sizeで概算し、文字列はUTF-8長に2を足すだけでJSON escapeを数えない。このためNULの未暗号化テキストはwire上3,072,213 bytesでも、モデルの1 MB文書制限では受理された。Cloudant内部の算定が完全に同一であるとは確認していない。単純にwire JSONが1 MB超なら必ずCloudantの文書制限で失敗する、とこの結果から断定してはいけない。公式CouchDBソースはcontext/couch_doc-3.5.0.erlとcontext/couch_ejson_size-3.5.0.erlに保存した。'); +paragraph('一括リクエストは別の制限である。分割器が作った500 KiBの異なる制御文字テキスト25個をまとめると76,805,358 bytesとなり、64,000,000-byte request制限で413になった。20個なら61,444,288 bytesで受理された。これは合成bulk requestの境界probeであり、アプリケーションの通常同期全体で同じbulkが組まれると実証したテストではない。現在の通常Couch同期のbatch_size=25では、暗号化文書が1 MB以内なら25 MB程度だが、暗号化しないテキストのescape膨張には件数制限だけでは十分でない。optional bulk chunk sendには独自のbyte制限もあるので、その経路と通常Pouch replicationの経路を区別する必要がある。'); +paragraph('CloudantモデルはローカルCouchDBで制限を模したものであり、IBM Cloudantサービス自体の同期・性能は測定していない。全probe後に両制限設定を復元し、合成文書のDBも削除した。Cloudantの互換性を約束するなら、leafだけでなくMetadataを含む最終文書とバッチの実サイズも方針の対象にするのが妥当である。'); +paragraph(`旧版と修正版の混在は、同じ旧依存リンクからビルドを復元し、旧entryが前回の保存物と完全一致することを確認して実測した（${recovery.originalEntryBitIdenticalToBaselineArchive}）。旧→新、新→旧の各3段階×15ファイル、計${validation.mixedVersionVerifiedFileStages}件でSHA-256が一致した。チャンクの保存形式とID方式は引き続き読める。再利用効率を揃えるには書込み端末の分割実装も揃える方がよい。最初の互換性試行は旧bundleの付随CJS不足で実行前に失敗し、そのログと復元記録を保存した。復元後の全試行は成功した。`); +const cutover = comparison.cutover.find(row => row.customChunkSize === 40); +paragraph(`窓の修正で分割境界が変わるため、既存ファイルを再保存する初回には追加チャンクが多く必要になり得る。ローカルの同じ初期データを値40で比較すると、新方式のunique raw ${mib(cutover.totalNewAlgorithmUniqueRawBytes)} MiB中、旧初期chunkに存在しない部分は ${mib(cutover.bytesNotPresentInOldAlgorithmInitialChunks)} MiBだった。これは内容/ID照合による算定で、実際のアップグレード時の転送量を測定した値ではない。以後の同じ方式での編集に対する改善とは分けて扱う。`); +paragraph('設定整理への提案は、通常の新規作成・推奨値をremote種別によらず4へ揃える方向で進め、旧設定の省略値や既存remoteで合意済みのtweakを一律に書き換えないことである。60と10を標準候補として分ける根拠は今回弱い。一方、Cloudantへ将来同期する意図を問うなら、現在の接続先ではなくvault全体の互換方針として扱い、全書込み端末で守る必要がある。暗号化・JSON化・バッチ化後の上限を内部で導出し、ユーザーへ組合せの数字を選ばせることは避けたい。'); +paragraph('現在のremote profile URIは接続欄だけを含み、customChunkSizeは共通設定である。通常のprofile切替でその値を変えないが、起動時のCloudant接続先補正やpreferred tweak採用では変わる経路がある。保存済みleafは転送時に再分割されない。Cloudant対応を後から有効化して値だけ下げても、既存の大きいleafがそのまま残るため、互換方針を変える移行では既存DBの検査・再構成も別途設計する必要がある。これは今回未実装の設計案である。'); +paragraph(`検証結果は、主測定42ケース・126段階×15ファイル=${validation.verifiedFileStages}件のSHA-256一致、保存後設定${validation.checkedPersistedConfigurations}件の条件一致、混在版${validation.mixedVersionVerifiedFileStages}件のSHA-256一致。通常Markdown、40/60同一、全分割の再構成、反復でのID・境界再現性、および同じIDの内容一致も成功した。`); +paragraph('保存物は生JSON、summary/CSV、comparison.json、validation.json、全入力corpus、各caseのログ・設定・P2Pカウンター、scripts、contextの完全な旧/新CLI bundleとlockfile、commonlibの修正差分・回帰テスト・検証ログ・tarball、およびartifacts.sha256である。巨大な作業用DB複製は含めず、再生成できる入力と証拠を保存する。旧baselineは親ディレクトリーにそのまま残している。'); +paragraph('再現時はLiveSync固定コミットとCommonlib固定コミットの作業ツリーを用意し、Commonlibのproduction.patchとsource内の回帰テストを反映する。Commonlibの元リポジトリーで検証し、保存済みtarballと同じ実体をLiveSyncにfile依存として入れる。scriptsをLiveSyncのutils/chunk-size-benchmarkへ置き、npm 10 ci、通常npm ci、check、unit、全app buildの順に実行する。旧版の再構築には公開0.1.44パッケージを使い、実行本体だけでなくdist全体を別ディレクトリーへ残す。新しい出力先とLIVESYNC_BENCH_VARIANT=rolling-window-fixedを使ってlocal/remote/cloudant probes/compatibility/p2pを逐次実行し、summarise、compare、verify-results、reportの順に集計する。ローカル専用fixtureの起動・停止にもcomposeのproject名を指定する。'); +await fs.writeFile(join(directory, 'report.txt'), lines.join('\n')); +console.log(JSON.stringify({ report: join(directory, 'report.txt'), bytes: Buffer.byteLength(lines.join('\n')) })); diff --git a/utils/chunk-size-benchmark/summarise.mjs b/utils/chunk-size-benchmark/summarise.mjs new file mode 100644 index 00000000..4ece8484 --- /dev/null +++ b/utils/chunk-size-benchmark/summarise.mjs @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import { join, resolve } from 'node:path'; + +const directory = resolve(process.argv[2] || '.bench/chunk-size'); +const local = JSON.parse(await fs.readFile(join(directory, 'results.json'), 'utf8')); +const optionalJson = async (path) => { try { return JSON.parse(await fs.readFile(path, 'utf8')); } catch (error) { if (error.code === 'ENOENT') return undefined; throw error; } }; +const remote = await optionalJson(join(directory, 'remote/results-rtt0.json')); +const p2p = await optionalJson(join(directory, 'p2p/results.json')); +const median = (numbers) => { const sorted = [...numbers].sort((a, b) => a - b); const mid = Math.floor(sorted.length / 2); return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; }; +const stats = (numbers) => ({ n: numbers.length, median: median(numbers), min: Math.min(...numbers), max: Math.max(...numbers) }); + +let normalMarkdownFileStages = 0; +for (const stage of local.candidates[0].stages) { + for (const file of stage.files.filter((file) => file.kind === 'markdown' && file.sourceBytes < 4 * 1024 * 1024)) { + const fingerprints = local.candidates.map((candidate) => candidate.stages.find((entry) => entry.name === stage.name).files.find((entry) => entry.path === file.path).chunkFingerprintSha256); + assert.equal(new Set(fingerprints).size, 1, `${stage.name}/${file.path}`); normalMarkdownFileStages++; + } +} + +const localRows = local.candidates.flatMap((candidate) => candidate.stages.map((stage) => { + const binary = stage.byKind.find((group) => group.kind === 'binary'); + const markdown = stage.byKind.find((group) => group.kind === 'markdown'); + const random = stage.files.find((file) => file.path === 'benchmark/random-8m.bin'); + return { + customChunkSize: candidate.customChunkSize, stage: stage.name, rawCeilingBytes: candidate.chunkSizeCeilingBytes, + allChunks: stage.files.reduce((sum, file) => sum + file.chunkCount, 0), binaryChunks: binary.chunkCount, + randomBinaryChunks: random.chunkCount, randomBinaryCpuMs: random.timing.primarySplitAndIdCpuMsMedian, + binaryNewRawBytes: binary.newUniqueChunkRawBytesVsHistory, markdownNewRawBytes: markdown.newUniqueChunkRawBytesVsHistory, + splitAndIdCpuMs: binary.timing.primarySplitAndIdCpuMsMedian + markdown.timing.primarySplitAndIdCpuMsMedian, + }; +})); + +function grouped(entries, makeRow) { + const groups = new Map(); + for (const entry of entries) for (const stage of entry.stages) { + const key = `${entry.provider || 'p2p'}/${entry.customChunkSize}/${stage.name}`; + const group = groups.get(key) || { provider: entry.provider || 'p2p', customChunkSize: entry.customChunkSize, stage: stage.name, samples: [] }; + group.samples.push(stage); groups.set(key, group); + } + return [...groups.values()].map(makeRow); +} + +const remoteRows = grouped((remote?.results || []).filter((entry) => entry.complete), (group) => { + const result = { provider: group.provider, customChunkSize: group.customChunkSize, stage: group.stage, repeats: group.samples.length }; + for (const direction of ['mirror', 'upload', 'download']) { + for (const metric of ['elapsedMs', 'userCpuMs', 'systemCpuMs', 'maxResidentSetKiB']) result[`${direction}_${metric}`] = stats(group.samples.map((stage) => stage[direction][metric])); + if (direction !== 'mirror') for (const metric of ['requests', 'requestBodyBytes', 'responseBodyBytes', 'maxRequestBodyBytes']) { + const values = group.samples.map((stage) => stage[direction].http[metric]).filter(Number.isFinite); + if (values.length) result[`${direction}_${metric}`] = stats(values); + } + } + for (const metric of ['objectCount', 'storedObjectBytes', 'chunkCount', 'newChunkCount', 'chunkJsonBytes', 'largestChunkJsonBytes', 'largestDocumentJsonBytes']) { + const values = group.samples.map((stage) => stage.remote[metric]).filter(Number.isFinite); + if (values.length) result[metric] = stats(values); + } + result.verifiedFilesPerSample = group.samples.map((stage) => stage.verifiedFiles); + return result; +}); + +const p2pRows = grouped((p2p?.results || []).filter((entry) => entry.complete), (group) => { + const result = { provider: 'p2p', customChunkSize: group.customChunkSize, stage: group.stage, repeats: group.samples.length }; + for (const metric of ['elapsedMs', 'userCpuMs', 'systemCpuMs', 'maxResidentSetKiB']) result[metric] = stats(group.samples.map((stage) => stage.sync[metric])); + result.currentRoundTripTime = stats(group.samples.map((stage) => stage.stats.selectedPair.currentRoundTripTime)); + for (const side of ['host', 'target']) for (const metric of ['payloadBytesSent', 'messagesSent']) result[`${side}_${metric}`] = stats(group.samples.map((stage) => stage.dataChannel[side][metric])); + for (const side of ['host', 'target']) for (const metric of ['userCpuMs', 'systemCpuMs', 'maxResidentSetKiB']) result[`${side}_${metric}`] = stats(group.samples.map((stage) => stage.dataChannel[side].processUsage[metric])); + result.paths = [...new Set(group.samples.map((stage) => stage.stats.selectedPath))]; + result.verifiedFilesPerSample = group.samples.map((stage) => stage.verifiedFiles); + return result; +}); + +const summary = { version: local.runtime.projectVersion, commonlib: local.runtime.commonlibVersion, variant: local.runtime.variant, commonlibChunksSha256: local.runtime.commonlibChunksSha256, corpus: local.corpus, + assertions: { ...local.assertions, normalMarkdownIdenticalAcrossAllSevenValues: true, normalMarkdownFileStages }, local: localRows, remote: remoteRows, p2p: p2pRows }; +await fs.writeFile(join(directory, 'summary.json'), JSON.stringify(summary, null, 2)); + +function csv(rows, fields) { + const cell = (value) => JSON.stringify(typeof value === 'object' && value !== null ? value.median : value ?? ''); + return `${fields.join(',')}\n${rows.map((row) => fields.map((field) => cell(row[field])).join(',')).join('\n')}\n`; +} +await fs.writeFile(join(directory, 'local-summary.csv'), csv(localRows, Object.keys(localRows[0]))); +if (remoteRows.length) await fs.writeFile(join(directory, 'remote-summary.csv'), csv(remoteRows, Object.keys(remoteRows[0]).filter((key) => key !== 'verifiedFilesPerSample'))); +if (p2pRows.length) await fs.writeFile(join(directory, 'p2p-summary.csv'), csv(p2pRows, Object.keys(p2pRows[0]).filter((key) => !['paths', 'verifiedFilesPerSample'].includes(key)))); +console.log(JSON.stringify({ directory, completeRemoteCases: remote?.results.filter((entry) => entry.complete).length || 0, completeP2PCases: p2p?.results.filter((entry) => entry.complete).length || 0, assertions: summary.assertions })); diff --git a/utils/chunk-size-benchmark/verify-results.mjs b/utils/chunk-size-benchmark/verify-results.mjs new file mode 100644 index 00000000..0c3bcd42 --- /dev/null +++ b/utils/chunk-size-benchmark/verify-results.mjs @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import { join, resolve } from 'node:path'; + +const directory = resolve(process.argv[2] || '.bench/rabin-window-fixed'); +const load = async name => JSON.parse(await fs.readFile(join(directory, name), 'utf8')); +const local = await load('results.json'); +const summary = await load('summary.json'); +const remote = await load('remote/results-rtt0.json'); +const p2p = await load('p2p/results.json'); +const manifest = await load('corpus/manifest.json'); +const sha256 = value => createHash('sha256').update(value).digest('hex'); +assert.equal(remote.corpusDigest, p2p.corpusDigest); +assert.equal(remote.corpusDigest, sha256(JSON.stringify(manifest.stages.map(stage => ({ name: stage.name, files: stage.files }))))); +assert.equal(local.corpus.manifestSha256, sha256(JSON.stringify(manifest))); +assert.equal(remote.results.length, 30); +assert.equal(p2p.results.length, 12); +let verified = 0; +let configurations = 0; +for (const [subdirectory, result] of [['remote', remote], ['p2p', p2p]]) { + assert(result.variant === 'rolling-window-fixed'); + const groups = new Map(); + for (const entry of result.results) { + assert(entry.complete && !entry.error, entry.name); + assert.equal(entry.stages.length, 3); + const key = `${entry.provider || 'p2p'}/${entry.customChunkSize}`; + groups.set(key, (groups.get(key) || 0) + 1); + for (const stage of entry.stages) { assert.equal(stage.verifiedFiles, 15); verified += stage.verifiedFiles; } + for (const side of ['source', 'target']) { + const settings = await load(join(subdirectory, entry.name, `${side}-settings.json`)); + assert.equal(settings.customChunkSize, entry.customChunkSize); + assert.equal(settings.chunkSplitterVersion, 'v3-rabin-karp'); + assert.equal(settings.minimumChunkSize, 20); + assert.equal(settings.idDerivationVersion, 1); + assert.equal(settings.encrypt, true); + assert.equal(settings.E2EEAlgorithm, 'v2'); + assert.equal(settings.useEden, false); + assert.equal(settings.enableCompression, false); + assert.equal(settings.autoAcceptCompatibleTweak, false); + assert.equal(settings.batch_size, 25); + configurations++; + } + } + assert([...groups.values()].every(count => count === 3)); +} +assert.equal(verified, 1890); +assert.equal(configurations, 84); +let mixedVerified = 0; +const compatibility = []; +for (const name of ['compatibility-old-to-new', 'compatibility-new-to-old']) { + const result = await load(`${name}/remote/results-rtt0.json`); + const completed = result.results.filter(entry => entry.complete); + assert.equal(completed.length, 1); + assert.notEqual(result.sourceCliSha256, result.targetCliSha256); + const entry = completed[0]; + assert(entry.complete && !entry.error); + assert.equal(entry.stages.length, 3); + for (const stage of entry.stages) { assert.equal(stage.verifiedFiles, 15); mixedVerified += stage.verifiedFiles; } + compatibility.push({ name, sourceCliSha256: result.sourceCliSha256, targetCliSha256: result.targetCliSha256, verifiedFiles: 45, retainedIncompleteSetupAttempts: result.results.filter(entry => !entry.complete).length }); +} +const model = await load('cloudant-model/constraint-result.json'); +assert(model.configurationRestored && model.sizeRejectionObserved); +assert(model.cases.some(entry => entry.customChunkSize === 4 && entry.complete)); +assert(model.cases.some(entry => entry.customChunkSize === 10 && !entry.complete)); +const boundary = await load('cloudant-json-boundary/results.json'); +assert(boundary.configurationRestored && boundary.requestConfigurationRestored); +assert.equal(boundary.cases.length, 8); +assert.equal(boundary.batches.length, 2); +assert(boundary.batches.some(entry => entry.count === 25 && entry.status === 413 && !entry.accepted)); +assert(boundary.batches.some(entry => entry.count === 20 && entry.accepted)); +assert(Object.values(local.assertions).every(value => value === true || typeof value === 'number')); +const result = { completedRemoteCases: 30, completedP2PCases: 12, repeatsPerCondition: 3, + verifiedFileStages: verified, checkedPersistedConfigurations: configurations, + sameCorpusDigestForRemoteAndP2P: true, localAssertions: summary.assertions, + compatibility, mixedVersionVerifiedFileStages: mixedVerified, + cloudantConstraintModel: { candidate4Completed: true, candidate10RejectedWith413: true, fixtureConfigurationRestored: true }, + cloudantJSONBoundary: { cases: boundary.cases.length, batches: boundary.batches.length, request25RejectedWith413: true, request20Accepted: true, fixtureConfigurationRestored: true, cloudantServiceMeasured: false }, + rabinWindowDiagnostic: { command: 'node --test utils/chunk-size-benchmark/rabin-window.test.mjs', exitCode: 0, productionFixApplied: true }, +}; +await fs.writeFile(join(directory, 'validation.json'), JSON.stringify(result, null, 2)); +console.log(JSON.stringify(result)); diff --git a/utils/flyio/generate_setupuri.test.ts b/utils/flyio/generate_setupuri.test.ts index 7c35f03e..bae7ab1b 100644 --- a/utils/flyio/generate_setupuri.test.ts +++ b/utils/flyio/generate_setupuri.test.ts @@ -1,6 +1,7 @@ import { decodeSettingsFromSetupURI, DEFAULT_SETTINGS, + PREFERRED_SETTING_SELF_HOSTED, } from "../setup/livesync-commonlib.ts"; function assert(condition: unknown, message: string): asserts condition { @@ -9,8 +10,14 @@ function assert(condition: unknown, message: string): asserts condition { Deno.test("generates a current self-hosted Setup URI through the published Commonlib contract", async () => { const scriptPath = new URL("./generate_setupuri.ts", import.meta.url); + const candidateConfig = Deno.env.get("LIVESYNC_TEST_COMMONLIB_CONFIG"); const command = new Deno.Command(Deno.execPath(), { - args: ["run", "-A", scriptPath.pathname], + args: [ + "run", + "-A", + ...(candidateConfig ? ["--config", candidateConfig, "--no-lock"] : []), + scriptPath.pathname, + ], env: { hostname: "https://couch.example.test", username: "alice", @@ -38,7 +45,9 @@ Deno.test("generates a current self-hosted Setup URI through the published Commo const recoveryCode = stdout.match(/sls-id-v1:[0-9a-f]{64}/u)?.[0]; assert(recoveryCode, "the generator did not print an ID recovery code"); assert( - (effectiveSettings as typeof effectiveSettings & { idDerivationKey?: string }).idDerivationKey === + (effectiveSettings as typeof effectiveSettings & { + idDerivationKey?: string; + }).idDerivationKey === recoveryCode.slice("sls-id-v1:".length), "the CouchDB Setup URI did not contain the generated ID key", ); @@ -47,8 +56,9 @@ Deno.test("generates a current self-hosted Setup URI through the published Commo "the CouchDB Setup URI left the imported device unconfigured", ); assert( - effectiveSettings.customChunkSize === 60, - "the Setup URI did not use the current self-hosted chunk-size recommendation", + effectiveSettings.customChunkSize === + PREFERRED_SETTING_SELF_HOSTED.customChunkSize, + "the Setup URI did not use the self-hosted chunk-size preference from Commonlib", ); assert( effectiveSettings.chunkSplitterVersion === "v3-rabin-karp", diff --git a/utils/setup/generate_setup_uri.test.ts b/utils/setup/generate_setup_uri.test.ts index 2bf802ff..99e11e30 100644 --- a/utils/setup/generate_setup_uri.test.ts +++ b/utils/setup/generate_setup_uri.test.ts @@ -1,6 +1,7 @@ import { decodeSettingsFromSetupURI, DEFAULT_SETTINGS, + PREFERRED_JOURNAL_SYNC, } from "./livesync-commonlib.ts"; import { generateSetupURI } from "./generate_setup_uri.ts"; @@ -55,8 +56,8 @@ Deno.test("generates an Object Storage Setup URI with a selected S3 profile", as "the Setup URI left the imported device unconfigured", ); assert( - effective.customChunkSize === 10, - "the journal chunk-size preset was not applied", + effective.customChunkSize === PREFERRED_JOURNAL_SYNC.customChunkSize, + "the Object Storage Setup URI did not use its Commonlib journal chunk-size preference", ); assert( effective.liveSync, @@ -122,7 +123,7 @@ Deno.test("generates a random-room P2P Setup URI without copying a device identi ); assert( effective.customChunkSize === 0, - "the P2P profile inherited the self-hosted CouchDB chunk-size recommendation", + "the P2P Setup URI did not retain its zero chunk-size value", ); assert( effective.sendChunksBulkMaxSize === 1, From 22ef4d073f1a30d49b5eb7034e48b02d42b16d81 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 09:39:18 +0000 Subject: [PATCH 07/28] Document module filename capitalisation --- devs.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devs.md b/devs.md index a4d3849d..4e820e90 100644 --- a/devs.md +++ b/devs.md @@ -204,6 +204,9 @@ The [multiple-device conflict test procedure](test/README.md#multiple-device-con ### File Structure Conventions +- **Class modules**: Start a new or renamed TypeScript or JavaScript filename with an upper-case letter when the file contains a class with the same name. +- **Function modules**: Start the filename with a lower-case letter when the module primarily exports functions, such as `initialFetchFastForward.ts`. Follow the surrounding filename style for the rest of the basename. Corresponding test files use the same basename. +- **Existing files**: Retain established names for unrelated files and separate conventions for components or other file types. - **Platform-specific code**: Use `.platform.ts` suffix (replaced with `.obsidian.ts` in production builds via esbuild) - **Development code**: Use `.dev.ts` suffix (replaced with `.prod.ts` in production) - **Path aliases**: `@/*` maps to `src/*`; Commonlib uses package exports rather than a source alias From 404d4279c0a0113e29c0272d5ae40b1b6485b57b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 10:13:46 +0000 Subject: [PATCH 08/28] Add Joined devices dialogue and version notices --- devs.md | 8 + docs/glossary.md | 4 + docs/settings.md | 10 + package.json | 1 + .../messages/LiveSyncProvisionalMessages.ts | 56 +++ src/common/replicatorProviders.ts | 8 + src/common/replicatorProviders.unit.spec.ts | 10 +- src/common/replicatorResources/index.ts | 5 + .../replicatorResources/joinedDevices.ts | 40 ++ src/main.ts | 2 + .../SettingDialogue/PaneRemoteConfig.ts | 17 + .../PaneRemoteConfig.unit.spec.ts | 27 +- .../dialogues/JoinedDevicesDialog.svelte | 391 +++++++++++++++ src/serviceFeatures/joinedDevices.ts | 64 +++ .../joinedDevices.unit.spec.ts | 69 +++ src/serviceFeatures/joinedDevicesCommand.ts | 5 + .../joinedDevicesDialogTypes.ts | 19 + src/serviceFeatures/useJoinedDevices.ts | 212 ++++++++ .../useJoinedDevices.unit.spec.ts | 174 +++++++ test/e2e-obsidian/README.md | 2 + test/e2e-obsidian/scripts/joined-devices.ts | 462 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + updates.md | 8 + 23 files changed, 1589 insertions(+), 6 deletions(-) create mode 100644 src/common/replicatorResources/joinedDevices.ts create mode 100644 src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte create mode 100644 src/serviceFeatures/joinedDevices.ts create mode 100644 src/serviceFeatures/joinedDevices.unit.spec.ts create mode 100644 src/serviceFeatures/joinedDevicesCommand.ts create mode 100644 src/serviceFeatures/joinedDevicesDialogTypes.ts create mode 100644 src/serviceFeatures/useJoinedDevices.ts create mode 100644 src/serviceFeatures/useJoinedDevices.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/joined-devices.ts diff --git a/devs.md b/devs.md index a4d3849d..5201c047 100644 --- a/devs.md +++ b/devs.md @@ -166,6 +166,14 @@ Legacy Modules remain grouped by directory: - `features/` contains optional features; and - `extras/` contains development and testing tools. +### UI commands and dialogue ownership + +Declare shared UI requests with `declareUICommand` from `@vrtmrz/livesync-commonlib/ui-commands`. The host's `services.UI.commands` registry owns one handler per declaration. A serviceFeature registers the handler, releases it on unload, and owns the associated dialogue lifetime. Callers receive a typed result or an explicit `unavailable` result; handler failures propagate. + +`OPEN_JOINED_DEVICES` is shared by Remote Configuration, version Notices, and the command palette. `useJoinedDevicesFeature` registers it once and gives the dialogue narrow refresh, forget, and cancellation callbacks. Each remote operation uses a settings snapshot and releases its finite resource in `finally`. Results from an old remote configuration or a closed dialogue are discarded. A write already sent to the captured remote can still complete after the dialogue closes. + +The registry belongs to one UIService instance. Reuse the same declaration object at each call site, and keep active UI cleanup in the feature and dialogue owner. `UIService` disposes the registry when the host's context emits the plug-in-unloaded event. + ### Key Architectural Components - **LiveSyncLocalDB** (`@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB`): Local PouchDB database wrapper diff --git a/docs/glossary.md b/docs/glossary.md index 6f3f1c3e..a498ce5e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -98,6 +98,10 @@ recovery guidance, or diagnostics intended for users. - **ID recovery code:** A versioned text form of the saved ID key which can be shown on the current device and imported without deriving a different key. Treat it as a secret. +- **Joined devices:** The dialogue showing device records held by the active + CouchDB or Object Storage remote, with the current device first. Records + describe participation, not current online activity. **Forget device** removes + an obsolete device record; it does not delete Vault files or revoke credentials. - **JWT Authentication:** An experimental CouchDB authentication option which uses a JSON Web Token instead of standard credentials. It requires a private key or secret, algorithm, expiry duration, subject, and key ID. diff --git a/docs/settings.md b/docs/settings.md index 0885589c..a64dbde6 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -234,6 +234,16 @@ Setting key: remoteType The active connection type. This is automatically projected to the legacy configuration when you activate a connection profile. +### Joined devices + +Open **Joined devices** to read the device records on the active CouchDB or Object Storage connection. The same dialogue is available from the command palette and version Notices. **Refresh** reads the remote again. This device appears first, including when its record is absent from the remote. Opening or refreshing this dialogue does not create a missing CouchDB database. + +The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. + +Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices are advisory and do not stop synchronisation. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. + +**Forget device** removes another device's participation records after confirmation. It preserves the remote lock state and every other device. It does not delete Vault files or revoke credentials. A device can register again on an unlocked remote; on a locked remote, the existing recovery process applies. The result is checked after saving to CouchDB; if it cannot be confirmed, refresh the list and try again. CouchDB stores these records together in a local document which does not reject simultaneous updates, so changes from other devices can be overwritten. Pausing synchronisation on the other devices avoids that overlap. Object Storage uses conditional updates; if the server cannot support them, the record is retained. + ### 2. Notification #### Notify when the estimated remote storage size exceeds on start up diff --git a/package.json b/package.json index a6ff13d4..7ea27ee8 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts", "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", + "test:e2e:obsidian:joined-devices": "tsx test/e2e-obsidian/scripts/joined-devices.ts", "test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts", "test:e2e:obsidian:database-diagnostics-couchdb": "tsx test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 34ae100b..2b809f40 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,62 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "Unknown platform": "Unknown platform", + "This removes only this device's record from the remote.": + "This removes only this device's record from the remote.", + "This remote is unlocked. A device that returns can register again.": + "This remote is unlocked. A device that returns can register again.", + "This remote is locked. Use the existing recovery process for devices that need to join again.": + "This remote is locked. Use the existing recovery process for devices that need to join again.", + "This remote is locked. If the device returns, use the existing recovery process to join it again.": + "This remote is locked. If the device returns, use the existing recovery process to join it again.", + "This remote does not support conditional updates, so the device record could not be removed.": + "This remote does not support conditional updates, so the device record could not be removed.", + "This remote does not currently have a record of joined devices.": + "This remote does not currently have a record of joined devices.", + "This list shows device information recorded by the remote. It does not show whether a device is online now.": + "This list shows device information recorded by the remote. It does not show whether a device is online now.", + "This device is locked out. Use the existing recovery process to join again.": + "This device is locked out. Use the existing recovery process to join again.", + "This device": "This device", + "This remote has been cleaned. Use the existing recovery process to join devices again as needed.": + "This remote has been cleaned. Use the existing recovery process to join devices again as needed.", + "The device record was forgotten.": "The device record was forgotten.", + "The device record was already absent.": "The device record was already absent.", + "The device list changed while you were trying to forget this device. Refresh the list before trying again.": + "The device list changed while you were trying to forget this device. Refresh the list before trying again.", + "The device information could not be loaded. Close this dialogue and open Joined devices again if the remote configuration has changed.": + "The device information could not be loaded. Close this dialogue and open Joined devices again if the remote configuration has changed.", + "Review device records for the active CouchDB or Object Storage connection.": + "Review device records for the active CouchDB or Object Storage connection.", + Refresh: "Refresh", + "Plug-in version": "Plug-in version", + Platform: "Platform", + "Other joined devices last reported an older plug-in version: ${versions}.": + "Other joined devices last reported an older plug-in version: ${versions}.", + "Other joined devices last reported a newer plug-in version: ${versions}.": + "Other joined devices last reported a newer plug-in version: ${versions}.", + Open: "Open", + "Not recorded": "Not recorded", + "Not accepted": "Not accepted", + "No joined devices are recorded on this remote.": "No joined devices are recorded on this remote.", + "Locked out": "Locked out", + "Loading joined devices…": "Loading joined devices…", + "Last recorded": "Last recorded", + "Joined devices could not be updated. Refresh the list and try again. If the remote configuration has changed, close this dialogue and open Joined devices again.": + "Joined devices could not be updated. Refresh the list and try again. If the remote configuration has changed, close this dialogue and open Joined devices again.", + "Joined devices are available after database startup with an active CouchDB or Object Storage connection.": + "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", + "Joined devices": "Joined devices", + "Forgetting…": "Forgetting…", + "Forgetting the record does not delete Vault files or revoke credentials.": + "Forgetting the record does not delete Vault files or revoke credentials.", + "Forget this device record?": "Forget this device record?", + "Forget device": "Forget device", + Close: "Close", + Cancel: "Cancel", + "App version": "App version", + Accepted: "Accepted", "Preparing follow-up report…": "Preparing follow-up report…", "Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.", "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.": diff --git a/src/common/replicatorProviders.ts b/src/common/replicatorProviders.ts index 845d8c2e..f3cc40b0 100644 --- a/src/common/replicatorProviders.ts +++ b/src/common/replicatorProviders.ts @@ -28,6 +28,8 @@ import { getObjectStorageReplicatorConfigurationIdentity, } from "./replicatorConfigurationIdentity"; import { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, createCouchDBConnectionProbeFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, @@ -115,6 +117,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getCouchDBReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncCouchDBReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createCouchDBJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability(createCouchDBConnectionProbeFactory(host)), [REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK]: supportedCapability( createCouchDBPreferredTweakProbeFactory(host) @@ -141,6 +146,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getObjectStorageReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncJournalReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createObjectStorageJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability( createObjectStorageConnectionProbeFactory(host) ), diff --git a/src/common/replicatorProviders.unit.spec.ts b/src/common/replicatorProviders.unit.spec.ts index 54a2c441..586e24d6 100644 --- a/src/common/replicatorProviders.unit.spec.ts +++ b/src/common/replicatorProviders.unit.spec.ts @@ -44,7 +44,15 @@ describe("central Replicator provider definitions", () => { it("keeps the retained remote-resource catalogue bounded", () => { expect .soft(Object.values(REMOTE_RESOURCE_KINDS).sort()) - .toEqual(["connection", "preferred-tweak", "security-seed", "synchronisation-information"].sort()); + .toEqual( + [ + "connection", + "joined-devices", + "preferred-tweak", + "security-seed", + "synchronisation-information", + ].sort() + ); }); it("lets Journal prepare its own fresh Security Seed while CouchDB uses central preparation", () => { diff --git a/src/common/replicatorResources/index.ts b/src/common/replicatorResources/index.ts index 9888e85c..f2f359f1 100644 --- a/src/common/replicatorResources/index.ts +++ b/src/common/replicatorResources/index.ts @@ -14,3 +14,8 @@ export { type SecuritySeedResourceHost, } from "./securitySeed"; export { createCouchDBSynchronisationInformationResourceFactory } from "./synchronisationInformation"; + +export { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, +} from "./joinedDevices"; diff --git a/src/common/replicatorResources/joinedDevices.ts b/src/common/replicatorResources/joinedDevices.ts new file mode 100644 index 00000000..a9acf1c1 --- /dev/null +++ b/src/common/replicatorResources/joinedDevices.ts @@ -0,0 +1,40 @@ +import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { + JoinedDevicesResource, + JoinedDevicesResourceFactory, + JoinedDevicesSnapshot, + ForgetJoinedDeviceResult, +} from "@vrtmrz/livesync-commonlib/replication"; +import { + LiveSyncCouchDBReplicator, + type LiveSyncCouchDBReplicatorEnv, +} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; +import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator"; +import { createReplicatorDisposer, snapshotRemoteSettings, type ResourceReplicator } from "./shared"; + +interface JoinedDevicesReplicator extends ResourceReplicator { + getJoinedDevices(setting: RemoteDBSettings): Promise; + forgetJoinedDevice(setting: RemoteDBSettings, deviceId: string): Promise; +} + +function createResource(replicator: JoinedDevicesReplicator, snapshot: RemoteDBSettings): JoinedDevicesResource { + return { + read: () => replicator.getJoinedDevices(snapshot), + forget: (deviceId) => replicator.forgetJoinedDevice(snapshot, deviceId), + dispose: createReplicatorDisposer(replicator), + }; +} + +export function createCouchDBJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncCouchDBReplicator(host), snapshotRemoteSettings(setting))); +} + +export function createObjectStorageJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncJournalReplicator(host), snapshotRemoteSettings(setting))); +} diff --git a/src/main.ts b/src/main.ts index 4dcec641..8aebf8ad 100644 --- a/src/main.ts +++ b/src/main.ts @@ -51,6 +51,7 @@ import { useInteractiveConflictResolutionFeature } from "./serviceFeatures/inter import { ConflictResolveModal } from "./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts"; import { useObsidianReplicationRibbonFeature } from "./serviceFeatures/obsidianReplicationRibbon.ts"; import { useStartupLifecycleFeature } from "./serviceFeatures/startupLifecycle"; +import { useJoinedDevicesFeature } from "./serviceFeatures/useJoinedDevices"; import { useDatabaseDiagnosticsFeature } from "./serviceFeatures/useDatabaseDiagnostics"; export type LiveSyncCore = LiveSyncBaseCore; export default class ObsidianLiveSyncPlugin extends Plugin { @@ -197,6 +198,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { useP2PReplicatorUI(core, core, replicator, createInteractiveP2PReplication(replicator)); useObsidianReplicationRibbonFeature(core); useDatabaseDiagnosticsFeature(core); + useJoinedDevicesFeature(core, serviceHub.context.noticeGroups); useRemoteConfiguration(core); useSetupProtocolFeature(core, setupManager); diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts index 8c608165..41fd1df8 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts @@ -7,6 +7,7 @@ import { type ObsidianLiveSyncSettings, LOG_LEVEL_VERBOSE, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { OPEN_JOINED_DEVICES } from "@/serviceFeatures/joinedDevicesCommand"; import { Menu, type ButtonComponent } from "@/deps.ts"; import { $msg } from "@/common/translation"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; @@ -101,6 +102,22 @@ export function paneRemoteConfig( paneEl: HTMLElement, { addPanel }: PageFunctions ): void { + new Setting(paneEl) + .setName($msg("Joined devices")) + .setDesc($msg("Review device records for the active CouchDB or Object Storage connection.")) + .addButton((button) => + button.setButtonText($msg("Open")).onClick(async () => { + const result = await this.services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + if (result.status === "unavailable" || result.value === "unavailable") { + this.services.API.addLog( + $msg( + "Joined devices are available after database startup with an active CouchDB or Object Storage connection." + ), + LOG_LEVEL_NOTICE + ); + } + }) + ); { /* E2EE */ const E2EEInitialProps = { diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts index 4d83048f..8b2a0187 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const runtime = vi.hoisted(() => ({ buttonClasses: [] as string[], - clickHandlers: [] as Array<() => Promise | void>, + clickHandlers: [] as Array<{ setting: string; run: () => Promise | void }>, panels: [] as Array<{ destroy: ReturnType }>, settingClasses: [] as string[], })); @@ -32,7 +32,10 @@ vi.mock("./LiveSyncSetting.ts", () => ({ }, }; - setName() { + settingName = ""; + + setName(name: string) { + this.settingName = name; return this; } @@ -41,6 +44,7 @@ vi.mock("./LiveSyncSetting.ts", () => ({ } addButton(callback: (button: unknown) => void) { + const setting = this.settingName; const button = { buttonEl: { classList: { @@ -53,7 +57,7 @@ vi.mock("./LiveSyncSetting.ts", () => ({ return this; }, onClick(callback: () => Promise | void) { - runtime.clickHandlers.push(callback); + runtime.clickHandlers.push({ setting, run: callback }); return this; }, setButtonText() { @@ -107,6 +111,7 @@ vi.mock("./remoteConfigBuffer.ts", () => ({ syncActivatedRemoteSettings: vi.fn(), })); +import { OPEN_JOINED_DEVICES } from "@/serviceFeatures/joinedDevicesCommand"; import { paneRemoteConfig } from "./PaneRemoteConfig.ts"; function createPanelElement(): HTMLElement { @@ -124,6 +129,18 @@ afterEach(() => { }); describe("paneRemoteConfig", () => { + it("opens joined devices through the shared UI command", async () => { + const invoke = vi.fn(async () => ({ status: "handled", value: "closed" })); + const host = { + editingSettings: { remoteConfigurations: {} }, + services: { UI: { commands: { invoke } } }, + }; + const addPanel = vi.fn(() => ({ then: vi.fn() })); + paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never); + await runtime.clickHandlers.find((handler) => handler.setting === "Joined devices")!.run(); + expect(invoke).toHaveBeenCalledWith(OPEN_JOINED_DEVICES, undefined); + }); + it("destroys the E2EE info panel when the settings page lifetime unloads", async () => { const callbacks: Array<() => unknown> = []; const lifetimeComponent = { @@ -187,7 +204,7 @@ describe("paneRemoteConfig", () => { })); paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never); - await runtime.clickHandlers[0](); + await runtime.clickHandlers.find((handler) => handler.setting === "Configure E2EE")!.run(); expect(setupManager.onlyE2EEConfiguration).toHaveBeenCalledOnce(); expect(host.editingSettings.encryptInternalMetadata).toBe(true); @@ -233,7 +250,7 @@ describe("paneRemoteConfig", () => { })); paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never); - await runtime.clickHandlers[0](); + await runtime.clickHandlers.find((handler) => handler.setting === "Configure E2EE")!.run(); expect(host.editingSettings.idDerivationVersion).toBe(1); expect(host.editingSettings.idDerivationKey).toBe(nextIdKey); diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte new file mode 100644 index 00000000..31b30356 --- /dev/null +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -0,0 +1,391 @@ + + + +
+

+ {translate("This list shows device information recorded by the remote. It does not show whether a device is online now.")} +

+ + {#if view} + {#if !view.milestonePresent} +

{translate("This remote does not currently have a record of joined devices.")}

+ {/if} + {#if view.cleaned} +

+ {translate( + "This remote has been cleaned. Use the existing recovery process to join devices again as needed." + )} +

+ {/if} + {#if view.locked} +

+ {translate( + "This remote is locked. Use the existing recovery process for devices that need to join again." + )} +

+ {/if} + {/if} + + {#if errorMessage} + + {/if} + {#if statusMessage} +

{statusMessage}

+ {/if} + {#if isRefreshing && !view} +

{translate("Loading joined devices…")}

+ {/if} + + {#if view} + {#if view.devices.length > 0} +
    + {#each view.devices as device (device.id)} +
  • +
    +
    + {#if device.self} + {translate("This device")} + {/if} + {device.id} +
    +
    + + {device.accepted ? translate("Accepted") : translate("Not accepted")} + + {#if device.lockedOut} + {translate("Locked out")} + {/if} +
    +
    + + {#if device.self && device.lockedOut} +

    + {translate( + "This device is locked out. Use the existing recovery process to join again." + )} +

    + {/if} + +
    +
    +
    {translate("Platform")}
    +
    {device.platform || translate("Unknown platform")}
    +
    +
    +
    {translate("Plug-in version")}
    +
    {device.pluginVersion || translate("Not recorded")}
    +
    +
    +
    {translate("App version")}
    +
    {device.appVersion || translate("Not recorded")}
    +
    +
    +
    {translate("Last recorded")}
    +
    {formatLastConnected(device.lastConnected)}
    +
    +
    + + {#if !device.self} + {#if confirmingId === device.id} +
    +

    {translate("Forget this device record?")}

    +

    + {translate("This removes only this device's record from the remote.")} +

    +

    + {translate( + "Forgetting the record does not delete Vault files or revoke credentials." + )} +

    +

    + {view.locked + ? translate( + "This remote is locked. If the device returns, use the existing recovery process to join it again." + ) + : translate("This remote is unlocked. A device that returns can register again.")} +

    +
    + + +
    +
    + {:else} +
    + +
    + {/if} + {/if} +
    • + {/each} +
+ {:else if !isRefreshing} +

{translate("No joined devices are recorded on this remote.")}

+ {/if} + {/if} + +
+ + +
+
+ + diff --git a/src/serviceFeatures/joinedDevices.ts b/src/serviceFeatures/joinedDevices.ts new file mode 100644 index 00000000..64bcc2cd --- /dev/null +++ b/src/serviceFeatures/joinedDevices.ts @@ -0,0 +1,64 @@ +import type { JoinedDevice, JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; +import type { JoinedDevicesView } from "./joinedDevicesDialogTypes"; + +/** Compare recognised SemVer values; unknown development labels have no ordering. */ +export function comparePluginVersions(left: string, right: string): number | undefined { + const parse = (value: string) => { + const match = + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/.exec( + value + ); + if (!match) return undefined; + const prerelease = match[4]?.split("."); + if (prerelease?.some((part) => /^0\d+$/.test(part))) return undefined; + return { core: match.slice(1, 4).map(BigInt), prerelease }; + }; + const a = parse(left); + const b = parse(right); + if (!a || !b) return undefined; + for (let i = 0; i < 3; i++) { + if (a.core[i] !== b.core[i]) return a.core[i] > b.core[i] ? 1 : -1; + } + if (!a.prerelease || !b.prerelease) return a.prerelease ? -1 : b.prerelease ? 1 : 0; + for (let i = 0; i < Math.max(a.prerelease.length, b.prerelease.length); i++) { + const x = a.prerelease[i]; + const y = b.prerelease[i]; + if (x === undefined) return -1; + if (y === undefined) return 1; + if (x === y) continue; + const xn = /^\d+$/.test(x); + const yn = /^\d+$/.test(y); + if (xn && yn) return BigInt(x) > BigInt(y) ? 1 : -1; + if (xn !== yn) return xn ? -1 : 1; + return x > y ? 1 : -1; + } + return 0; +} + +export function joinedDevicesView( + snapshot: JoinedDevicesSnapshot, + self: Omit +): JoinedDevicesView { + const ownRecord = snapshot.devices.find((device) => device.id === self.id); + const ownDevice: JoinedDevice = { ...ownRecord, ...self, accepted: ownRecord?.accepted ?? false }; + return { + ...snapshot, + devices: [ownDevice, ...snapshot.devices.filter((device) => device.id !== self.id)].map((device) => ({ + ...device, + self: device.id === self.id, + lockedOut: snapshot.locked && !device.accepted, + })), + }; +} + +export function differentDeviceVersions(snapshot: JoinedDevicesSnapshot, ownId: string, ownVersion: string) { + const newer = new Set(); + const older = new Set(); + for (const device of snapshot.devices) { + if (device.id === ownId || !device.pluginVersion) continue; + const comparison = comparePluginVersions(device.pluginVersion, ownVersion); + if (comparison !== undefined && comparison > 0) newer.add(device.pluginVersion); + if (comparison !== undefined && comparison < 0) older.add(device.pluginVersion); + } + return { newer: [...newer].sort(), older: [...older].sort() }; +} diff --git a/src/serviceFeatures/joinedDevices.unit.spec.ts b/src/serviceFeatures/joinedDevices.unit.spec.ts new file mode 100644 index 00000000..a5e4f8c8 --- /dev/null +++ b/src/serviceFeatures/joinedDevices.unit.spec.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { comparePluginVersions, differentDeviceVersions, joinedDevicesView } from "./joinedDevices"; +import type { JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; + +const empty: JoinedDevicesSnapshot = { milestonePresent: false, locked: false, cleaned: false, devices: [] }; + +describe("joined device presentation", () => { + it("always shows this device first, including before its remote record exists", () => { + const view = joinedDevicesView( + { ...empty, devices: [{ id: "another", accepted: true }] }, + { id: "self", platform: "ios", pluginVersion: "1.0.39" } + ); + expect(view.devices.map((device) => device.id)).toEqual(["self", "another"]); + expect(view.devices[0]).toMatchObject({ self: true, accepted: false, lockedOut: false, platform: "ios" }); + }); + it("keeps the remote last-recorded date and acceptance while showing current local versions", () => { + const view = joinedDevicesView( + { + ...empty, + milestonePresent: true, + locked: true, + devices: [ + { id: "other", accepted: true }, + { id: "self", accepted: false, lastConnected: 123, pluginVersion: "1.0.0" }, + ], + }, + { id: "self", pluginVersion: "1.1.0" } + ); + expect(view.devices[0]).toMatchObject({ + self: true, + lockedOut: true, + lastConnected: 123, + pluginVersion: "1.1.0", + }); + expect(view.devices[1].lockedOut).toBe(false); + }); +}); + +describe("version advisories", () => { + it.each([ + ["1.10.0", "1.9.9", 1], + ["1.0.0", "1.0.0-rc.1", 1], + ["1.0.0-beta.2", "1.0.0-beta.10", -1], + ["1.0.0-1", "1.0.0-alpha", -1], + ["1.0.0-alpha", "1.0.0-alpha.1", -1], + ["1.0.0+first", "1.0.0+second", 0], + ["99999999999999999.0.0", "99999999999999998.0.0", 1], + ])("compares %s with %s", (a, b, result) => expect(comparePluginVersions(a, b)).toBe(result)); + it.each(["", "development", "1.0", "01.0.0", "1.0.0-01", "v1.0.0"])( + "does not order unknown version %s", + (version) => { + expect(comparePluginVersions(version, "1.0.0")).toBeUndefined(); + expect(comparePluginVersions("1.0.0", version)).toBeUndefined(); + } + ); + it("excludes this device and unknown versions, and deduplicates versions", () => { + const snapshot = { + ...empty, + devices: [ + { id: "self", accepted: false, pluginVersion: "9.0.0" }, + { id: "a", accepted: true, pluginVersion: "2.0.0" }, + { id: "b", accepted: false, pluginVersion: "2.0.0" }, + { id: "c", accepted: false, pluginVersion: "0.9.0" }, + { id: "d", accepted: true, pluginVersion: "custom" }, + ], + }; + expect(differentDeviceVersions(snapshot, "self", "1.0.0")).toEqual({ newer: ["2.0.0"], older: ["0.9.0"] }); + }); +}); diff --git a/src/serviceFeatures/joinedDevicesCommand.ts b/src/serviceFeatures/joinedDevicesCommand.ts new file mode 100644 index 00000000..c661c560 --- /dev/null +++ b/src/serviceFeatures/joinedDevicesCommand.ts @@ -0,0 +1,5 @@ +import { declareUICommand } from "@vrtmrz/livesync-commonlib/ui-commands"; + +export const OPEN_JOINED_DEVICES = declareUICommand( + "livesync:joined-devices" +); diff --git a/src/serviceFeatures/joinedDevicesDialogTypes.ts b/src/serviceFeatures/joinedDevicesDialogTypes.ts new file mode 100644 index 00000000..59b28ba6 --- /dev/null +++ b/src/serviceFeatures/joinedDevicesDialogTypes.ts @@ -0,0 +1,19 @@ +import type { ForgetJoinedDeviceResult, JoinedDevice } from "@vrtmrz/livesync-commonlib/replication"; + +export interface JoinedDeviceRow extends JoinedDevice { + readonly self: boolean; + readonly lockedOut: boolean; +} + +export interface JoinedDevicesView { + readonly milestonePresent: boolean; + readonly locked: boolean; + readonly cleaned: boolean; + readonly devices: readonly JoinedDeviceRow[]; +} + +export interface JoinedDevicesDialogOptions { + refresh(): Promise; + forget(id: string): Promise; + cancel(): void; +} diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts new file mode 100644 index 00000000..37d50b0f --- /dev/null +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -0,0 +1,212 @@ +import type { NecessaryServices } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import { + NODEINFO_DOCID, + REMOTE_COUCHDB, + REMOTE_MINIO, + type EntryNodeInfo, + type RemoteDBSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { REMOTE_RESOURCE_KINDS, type JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; +import type { ObsidianNoticeGroups } from "@/modules/services/ObsidianNoticeGroups"; +import JoinedDevicesDialog from "@/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte"; +import { + getCouchDBReplicatorConfigurationIdentity, + getObjectStorageReplicatorConfigurationIdentity, +} from "@/common/replicatorConfigurationIdentity"; +import { $msg } from "@/common/translation"; +import { OPEN_JOINED_DEVICES } from "./joinedDevicesCommand"; +import { differentDeviceVersions, joinedDevicesView } from "./joinedDevices"; +import type { JoinedDevicesDialogOptions } from "./joinedDevicesDialogTypes"; + +type JoinedDevicesHost = NecessaryServices< + "API" | "UI" | "setting" | "database" | "replicator" | "replication" | "appLifecycle", + never +>; +const noticeGroup = "joined-device-versions"; + +function remoteIdentity(settings: RemoteDBSettings): string | undefined { + if (settings.remoteType === REMOTE_COUCHDB && settings.couchDB_URI && settings.couchDB_DBNAME) { + return getCouchDBReplicatorConfigurationIdentity(settings); + } + if (settings.remoteType === REMOTE_MINIO && settings.endpoint && settings.bucket) { + return getObjectStorageReplicatorConfigurationIdentity(settings); + } + return undefined; +} + +/** Own the dialogue, advisory Notices, and their per-session state for one host. */ +export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: ObsidianNoticeGroups): void { + const { services } = host; + let disposed = false; + let active: AbortController | undefined; + let checking = false; + let lastCheck: { identity: string; at: number } | undefined; + const notified = new Map>(); + const available = () => + !disposed && + services.database.isDatabaseReady() && + remoteIdentity(services.setting.currentSettings()) !== undefined; + + const notify = (snapshot: JoinedDevicesSnapshot, ownId: string, identity: string) => { + if (disposed || identity !== remoteIdentity(services.setting.currentSettings())) return; + const version = services.API.getPluginVersion(); + const differences = differentDeviceVersions(snapshot, ownId, version); + const seen = notified.get(identity) ?? new Set(); + notified.set(identity, seen); + for (const direction of ["newer", "older"] as const) { + const versions = differences[direction]; + if (versions.length === 0) continue; + const key = JSON.stringify([version, direction, versions]); + if (seen.has(key)) continue; + seen.add(key); + notices.setItem(noticeGroup, direction, { + message: + direction === "newer" + ? $msg("Other joined devices last reported a newer plug-in version: ${versions}.", { + versions: versions.join(", "), + }) + : $msg("Other joined devices last reported an older plug-in version: ${versions}.", { + versions: versions.join(", "), + }), + action: { + label: $msg("Joined devices"), + onSelect: () => { + if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { + void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + } + }, + }, + }); + notices.finish(noticeGroup, { durationMs: 15000 }); + } + }; + + const createOperations = (controller: AbortController): JoinedDevicesDialogOptions => { + const settings = { ...services.setting.currentSettings() }; + const identity = remoteIdentity(settings); + const database = services.database.localDatabase; + const assertCurrent = () => { + if ( + disposed || + controller.signal.aborted || + !identity || + identity !== remoteIdentity(services.setting.currentSettings()) || + !services.database.isDatabaseReady() || + services.database.localDatabase !== database + ) { + throw new Error("Joined devices are no longer available for this dialogue"); + } + }; + const ownId = async () => { + assertCurrent(); + const info = await database.localDatabase.get(NODEINFO_DOCID); + assertCurrent(); + if (typeof info.nodeid !== "string" || !info.nodeid) throw new Error("Device identity is unavailable"); + return info.nodeid; + }; + return { + refresh: async () => { + const id = await ownId(); + const resource = await services.replicator.createRemoteResource( + REMOTE_RESOURCE_KINDS.JOINED_DEVICES, + settings + ); + if (!resource) throw new Error("Joined devices are unavailable"); + try { + assertCurrent(); + const snapshot = await resource.read(); + assertCurrent(); + notify(snapshot, id, identity!); + return joinedDevicesView(snapshot, { + id, + platform: services.API.getPlatform(), + pluginVersion: services.API.getPluginVersion(), + appVersion: services.API.getAppVersion(), + }); + } finally { + await resource.dispose(); + } + }, + forget: async (id) => { + if (id === (await ownId())) throw new Error("This device cannot be forgotten"); + const resource = await services.replicator.createRemoteResource( + REMOTE_RESOURCE_KINDS.JOINED_DEVICES, + settings + ); + if (!resource) throw new Error("Joined devices are unavailable"); + try { + assertCurrent(); + const result = await resource.forget(id); + assertCurrent(); + return result; + } finally { + await resource.dispose(); + } + }, + cancel: () => controller.abort(), + }; + }; + + const unregister = services.UI.commands.register(OPEN_JOINED_DEVICES, async () => { + if (!available()) return "unavailable"; + if (active) return "already-open"; + const controller = new AbortController(); + active = controller; + try { + await services.UI.dialogManager.open<"close", JoinedDevicesDialogOptions>( + JoinedDevicesDialog, + createOperations(controller) + ); + } catch { + // The host may close its dialogues during unload. + } finally { + controller.abort(); + if (active === controller) active = undefined; + } + return "closed"; + }); + + const checkVersions = () => { + if (!available() || checking) return; + const identity = remoteIdentity(services.setting.currentSettings())!; + if (lastCheck?.identity === identity && Date.now() - lastCheck.at < 60000) return; + lastCheck = { identity, at: Date.now() }; + checking = true; + const controller = new AbortController(); + void Promise.resolve() + .then(() => createOperations(controller).refresh()) + .catch(() => { + // Advisory inspection never changes the outcome of synchronisation. + }) + .finally(() => { + controller.abort(); + checking = false; + }); + }; + const stopBeforeReplicate = services.replication.onBeforeReplicate.addHandler(() => { + checkVersions(); + return Promise.resolve(true); + }); + services.appLifecycle.onInitialise.addHandler(() => { + services.API.addCommand({ + id: "joined-devices", + name: $msg("Joined devices"), + icon: "monitor-smartphone", + checkCallback: (checking) => { + if (!available()) return false; + if (!checking) void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + return true; + }, + }); + return Promise.resolve(true); + }); + services.appLifecycle.onUnload.addHandler(() => { + disposed = true; + active?.abort(); + unregister(); + stopBeforeReplicate(); + notices.hide(noticeGroup); + notified.clear(); + return Promise.resolve(true); + }); +} diff --git a/src/serviceFeatures/useJoinedDevices.unit.spec.ts b/src/serviceFeatures/useJoinedDevices.unit.spec.ts new file mode 100644 index 00000000..7806508a --- /dev/null +++ b/src/serviceFeatures/useJoinedDevices.unit.spec.ts @@ -0,0 +1,174 @@ +import { describe, expect, it, vi } from "vitest"; +import { UICommandRegistry } from "@vrtmrz/livesync-commonlib/ui-commands"; +import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings"; +import { REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; +import type { JoinedDevicesDialogOptions } from "./joinedDevicesDialogTypes"; +vi.mock("@/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte", () => ({ default: {} })); +vi.mock("@/common/translation", () => ({ $msg: (message: string) => message })); +import { useJoinedDevicesFeature } from "./useJoinedDevices"; +import { OPEN_JOINED_DEVICES } from "./joinedDevicesCommand"; + +function fixture() { + const settings = { + ...createNewVaultSettings(), + remoteType: REMOTE_COUCHDB, + couchDB_URI: "https://example.test", + couchDB_DBNAME: "vault", + }; + const initialise: Array<() => Promise> = []; + const unload: Array<() => Promise> = []; + const before: Array<() => Promise> = []; + const stopBefore = vi.fn(); + const commands = new UICommandRegistry(); + const read = vi.fn<() => Promise>(async () => ({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [{ id: "other", accepted: true, pluginVersion: "2.0.0" }], + })); + const forget = vi.fn(async (_id: string) => "forgotten" as const); + const dispose = vi.fn(async () => {}); + const createResource = vi.fn(async () => ({ read, forget, dispose })); + let close!: () => void; + const dialog = vi.fn( + (_component: unknown, _options: JoinedDevicesDialogOptions) => + new Promise((resolve) => { + close = resolve; + }) + ); + const notices = { setItem: vi.fn(), finish: vi.fn(), hide: vi.fn() }; + const palette: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = []; + const database = { localDatabase: { get: vi.fn(async () => ({ nodeid: "self" })) } }; + const host = { + services: { + API: { + getPluginVersion: () => "1.0.0", + getAppVersion: () => "1.9.0", + getPlatform: () => "desktop", + addCommand: (command: (typeof palette)[number]) => palette.push(command), + }, + UI: { commands, dialogManager: { open: dialog } }, + setting: { currentSettings: () => settings }, + database: { isDatabaseReady: () => true, localDatabase: database }, + replicator: { createRemoteResource: createResource }, + replication: { + onBeforeReplicate: { + addHandler: (handler: (typeof before)[number]) => { + before.push(handler); + return stopBefore; + }, + }, + }, + appLifecycle: { + onInitialise: { addHandler: (handler: (typeof initialise)[number]) => initialise.push(handler) }, + onUnload: { addHandler: (handler: (typeof unload)[number]) => unload.push(handler) }, + }, + }, + serviceModules: {}, + } as unknown as Parameters[0]; + useJoinedDevicesFeature(host, notices as unknown as Parameters[1]); + return { + settings, + initialise, + unload, + before, + stopBefore, + commands, + read, + forget, + dispose, + createResource, + dialog, + notices, + palette, + database, + close: () => close(), + }; +} + +async function open(data: ReturnType) { + const pending = data.commands.invoke(OPEN_JOINED_DEVICES, undefined); + const options = data.dialog.mock.calls[0][1]; + return { options, pending }; +} + +describe("joined devices service feature", () => { + it("opens one dialogue through the command and disposes each completed read", async () => { + const data = fixture(); + const { options, pending } = await open(data); + expect(await data.commands.invoke(OPEN_JOINED_DEVICES, undefined)).toEqual({ + status: "handled", + value: "already-open", + }); + expect((await options.refresh()).devices[0]).toMatchObject({ id: "self", self: true }); + expect(data.dispose).toHaveBeenCalledTimes(1); + data.close(); + expect(await pending).toEqual({ status: "handled", value: "closed" }); + await expect(options.refresh()).rejects.toThrow(); + }); + it("prevents self removal and fences an old dialogue after a remote switch", async () => { + const data = fixture(); + const { options, pending } = await open(data); + await expect(options.forget("self")).rejects.toThrow(); + data.settings.couchDB_DBNAME = "new-vault"; + await expect(options.forget("other")).rejects.toThrow(); + expect(data.forget).not.toHaveBeenCalled(); + expect(data.createResource).not.toHaveBeenCalled(); + data.close(); + await pending; + }); + it("discards a stale read, releases the resource, and emits no stale Notice", async () => { + const data = fixture(); + let resolve!: (value: JoinedDevicesSnapshot) => void; + data.read.mockReturnValueOnce( + new Promise((done) => { + resolve = done; + }) + ); + const { options, pending } = await open(data); + const reading = options.refresh(); + await vi.waitFor(() => expect(data.read).toHaveBeenCalled()); + data.settings.couchDB_DBNAME = "new-vault"; + resolve({ milestonePresent: true, locked: false, cleaned: false, devices: [] }); + await expect(reading).rejects.toThrow(); + expect(data.dispose).toHaveBeenCalledOnce(); + expect(data.notices.setItem).not.toHaveBeenCalled(); + data.close(); + await pending; + }); + it("deduplicates advisory Notices and fences their actions after configuration changes", async () => { + const data = fixture(); + const { options, pending } = await open(data); + await options.refresh(); + await options.refresh(); + expect(data.notices.setItem).toHaveBeenCalledTimes(1); + const action = data.notices.setItem.mock.calls[0][2].action; + data.close(); + await pending; + data.settings.couchDB_DBNAME = "new-vault"; + action.onSelect(); + expect(data.dialog).toHaveBeenCalledTimes(1); + }); + it("keeps inspection failures out of the replication result and unregisters on unload", async () => { + const data = fixture(); + data.read.mockRejectedValue(new Error("unavailable")); + expect(await data.before[0]()).toBe(true); + await vi.waitFor(() => expect(data.dispose).toHaveBeenCalledOnce()); + expect(await data.before[0]()).toBe(true); + expect(data.read).toHaveBeenCalledTimes(1); + for (const unload of data.unload) await unload(); + expect(await data.commands.invoke(OPEN_JOINED_DEVICES, undefined)).toEqual({ status: "unavailable" }); + expect(data.stopBefore).toHaveBeenCalledOnce(); + expect(data.notices.hide).toHaveBeenCalled(); + }); + it("uses the same UI command from the command palette", async () => { + const data = fixture(); + for (const initialise of data.initialise) await initialise(); + expect(data.palette[0].checkCallback(true)).toBe(true); + expect(data.dialog).not.toHaveBeenCalled(); + data.palette[0].checkCallback(false); + expect(data.dialog).toHaveBeenCalledOnce(); + data.close(); + }); +}); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..d5522234 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -125,6 +125,8 @@ The underlying `test:e2e:obsidian:` scripts remain available for an im `test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. +`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and forgets one device through the inline confirmation. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. + `test:e2e:obsidian:settings-ui` starts with a pending compatibility review and verifies the dedicated pause summary, its detailed explanation, and the explicit resume action in a temporary real Obsidian session. It captures the desktop summary and the iPhone-sized summary and detail dialogues; the mobile checks cover viewport containment, horizontal overflow, safe-area containment, and the close control's touch target. It confirms that the acknowledged internal version advances only after the review is accepted, and checks that the Change Log contains no acknowledgement control. It then enables Advanced mode and persists one numeric Advanced setting. On Obsidian before 1.13, this covers the imperative settings fallback used by the `SettingSpec` proof. On Obsidian 1.13 or later, it verifies the task-oriented landing order, the separate Synchronisation group containing Remote Configuration and Sync Settings, and the General Settings group containing Appearance, Logging, and Extra menus. It reopens and cancels onboarding from Quick Setup, opens all 14 nested settings pages, searches globally for the Advanced control, captures the landing page, search result, Advanced page, and mobile landing page, and verifies that the value remains after the settings dialogue is closed and reopened. The mobile landing check confirms that Remote Configuration is reachable without initial scrolling. Finally, it selects the Synchronisation Settings page and verifies that the deletion panel still exposes the effective 'Keep empty folder' setting without presenting the legacy `trashInsteadDelete` control, whose value no longer changes Obsidian deletion behaviour. The default runner uses the maintained pre-1.13 Obsidian fixture and therefore exercises the complete compatibility-review, mobile-layout, and imperative-settings path. To exercise only the native settings contract against an additional Obsidian 1.13-or-later installation, supply its executable and companion CLI explicitly: diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts new file mode 100644 index 00000000..d5736139 --- /dev/null +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -0,0 +1,462 @@ +import { DEVICE_ID_PREFERRED, MILESTONE_DOCID, NODEINFO_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + fetchCouchDbDocument, + loadCouchDbConfig, + makeUniqueDatabaseName, + putCouchDbDocument, + type CouchDbDocument, +} from "../runner/couchdb.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + assertEqual, + assertE2eCompatibilityUnpaused, + configureCouchDb, + createE2eCouchDbPluginData, + prepareRemote, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { + assertLocatorHasMinimumTouchTarget, + assertLocatorWithinSafeArea, + assertLocatorWithinViewport, + assertNoHorizontalOverflow, +} from "@vrtmrz/obsidian-test-session"; +import { iPhoneSafeArea, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { + captureObsidianDialogue, + openLiveSyncSettings, + waitForVisibleObsidianDialogue, + withObsidianPage, +} from "../runner/ui.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; +import type { Locator, Page } from "playwright"; + +const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_JOINED_DEVICES_TIMEOUT_MS ?? 10000); +const commandId = "obsidian-livesync:joined-devices"; +const targetId = "joined-devices-target"; +const retainedId = "joined-devices-retained"; +const initialPlatform = "E2E Android tablet"; +const refreshedPlatform = "E2E refreshed platform"; +const connectedAt = Date.UTC(2026, 0, 12, 9, 30, 0); + +type SelfDeviceInfo = { + id: string; + platform: string; + pluginVersion: string; + appVersion: string; +}; + +type DeviceInfo = { + device_name: string; + vault_name: string; + app_version: string; + plugin_version: string; + platform: string; + progress: string; + last_connected: number; +}; + +type JoinedDevicesMilestone = CouchDbDocument & { + accepted_nodes: string[]; + node_info: Record; + locked: boolean; + cleaned: boolean; + node_chunk_info: Record; + tweak_values: Record>; +}; + +type DeviceFixture = { + milestone: JoinedDevicesMilestone; + preferredTweaks: Record; +}; + +function assert(condition: boolean, message: string): void { + if (!condition) throw new Error(message); +} + +function deviceCard(dialogue: Locator, id: string): Locator { + return dialogue.locator(".device-card").filter({ hasText: id }); +} + +async function assertDetail(card: Locator, label: string, expected: string): Promise { + const rows = card.locator(".device-details > div"); + for (let index = 0; index < (await rows.count()); index++) { + const row = rows.nth(index); + if ((await row.locator("dt").innerText()).trim() === label) { + assertEqual((await row.locator("dd").innerText()).trim(), expected, `${label} did not match.`); + return; + } + } + throw new Error(`The device details did not contain '${label}'.`); +} + +async function readSelfDeviceInfo(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + `const nodeInfo=await core.services.database.localDatabase.localDatabase.get(${JSON.stringify(NODEINFO_DOCID)});`, + "return JSON.stringify({", + "id:nodeInfo.nodeid,", + "platform:core.services.API.getPlatform(),", + "pluginVersion:core.services.API.getPluginVersion(),", + "appVersion:core.services.API.getAppVersion(),", + "});", + "})()", + ].join(""), + env + ); +} + +function createFixture(self: SelfDeviceInfo): DeviceFixture { + const preferredTweaks = { customChunkSize: 60, chunkSplitterVersion: "v3-rabin-karp" }; + const milestone: JoinedDevicesMilestone = { + _id: MILESTONE_DOCID, + type: "milestoneinfo", + created: Date.now(), + // The current device is deliberately absent: it must still be listed first and identified as locked out. + accepted_nodes: [targetId, retainedId], + node_info: { + [targetId]: { + device_name: "E2E target device", + vault_name: "Joined devices target fixture", + app_version: "1.99.0-test", + plugin_version: "999.0.0", + platform: initialPlatform, + progress: "ready", + last_connected: connectedAt, + }, + [retainedId]: { + device_name: "E2E retained device", + vault_name: "Joined devices retained fixture", + app_version: "1.88.0-test", + plugin_version: "0.1.0", + platform: "E2E Linux desktop", + progress: "ready", + last_connected: connectedAt - 86400000, + }, + }, + locked: true, + cleaned: true, + node_chunk_info: { + [targetId]: { min: 1, max: 3, current: 3 }, + [retainedId]: { min: 1, max: 2, current: 2 }, + }, + tweak_values: { + [targetId]: { customChunkSize: 48 }, + [retainedId]: { customChunkSize: 60 }, + [DEVICE_ID_PREFERRED]: preferredTweaks, + }, + }; + assert(self.id !== targetId && self.id !== retainedId, "The local device identity collided with a fixture ID."); + return { milestone, preferredTweaks }; +} + +async function openFromRemoteSettings(page: Page): Promise { + const settings = await openLiveSyncSettings(page, uiTimeoutMs); + const remoteConfiguration = await settings.openPage("Remote Configuration"); + await remoteConfiguration.getByRole("button", { name: "Open", exact: true }).click({ timeout: uiTimeoutMs }); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + return dialogue; +} + +async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, page: Page): Promise { + const cards = dialogue.locator(".device-card"); + assertEqual( + await cards.count(), + 3, + "The joined devices dialogue did not show the self, target, and retained records." + ); + const selfCard = cards.nth(0); + assert(Boolean(self.id), "The current device identity was not read from the local NODEINFO_DOCID record."); + assertEqual( + (await selfCard.locator("code.device-id").innerText()).trim(), + self.id, + "The current device was not listed first." + ); + await selfCard.getByText("This device", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await selfCard.getByText("Locked out", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await selfCard.getByText("Not accepted", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await selfCard + .getByText("This device is locked out.", { exact: false }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await assertDetail(selfCard, "Platform", self.platform); + await assertDetail(selfCard, "Plug-in version", self.pluginVersion); + await assertDetail(selfCard, "App version", self.appVersion); + assertEqual( + await selfCard.getByRole("button", { name: "Forget device", exact: true }).count(), + 0, + "The current device exposed a forget control." + ); + + const targetCard = deviceCard(dialogue, targetId); + await assertDetail(targetCard, "Platform", initialPlatform); + await assertDetail(targetCard, "Plug-in version", "999.0.0"); + await assertDetail(targetCard, "App version", "1.99.0-test"); + const targetDate = await page.evaluate((timestamp) => new Date(timestamp).toLocaleString(), connectedAt); + await assertDetail(targetCard, "Last recorded", targetDate); + await targetCard.getByText("Accepted", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + + const retainedCard = deviceCard(dialogue, retainedId); + await assertDetail(retainedCard, "Platform", "E2E Linux desktop"); + await assertDetail(retainedCard, "Plug-in version", "0.1.0"); + await assertDetail(retainedCard, "App version", "1.88.0-test"); + await dialogue + .getByText( + "This list shows device information recorded by the remote. It does not show whether a device is online now.", + { exact: true } + ) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dialogue + .getByText("This remote is locked.", { exact: false }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dialogue + .getByText("This remote has been cleaned.", { exact: false }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); +} + +async function refreshRemotePlatform( + page: Page, + config: Awaited>, + dbName: string +): Promise { + const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; + await putCouchDbDocument(config, dbName, { + ...milestone, + node_info: { + ...milestone.node_info, + [targetId]: { ...milestone.node_info[targetId], platform: refreshedPlatform }, + }, + }); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByRole("button", { name: "Refresh", exact: true }).click({ timeout: uiTimeoutMs }); + await deviceCard(dialogue, targetId).getByText(refreshedPlatform, { exact: true }).waitFor({ + state: "visible", + timeout: uiTimeoutMs, + }); +} + +async function showForgetConfirmation(dialogue: Locator): Promise { + const targetCard = deviceCard(dialogue, targetId); + await targetCard.getByRole("button", { name: "Forget device", exact: true }).click({ timeout: uiTimeoutMs }); + await targetCard.getByText("Forget this device record?", { exact: true }).waitFor({ + state: "visible", + timeout: uiTimeoutMs, + }); +} + +async function confirmForgetTarget(dialogue: Locator): Promise { + const targetCard = deviceCard(dialogue, targetId); + await targetCard.getByRole("button", { name: "Forget device", exact: true }).click({ timeout: uiTimeoutMs }); + await dialogue.getByText("The device record was forgotten.", { exact: true }).waitFor({ + state: "visible", + timeout: uiTimeoutMs, + }); + await targetCard.waitFor({ state: "detached", timeout: uiTimeoutMs }); +} + +async function assertTargetForgotten( + config: Awaited>, + dbName: string, + fixture: DeviceFixture +): Promise { + const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; + assertEqual( + JSON.stringify(milestone.accepted_nodes), + JSON.stringify([retainedId]), + "Forgetting changed the wrong accepted device records." + ); + assertEqual( + JSON.stringify(Object.keys(milestone.node_info)), + JSON.stringify([retainedId]), + "Forgetting changed the wrong device metadata records." + ); + assertEqual( + JSON.stringify(Object.keys(milestone.node_chunk_info)), + JSON.stringify([retainedId]), + "Forgetting changed the wrong chunk metadata records." + ); + assertEqual( + JSON.stringify(Object.keys(milestone.tweak_values).sort()), + JSON.stringify([DEVICE_ID_PREFERRED, retainedId].sort()), + "Forgetting changed the wrong tweak metadata records." + ); + assertEqual(milestone.locked, true, "Forgetting changed the remote lock state."); + assertEqual(milestone.cleaned, true, "Forgetting changed the remote cleaned state."); + assertEqual( + JSON.stringify(milestone.tweak_values[DEVICE_ID_PREFERRED]), + JSON.stringify(fixture.preferredTweaks), + "Forgetting changed the preferred-ID tweak record." + ); + assertEqual( + JSON.stringify(milestone.node_info[retainedId]), + JSON.stringify(fixture.milestone.node_info[retainedId]), + "Forgetting changed the retained device metadata." + ); +} + +async function assertScrollableMobileDialogue(page: Page, container: Locator): Promise { + const dialogue = container.locator(".modal").last(); + const closeButton = container.locator(".modal-close-button, .modal-header-button").last(); + await assertLocatorWithinViewport(page, dialogue, { label: "Joined devices dialogue" }); + await assertNoHorizontalOverflow(page, dialogue, { label: "Joined devices dialogue" }); + await assertLocatorWithinSafeArea(page, dialogue, { + label: "Joined devices dialogue", + safeAreaInsets: iPhoneSafeArea, + }); + const buttons = [closeButton, ...(await dialogue.locator("button:visible").all())]; + for (const button of buttons) { + // Device records form a scrollable list; every control must be reachable after scrolling. + await button.scrollIntoViewIfNeeded(); + const label = (await button.innerText()).trim() || "Close dialogue"; + await assertLocatorWithinViewport(page, button, { label }); + await assertLocatorWithinSafeArea(page, button, { label, safeAreaInsets: iPhoneSafeArea }); + await assertNoHorizontalOverflow(page, button, { label }); + await assertLocatorHasMinimumTouchTarget(page, button, { label }); + } + await container.locator(".modal-content").evaluate((element) => element.scrollTo(0, 0)); +} + +async function dismissNotices(page: Page): Promise { + const notices = page.locator(".notice:visible"); + for (let remaining = 20; remaining > 0 && (await notices.count()) > 0; remaining--) { + const notice = await notices.first().elementHandle(); + if (!notice) continue; + try { + await notice.click({ position: { x: 20, y: 20 }, timeout: uiTimeoutMs }); + await notice.waitForElementState("hidden", { timeout: uiTimeoutMs }); + } finally { + await notice.dispose(); + } + } + assertEqual(await notices.count(), 0, "Notices still obscured the dialogue screenshot."); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + + const couchDb = await loadCouchDbConfig(); + const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "joined-devices"); + const vault = await createTemporaryVault(); + let session: ObsidianLiveSyncSession | undefined; + + try { + await assertCouchDbReachable(couchDb); + await createCouchDbDatabase(couchDb, dbName); + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli.binary, + vault, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + pluginData: createE2eCouchDbPluginData({ + uri: couchDb.uri, + username: couchDb.username, + password: couchDb.password, + dbName, + }), + }); + await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); + await assertE2eCompatibilityUnpaused(cli.binary, session.cliEnv, session.remoteDebuggingPort); + await configureCouchDb(cli.binary, session.cliEnv, { + uri: couchDb.uri, + username: couchDb.username, + password: couchDb.password, + dbName, + }); + await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); + await prepareRemote(cli.binary, session.cliEnv); + + const self = await readSelfDeviceInfo(cli.binary, session.cliEnv); + const fixture = createFixture(self); + const preparedMilestone = await fetchCouchDbDocument(couchDb, dbName, MILESTONE_DOCID); + assert(Boolean(preparedMilestone._rev), "Remote preparation did not create a revisioned milestone document."); + await putCouchDbDocument(couchDb, dbName, { ...fixture.milestone, _rev: preparedMilestone._rev }); + + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { + await page.setViewportSize({ width: 1280, height: 1100 }); + const dialogue = await openFromRemoteSettings(page); + await assertInitialDialogue(dialogue, self, page); + await page + .locator(".notice") + .getByText("Other joined devices last reported a newer plug-in version: 999.0.0.", { exact: true }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + }); + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByRole("button", { name: "Close", exact: true }).click({ timeout: uiTimeoutMs }); + await page.locator(".notice").getByRole("button", { name: "Joined devices", exact: true }).first().click(); + const reopened = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await reopened.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dismissNotices(page); + }); + await captureObsidianDialogue( + session.remoteDebuggingPort, + "joined-devices-forget-confirmation.png", + async (page) => { + await refreshRemotePlatform(page, couchDb, dbName); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await showForgetConfirmation(dialogue); + await deviceCard(dialogue, targetId).locator(".forget-confirmation").scrollIntoViewIfNeeded(); + } + ); + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-after-forget.png", async (page) => { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await confirmForgetTarget(dialogue); + await dialogue.locator(".modal-content").evaluate((element) => element.scrollTo(0, 0)); + }); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByRole("button", { name: "Close", exact: true }).click({ timeout: uiTimeoutMs }); + const settings = await openLiveSyncSettings(page, uiTimeoutMs); + await settings.close(); + }); + await assertTargetForgotten(couchDb, dbName, fixture); + + await setObsidianMobileTestMode(session.remoteDebuggingPort, true, uiTimeoutMs); + await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-mobile.png", async (page) => { + const opened = await page.evaluate((id) => { + const app = ( + window as typeof window & { + app?: { commands?: { executeCommandById?(commandId: string): boolean } }; + } + ).app; + return app?.commands?.executeCommandById?.(id) === true; + }, commandId); + assert(opened, "The Joined devices command did not open the dialogue."); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByText(retainedId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await assertScrollableMobileDialogue(page, dialogue); + assertEqual( + await dialogue.locator(".device-card").count(), + 2, + "The refreshed mobile list did not retain both remaining devices." + ); + await dismissNotices(page); + }); + + console.log(`Joined devices scenario passed against CouchDB database ${dbName}.`); + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { + await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => { + console.warn(error instanceof Error ? error.message : error); + }); + } + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 7f766c67..9143b830 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -8,6 +8,7 @@ const focusedScenarios = new Set([ "smoke", "onboarding-invitation", "dialog-mounts", + "joined-devices", "database-diagnostics", "database-diagnostics-couchdb", "conflict-dialog-policy", diff --git a/updates.md b/updates.md index ca2e3497..d99fa163 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,14 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +- We can review participating devices through **Joined devices** in Remote Configuration, the command palette, or a version Notice. + - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. + - Advisory Notices help us find devices which last reported newer or older plug-in versions. +- We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. + - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. + ## 1.0.39 20th October, 2026 From e56098eb5728a25303945fd8c42ba15ff3dabae9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 13:53:56 +0000 Subject: [PATCH 09/28] Fix diagnostics checks across Obsidian renderers --- .../scripts/database-diagnostics.ts | 119 +++++++++++------- 1 file changed, 74 insertions(+), 45 deletions(-) diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts index 773c60c2..df54a7ba 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -31,6 +31,18 @@ type DiagnosticPageGlobal = typeof globalThis & { diagnosticDownload?: { name: string; content: string }; }; +async function findVisibleElement(page: Page, selector: string): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + for (const candidate of page.context().pages()) { + const element = candidate.locator(selector); + if (await element.isVisible()) return element; + } + await page.waitForTimeout(50); + } + throw new Error(`No Obsidian renderer contains the visible element: ${selector}`); +} + async function openCommand(page: Page, kind: "usage" | "integrity") { const id = kind === "usage" ? "analyse-database" : "inspect-database-integrity"; const opened = await page.evaluate( @@ -243,11 +255,14 @@ async function assertDiagnosticFileLookup( session: ObsidianLiveSyncSession, mobile = false ): Promise { + const dialoguePage = hatch.page(); + const viewport = page.viewportSize(); + if (viewport) await dialoguePage.setViewportSize(viewport); const setting = hatch.locator(".setting-item").filter({ - has: page.getByText("Find a file from a report pseudonym", { exact: true }), + has: hatch.page().getByText("Find a file from a report pseudonym", { exact: true }), }); await setting.getByRole("button", { name: "Find file", exact: true }).click(); - const dialogue = page.locator(".diagnostic-file-lookup"); + const dialogue = dialoguePage.locator(".diagnostic-file-lookup"); await dialogue.waitFor({ state: "visible", timeout }); const input = dialogue.getByLabel("Report path pseudonym", { exact: true }); await input.fill("file-001"); @@ -259,8 +274,8 @@ async function assertDiagnosticFileLookup( await candidates.first().waitFor({ timeout }); assert.equal(await candidates.count(), 2, "Distinct Metadata IDs claiming the same path must remain selectable"); if (mobile) { - await assertNoHorizontalOverflow(page, dialogue, { label: "Pseudonym file lookup matches" }); - await assertNoHorizontalOverflow(page, dialogue.locator(".lookup-candidates"), { + await assertNoHorizontalOverflow(dialoguePage, dialogue, { label: "Pseudonym file lookup matches" }); + await assertNoHorizontalOverflow(dialoguePage, dialogue.locator(".lookup-candidates"), { label: "Pseudonym file lookup candidate list", }); assert.ok( @@ -270,7 +285,7 @@ async function assertDiagnosticFileLookup( await captureObsidianElement( session.remoteDebuggingPort, "database-diagnostics-file-matches-mobile.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".diagnostic-file-lookup") }) + (page) => findVisibleElement(page, ".modal-container:has(.diagnostic-file-lookup)") ); } const shadow = candidates.filter({ hasText: shadowId }); @@ -294,28 +309,30 @@ async function assertDiagnosticFileLookup( for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) { assert.ok(!json.includes(privateValue), "A follow-up report must preserve the original report's pseudonyms"); } - await installDiagnosticExportInterception(page); + await installDiagnosticExportInterception(dialoguePage); await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); - assert.equal(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), json); + assert.equal(await dialoguePage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), json); await dialogue.getByRole("button", { name: "Save report", exact: true }).click(); - await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { timeout }); - assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { + await dialoguePage.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { + timeout, + }); + assert.deepEqual(await dialoguePage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { name: "diagnostic-file-report.json", content: json, }); - await assertDesktopDiagnosticResultLayout(page, dialogue, "saving a follow-up report"); + await assertDesktopDiagnosticResultLayout(dialoguePage, dialogue, "saving a follow-up report"); if (mobile) { - const modal = page.locator(".modal-container").filter({ has: dialogue }); - await assertMobileDialogueLayout(page, modal, "Pseudonym file follow-up report"); - await assertNoHorizontalOverflow(page, dialogue, { label: "Pseudonym file follow-up content" }); - await assertNoHorizontalOverflow(page, dialogue.locator(".report-token"), { + const modal = dialoguePage.locator(".modal-container").filter({ has: dialogue }); + await assertMobileDialogueLayout(dialoguePage, modal, "Pseudonym file follow-up report"); + await assertNoHorizontalOverflow(dialoguePage, dialogue, { label: "Pseudonym file follow-up content" }); + await assertNoHorizontalOverflow(dialoguePage, dialogue.locator(".report-token"), { label: "Pseudonym file follow-up token", }); } await captureObsidianElement( session.remoteDebuggingPort, mobile ? "database-diagnostics-file-follow-up-mobile.png" : "database-diagnostics-file-follow-up.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".diagnostic-file-lookup") }) + (page) => findVisibleElement(page, ".modal-container:has(.diagnostic-file-lookup)") ); await dialogue.getByRole("radio", { name: "MARKDOWN", exact: true }).check(); const markdown = await dialogue.locator("textarea").inputValue(); @@ -379,10 +396,10 @@ async function assertFollowUpStopsAfterClose(page: Page, hatch: Locator, token: })()`); try { const setting = hatch.locator(".setting-item").filter({ - has: page.getByText("Find a file from a report pseudonym", { exact: true }), + has: hatch.page().getByText("Find a file from a report pseudonym", { exact: true }), }); await setting.getByRole("button", { name: "Find file", exact: true }).click(); - const dialogue = page.locator(".diagnostic-file-lookup"); + const dialogue = hatch.page().locator(".diagnostic-file-lookup"); await dialogue.getByLabel("Report path pseudonym", { exact: true }).fill(token); await dialogue.getByRole("button", { name: "Find file", exact: true }).click(); const candidate = dialogue.locator("[data-diagnostic-candidate]").filter({ hasText: shadowId }); @@ -466,7 +483,7 @@ async function main(): Promise { const activeSession = session; let usageJSON = ""; let usageFixtureRow: { path: string; id: string; revision: string } | undefined; - await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const desktopScreenshot = await withObsidianPage(session.remoteDebuggingPort, async (page) => { await page.setViewportSize({ width: 800, height: 600 }); const choice = await openCommand(page, "usage"); await choice.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).focus(); @@ -560,13 +577,15 @@ async function main(): Promise { hatch = await (await openLiveSyncSettings(page, timeout)).openPage("Hatch"); await assertFollowUpStopsAfterClose(page, hatch, usageFixtureRow.path); await captureObsidianElement(activeSession.remoteDebuggingPort, "database-diagnostics-hatch.png", (page) => - page.locator(".modal.mod-settings") + findVisibleElement(page, ".modal.mod-settings") ); const setting = hatch .locator(".setting-item") - .filter({ has: page.getByText("Inspect database integrity", { exact: true }) }); + .filter({ has: hatch.page().getByText("Inspect database integrity", { exact: true }) }); await setting.getByRole("button", { name: "Inspect", exact: true }).click(); - const progressDialogue = page.locator(".database-diagnostic"); + const progressDialogue = await findVisibleElement(page, ".database-diagnostic"); + const progressPage = progressDialogue.page(); + await progressPage.setViewportSize({ width: 800, height: 600 }); assert.equal( await progressDialogue .getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }) @@ -579,7 +598,7 @@ async function main(): Promise { .isChecked(), true ); - await assertDesktopDiagnosticChoiceLayout(page, progressDialogue); + await assertDesktopDiagnosticChoiceLayout(progressPage, progressDialogue); await installDiagnosticReadGate(cli, activeSession, shadowId, "metadata"); try { await progressDialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); @@ -593,7 +612,7 @@ async function main(): Promise { await captureObsidianElement( activeSession.remoteDebuggingPort, "database-diagnostics-progress.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await progressDialogue.getByRole("button", { name: "Cancel inspection", exact: true }).click(); await evalObsidianJson( @@ -601,7 +620,7 @@ async function main(): Promise { `(async()=>{globalThis.databaseDiagnosticReadGate?.release();return JSON.stringify(true);})()`, activeSession.cliEnv ); - await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await progressPage.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); await assertFormatChoices(progressDialogue, ["MARKDOWN", "JSON"]); await chooseFormat(progressDialogue, "json"); assert.equal(JSON.parse(await progressDialogue.locator("textarea").inputValue()).status, "cancelled"); @@ -612,33 +631,41 @@ async function main(): Promise { } await setting.getByRole("button", { name: "Inspect", exact: true }).click(); - const integrity = page.locator(".database-diagnostic"); + const integrity = await findVisibleElement(page, ".database-diagnostic"); + const integrityPage = integrity.page(); assert.equal( await integrity.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).isChecked(), true ); await integrity.getByRole("button", { name: "Begin inspection", exact: true }).click(); - await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await integrityPage.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); await assertFormatChoices(integrity, ["MARKDOWN", "JSON"]); assert.equal(await integrity.getByRole("radio", { name: "MARKDOWN", exact: true }).isChecked(), true); const markdownReport = await integrity.locator("textarea").inputValue(); const markdownDetailsWithCodeBlock = /
\s*Database integrity report \(partial\)<\/summary>\s*(`{3,})text\r?\n[\s\S]*?\1\s*<\/details>/; assert.match(markdownReport, markdownDetailsWithCodeBlock); - await installDiagnosticExportInterception(page); + await installDiagnosticExportInterception(integrityPage); await integrity.getByRole("button", { name: "Copy report", exact: true }).click(); assert.equal( - await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), + await integrityPage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), markdownReport ); await integrity.getByRole("button", { name: "Save report", exact: true }).click(); - await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { - timeout, - }); - assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { - name: "database-integrity-partial.md", - content: markdownReport, - }); + await integrityPage.waitForFunction( + () => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, + undefined, + { + timeout, + } + ); + assert.deepEqual( + await integrityPage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), + { + name: "database-integrity-partial.md", + content: markdownReport, + } + ); await chooseFormat(integrity, "json"); const reportText = await integrity.locator("textarea").inputValue(); const integrityReport = JSON.parse(reportText); @@ -670,15 +697,17 @@ async function main(): Promise { ); for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) assert.ok(!reportText.includes(privateValue)); + const screenshot = await captureObsidianElement( + activeSession.remoteDebuggingPort, + "database-diagnostics-desktop.png", + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") + ); + await integrity.getByRole("button", { name: "Close", exact: true }).click(); + await integrity.waitFor({ state: "hidden", timeout }); + await settingsNavigator.close(); + return screenshot; }); - const desktopScreenshot = await captureObsidianElement( - session.remoteDebuggingPort, - "database-diagnostics-desktop.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) - ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { - await page.locator(".database-diagnostic").getByRole("button", { name: "Close", exact: true }).click(); - await page.keyboard.press("Escape"); const localOnly = await openCommand(page, "integrity"); await localOnly.getByRole("checkbox", { name: "Include remote database", exact: true }).uncheck(); await localOnly.getByRole("button", { name: "Begin inspection", exact: true }).click(); @@ -780,7 +809,7 @@ async function main(): Promise { const mobileScreenshot = await captureObsidianElement( session.remoteDebuggingPort, "database-diagnostics-mobile.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { await page.setViewportSize({ width: 375, height: 667 }); @@ -828,7 +857,7 @@ async function main(): Promise { const mobileResultScreenshot = await captureObsidianElement( session.remoteDebuggingPort, "database-diagnostics-mobile-result.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { await page.setViewportSize({ width: 375, height: 667 }); @@ -857,7 +886,7 @@ async function main(): Promise { await captureObsidianElement( activeSession.remoteDebuggingPort, "database-diagnostics-mobile-saved.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await dialogue.getByRole("button", { name: "Close", exact: true }).click(); await dialogue.waitFor({ state: "hidden", timeout }); From 96ad2737a7e846205e052da371e9a17cf8dfe43e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 13:56:54 +0000 Subject: [PATCH 10/28] Adopt Commonlib 0.2.0 with injected device storage --- devs.md | 2 + package-lock.json | 18 +- package.json | 3 +- src/apps/browser/LiveSyncBrowserUIService.ts | 4 +- .../createLiveSyncBrowserServiceHub.ts | 50 +---- ...eateLiveSyncBrowserServiceHub.unit.spec.ts | 43 ++++- src/apps/cli/README.md | 4 +- src/apps/cli/main.bootstrap.unit.spec.ts | 21 ++ src/apps/cli/main.ts | 4 - src/apps/cli/package.json | 2 +- src/apps/cli/services/NodeLocalStorage.ts | 159 +++++++-------- .../services/NodeLocalStorage.unit.spec.ts | 127 +++++++++--- src/apps/cli/services/NodeServiceHub.ts | 19 +- src/apps/cli/services/NodeSettingService.ts | 42 ---- src/apps/webapp/WebAppRuntime.ts | 14 +- src/apps/webpeer/src/WebPeerRuntime.ts | 10 +- src/common/replicatorProviders.ts | 8 + src/common/replicatorProviders.unit.spec.ts | 22 ++- src/common/replicatorResources.unit.spec.ts | 69 +++++++ src/common/replicatorResources/index.ts | 5 + .../replicatorResources/joinedDevices.ts | 40 ++++ src/common/turnSettingsPrivacy.unit.spec.ts | 12 +- .../P2PReplicator/P2PReplicatorPane.svelte | 134 +++++++++++-- .../ReviewHarness/ReviewHarnessView.ts | 3 +- .../ReviewHarness/reviewHarnessController.ts | 27 ++- .../reviewHarnessController.unit.spec.ts | 151 +++++++++++++-- .../features/ModuleObsidianSettingTab.ts | 6 +- .../ModuleObsidianSettingTab.unit.spec.ts | 34 +++- ...iveSyncSettingTab.declarative.unit.spec.ts | 14 +- .../ObsidianLiveSyncSettingTab.ts | 30 ++- .../ObsidianLiveSyncSettingTab.unit.spec.ts | 65 ++++++- .../features/SetupManager.unit.spec.ts | 7 +- .../SetupWizard/dialogs/SetupRemoteP2P.svelte | 44 ++++- src/modules/services/ObsidianServiceHub.ts | 9 +- src/modules/services/ObsidianServices.ts | 7 +- .../services/ObsidianSettingService.ts | 14 -- .../ObsidianSettingService.unit.spec.ts | 39 ++++ src/modules/services/ObsidianUIService.ts | 4 +- src/serviceFeatures/compatibilityReview.ts | 18 +- .../compatibilityReview.unit.spec.ts | 128 +++++++++++- src/serviceFeatures/redFlag.simpleFetch.ts | 37 ++-- .../redFlag.simpleFetch.unit.spec.ts | 84 +++++++- src/serviceFeatures/redFlag.unit.spec.ts | 22 ++- src/serviceFeatures/useReviewHarness.ts | 14 +- .../useReviewHarness.unit.spec.ts | 82 +++++++- test/apps/webapp/WebAppRuntime.unit.spec.ts | 17 +- test/apps/webpeer/WebPeerRuntime.unit.spec.ts | 41 +++- test/contracts/serviceContext.ts | 5 +- test/e2e-obsidian/runner/liveSyncWorkflow.ts | 4 +- test/e2e-obsidian/runner/upgradeWorkflow.ts | 4 +- test/e2e-obsidian/scripts/dialog-mounts.ts | 182 ++++++++++++------ test/e2e-obsidian/scripts/review-harness.ts | 14 +- test/e2e-obsidian/scripts/settings-ui.ts | 26 ++- updates.md | 4 + utils/couchdb/livesync-commonlib.ts | 4 +- utils/flyio/deno.lock | 57 +++++- utils/flyio/generate_setupuri.test.ts | 10 +- utils/livesync-commonlib-version.ts | 2 +- utils/setup/generate_setup_uri.test.ts | 7 +- utils/setup/livesync-commonlib.ts | 12 +- 60 files changed, 1499 insertions(+), 541 deletions(-) delete mode 100644 src/apps/cli/services/NodeSettingService.ts create mode 100644 src/common/replicatorResources/joinedDevices.ts diff --git a/devs.md b/devs.md index a4d3849d..77c31028 100644 --- a/devs.md +++ b/devs.md @@ -309,6 +309,8 @@ Existing legacy Modules continue to register their handlers in `onBindFunction() database and Replicator readiness. See [the declarative settings adapter ADR](docs/adr/2026_08_declarative_settings_adapter.md). - Use `this.services.setting.saveSettingData()` instead of using plugin methods directly +- Inject `@vrtmrz/value-store` at host composition. `setting.deviceValues` uses exact device-local keys; `setting.config` and the Service Hub `config` share the existing Vault and database-suffix scope. Await reads, writes, and deletions before advancing the workflow. Browser and Obsidian hosts retain their Web Storage keys, and the CLI retains its existing flat JSON string map. +- Seed the settings tab with `await settingTab.loadSettings()` before registration. Its editing buffer remains synchronous for Obsidian rendering; storage failures must not replace a previously loaded value with an empty default. ### Database Operations diff --git a/package-lock.json b/package-lock.json index 1d8d4dfb..4cf6a3e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,9 +23,10 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.44", + "@vrtmrz/livesync-commonlib": "0.2.0", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", + "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", "idb": "^8.0.3", "markdown-it": "^14.3.2", @@ -4115,9 +4116,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.44", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.44.tgz", - "integrity": "sha512-GjXNSLTnWLZzdgrhW3sIOsLqGLP4iMgypPMtcBc2Ju2vMq64TRA1Llfoya1KRIeu6m0G7Xu3ZtkWARTRcHkYYg==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.0.tgz", + "integrity": "sha512-K24Pkke+FTjZI7DjIV7qkZk+BmC9VebN3da+2cVD9SWOgoGl7OvDyXTHDafbrmmM4vi4ydaydAEh9glO1xHF6A==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -4128,6 +4129,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@trystero-p2p/nostr": "0.25.3", + "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", "events": "^3.3.0", "fflate": "^0.8.2", @@ -4139,9 +4141,11 @@ "pouchdb-adapter-idb": "^9.0.0", "pouchdb-adapter-indexeddb": "^9.0.0", "pouchdb-adapter-memory": "^9.0.0", + "pouchdb-checkpointer": "9.0.0", "pouchdb-core": "^9.0.0", "pouchdb-errors": "^9.0.0", "pouchdb-find": "^9.0.0", + "pouchdb-generate-replication-id": "9.0.0", "pouchdb-mapreduce": "^9.0.0", "pouchdb-merge": "^9.0.0", "pouchdb-replication": "^9.0.0", @@ -4210,6 +4214,12 @@ "integrity": "sha512-njT2BSFh57imwGDxWXJOcGbZOYFVNGmeXMrS7/RUdBoAeOjWzJB9TxI7WcR1sgQIXKh8TE9hqALU8lue5b8dRw==", "license": "MIT" }, + "node_modules/@vrtmrz/value-store": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@vrtmrz/value-store/-/value-store-0.1.0.tgz", + "integrity": "sha512-M62f1j0bnJkqpgI5sYxnkXUAOSDEwQbYrLf+w58gCCm4sjIXDmFzDJd2duXabRbktBlACt9P99yjN/9gOAPxrA==", + "license": "MIT" + }, "node_modules/abstract-leveldown": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", diff --git a/package.json b/package.json index a6ff13d4..f434524a 100644 --- a/package.json +++ b/package.json @@ -193,9 +193,10 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.44", + "@vrtmrz/livesync-commonlib": "0.2.0", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", + "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", "idb": "^8.0.3", "markdown-it": "^14.3.2", diff --git a/src/apps/browser/LiveSyncBrowserUIService.ts b/src/apps/browser/LiveSyncBrowserUIService.ts index ea23cf5a..c0ab9d00 100644 --- a/src/apps/browser/LiveSyncBrowserUIService.ts +++ b/src/apps/browser/LiveSyncBrowserUIService.ts @@ -1,6 +1,6 @@ import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; import type { AppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService"; -import type { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService"; +import type { IConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import type { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import type { ReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/base/ReplicatorService"; import type { InjectableAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAPIService"; @@ -11,7 +11,7 @@ import { BrowserSvelteDialogManager } from "./BrowserSvelteDialogManager"; export interface LiveSyncBrowserUIServiceDependencies { API: InjectableAPIService; appLifecycle: AppLifecycleService; - config: ConfigService; + config: IConfigService; control: ControlService; replicator: ReplicatorService; } diff --git a/src/apps/browser/createLiveSyncBrowserServiceHub.ts b/src/apps/browser/createLiveSyncBrowserServiceHub.ts index 76c671c0..3197114a 100644 --- a/src/apps/browser/createLiveSyncBrowserServiceHub.ts +++ b/src/apps/browser/createLiveSyncBrowserServiceHub.ts @@ -1,11 +1,10 @@ import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { KeyValueDatabaseFactory } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase"; import { PouchDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/pouchdb-browser"; -import { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService"; import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService"; import { KeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/KeyValueDBService"; -import type { ISettingService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import type { ValueDeleter, ValueStore } from "@vrtmrz/value-store"; import { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAppLifecycleService"; import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService"; @@ -24,10 +23,7 @@ import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/ import { setLang, translateLiveSyncMessage } from "@/common/translation"; import { BrowserConfirm } from "./BrowserConfirm"; import { createBrowserKeyValueDatabaseFactory } from "./BrowserKeyValueDatabase"; -import { - LiveSyncBrowserAPIService, - type LiveSyncBrowserAPIServiceOptions, -} from "./LiveSyncBrowserAPIService"; +import { LiveSyncBrowserAPIService, type LiveSyncBrowserAPIServiceOptions } from "./LiveSyncBrowserAPIService"; import { LiveSyncBrowserUIService } from "./LiveSyncBrowserUIService"; export interface LiveSyncBrowserSettingsPersistence { @@ -43,6 +39,7 @@ export interface LiveSyncBrowserRestartPolicy { } export interface LiveSyncBrowserServiceHubOptions { + deviceValues: ValueStore & ValueDeleter; context?: T; getSystemVaultName?: () => string; settings?: LiveSyncBrowserSettingsPersistence; @@ -51,35 +48,12 @@ export interface LiveSyncBrowserServiceHubOptions { API?: Omit; } -class LiveSyncBrowserAppLifecycleService< - T extends ServiceContext, -> extends InjectableAppLifecycleService {} +class LiveSyncBrowserAppLifecycleService extends InjectableAppLifecycleService {} class LiveSyncBrowserDatabaseService extends DatabaseService {} class LiveSyncBrowserKeyValueDBService extends KeyValueDBService {} -class LiveSyncBrowserConfigService extends ConfigService { - constructor( - context: T, - private readonly setting: ISettingService - ) { - super(context); - } - - getSmallConfig(key: string): string | null { - return this.setting.getSmallConfig(key); - } - - setSmallConfig(key: string, value: string): void { - this.setting.setSmallConfig(key, value); - } - - deleteSmallConfig(key: string): void { - this.setting.deleteSmallConfig(key); - } -} - /** LiveSync-owned service composition shared by WebApp and WebPeer. */ export class LiveSyncBrowserServiceHub extends InjectableServiceHub { constructor(options: LiveSyncBrowserServiceHubOptions) { @@ -98,17 +72,14 @@ export class LiveSyncBrowserServiceHub extends Injecta const setting = new InjectableSettingService(context, { APIService: API, onDisplayLanguageChanged: setLang, + deviceValues: options.deviceValues, }); const settingsPersistence = options.settings; setting.loadData.setHandler( - settingsPersistence - ? () => settingsPersistence.load() - : () => Promise.resolve(undefined) + settingsPersistence ? () => settingsPersistence.load() : () => Promise.resolve(undefined) ); setting.saveData.setHandler( - settingsPersistence - ? (settings) => settingsPersistence.save(settings) - : () => Promise.resolve() + settingsPersistence ? (settings) => settingsPersistence.save(settings) : () => Promise.resolve() ); const appLifecycle = new LiveSyncBrowserAppLifecycleService(context, { @@ -142,7 +113,7 @@ export class LiveSyncBrowserServiceHub extends Injecta setting, API, }); - const config = new LiveSyncBrowserConfigService(context, setting); + const config = setting.config; const replicator = new InjectableReplicatorService(context, { settingService: setting, appLifecycleService: appLifecycle, @@ -163,8 +134,7 @@ export class LiveSyncBrowserServiceHub extends Injecta databaseService: database, }); const keyValueDB = new LiveSyncBrowserKeyValueDBService(context, { - openKeyValueDatabase: - options.openKeyValueDatabase ?? createBrowserKeyValueDatabaseFactory(), + openKeyValueDatabase: options.openKeyValueDatabase ?? createBrowserKeyValueDatabaseFactory(), appLifecycle, databaseEvents, vault, @@ -209,7 +179,7 @@ export class LiveSyncBrowserServiceHub extends Injecta } export function createLiveSyncBrowserServiceHub( - options: LiveSyncBrowserServiceHubOptions = {} + options: LiveSyncBrowserServiceHubOptions ): LiveSyncBrowserServiceHub { return new LiveSyncBrowserServiceHub(options); } diff --git a/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts b/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts index 06f90938..558f4fb9 100644 --- a/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts +++ b/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts @@ -1,5 +1,6 @@ import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { MemoryValueStore } from "@vrtmrz/value-store"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -21,7 +22,8 @@ describe("LiveSync browser service context contract", () => { const context = createServiceContext({ translate: (key) => `webapp:${key}`, }); - const hub = createLiveSyncBrowserServiceHub({ context }); + const deviceValues = new MemoryValueStore(); + const hub = createLiveSyncBrowserServiceHub({ context, deviceValues }); expect(observeServiceContext(context, "moduleLocalDatabase.logWaitingForReady")).toEqual({ translation: "webapp:moduleLocalDatabase.logWaitingForReady", @@ -32,20 +34,17 @@ describe("LiveSync browser service context contract", () => { expect(SERVICE_CONTEXT_MEMBERS.filter((member) => !composition.servicesUsingExpectedContext[member])).toEqual( [] ); + expect(hub.setting.deviceValues).toBe(deviceValues); + expect(hub.config).toBe(hub.setting.config); }); it("binds browser identity, persistence, restart policy, and native Fetch while composing the hub", async () => { - const deviceLocalSettings = new Map(); - vi.stubGlobal("localStorage", { - getItem: (key: string) => deviceLocalSettings.get(key) ?? null, - setItem: (key: string, value: string) => deviceLocalSettings.set(key, value), - removeItem: (key: string) => deviceLocalSettings.delete(key), - clear: () => deviceLocalSettings.clear(), - }); + const deviceValues = new MemoryValueStore(); const savedSettings: (typeof DEFAULT_SETTINGS)[] = []; const restartCalls: string[] = []; const nativeFetch = vi.fn(async () => new Response("ok")); const options: LiveSyncBrowserServiceHubOptions> = { + deviceValues, getSystemVaultName: () => "browser-vault", settings: { load: async () => ({ @@ -69,6 +68,8 @@ describe("LiveSync browser service context contract", () => { const hub = createLiveSyncBrowserServiceHub(options); + expect(hub.setting.deviceValues).toBe(deviceValues); + expect(hub.config).toBe(hub.setting.config); expect(hub.API.getSystemVaultName()).toBe("browser-vault"); expect(hub.API.getPlatform()).toBe("browser"); const response = await hub.API.nativeFetch("https://example.invalid/"); @@ -86,4 +87,30 @@ describe("LiveSync browser service context contract", () => { expect(hub.appLifecycle.isReloadingScheduled()).toBe(true); expect(restartCalls).toEqual(["schedule", "perform", "ask:restart now"]); }); + + it("keeps each browser composition bound to its own device value store", async () => { + const firstValues = new MemoryValueStore(); + const secondValues = new MemoryValueStore(); + const firstHub = createLiveSyncBrowserServiceHub({ + deviceValues: firstValues, + getSystemVaultName: () => "first-vault", + }); + const secondHub = createLiveSyncBrowserServiceHub({ + deviceValues: secondValues, + getSystemVaultName: () => "second-vault", + }); + + await firstHub.config.setValue("probe", "first"); + await secondHub.config.setValue("probe", "second"); + + expect(firstHub.setting.deviceValues).toBe(firstValues); + expect(secondHub.setting.deviceValues).toBe(secondValues); + expect(firstHub.config).toBe(firstHub.setting.config); + expect(secondHub.config).toBe(secondHub.setting.config); + expect(firstHub.config).not.toBe(secondHub.config); + await expect(firstHub.config.getValue("probe")).resolves.toBe("first"); + await expect(secondHub.config.getValue("probe")).resolves.toBe("second"); + await expect(firstValues.getValue("first-vault-probe")).resolves.toBe("first"); + await expect(secondValues.getValue("second-vault-probe")).resolves.toBe("second"); + }); }); diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 17674fc3..fb2a7827 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -719,8 +719,8 @@ src/apps/cli/ │ └── ServiceFileAccessImpl.ts ├── services/ │ ├── NodeKeyValueDBService.ts -│ ├── NodeServiceHub.ts -│ └── NodeSettingService.ts +│ ├── NodeLocalStorage.ts +│ └── NodeServiceHub.ts ├── test/ │ ├── test-e2e-two-vaults-common.sh │ ├── test-e2e-two-vaults-matrix.sh diff --git a/src/apps/cli/main.bootstrap.unit.spec.ts b/src/apps/cli/main.bootstrap.unit.spec.ts index 3fcc3913..033771a1 100644 --- a/src/apps/cli/main.bootstrap.unit.spec.ts +++ b/src/apps/cli/main.bootstrap.unit.spec.ts @@ -88,6 +88,27 @@ describe("CLI database preparation", () => { await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`); } + it("loads legacy device values and persists configuration through the service-owned store", async () => { + const localStoragePath = join(directory, ".livesync", "runtime", "local-storage.json"); + await mkdir(join(directory, ".livesync", "runtime"), { recursive: true }); + await writeFile(localStoragePath, JSON.stringify({ ["__proto__"]: "legacy value" })); + let legacyValue: string | null | undefined; + const runner = vi.fn(async (_options, { core }) => { + legacyValue = await core.services.setting.deviceValues.getValue("__proto__"); + expect(core.services.config).toBe(core.services.setting.config); + await core.services.config.setValue("bootstrap-probe", "config value"); + return false; + }); + + await start("ls", runner); + + expect(legacyValue).toBe("legacy value"); + const stored = JSON.parse(await readFile(localStoragePath, "utf-8")) as Record; + expect(Object.prototype.hasOwnProperty.call(stored, "__proto__")).toBe(true); + expect(stored["__proto__"]).toBe("legacy value"); + expect(Object.values(stored)).toContain("config value"); + }); + it.each([REMOTE_P2P, REMOTE_COUCHDB])( "restores configured P2P automation before the host connects (%s)", async (remoteType) => { diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 028bd465..cea0ec35 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -1,6 +1,5 @@ import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation"; import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub"; -import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage"; import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore"; import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules"; import { @@ -78,7 +77,6 @@ const VAULT_SYNC_MODES: Readonly>> }, }; -ensureGlobalNodeLocalStorage(); defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG; /** Injectable command boundary used by CLI integration probes. */ @@ -400,8 +398,6 @@ export async function main( const settingsPath = options.settingsPath ? path.resolve(options.settingsPath) : path.join(databasePath, SETTINGS_FILE); - configureNodeLocalStorage(path.join(databasePath, ".livesync", "runtime", "local-storage.json")); - // Resolve vault path: mirror positional argument takes priority, // then --vault flag, otherwise fall back to databasePath. const vaultPath = diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 538c0d99..367e9fce 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -12,7 +12,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", diff --git a/src/apps/cli/services/NodeLocalStorage.ts b/src/apps/cli/services/NodeLocalStorage.ts index d8149c38..c6997c60 100644 --- a/src/apps/cli/services/NodeLocalStorage.ts +++ b/src/apps/cli/services/NodeLocalStorage.ts @@ -1,115 +1,86 @@ +import { createDeletableValueStore, sanitiseValueStoreError, ValueStoreError } from "@vrtmrz/value-store"; +import type { ValueDeleter, ValueStore } from "@vrtmrz/value-store"; import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node"; -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; -type LocalStorageShape = { - getItem(key: string): string | null; - setItem(key: string, value: string): void; - removeItem(key: string): void; - clear(): void; -}; +type NodeLocalStorage = ValueStore & ValueDeleter; -class PersistentNodeLocalStorage { - private storagePath: string | undefined; - private localStore: Record = {}; +class JsonFileValueBackend { + private values: Map; - configure(storagePath: string) { - if (this.storagePath === storagePath) { - return; - } - this.storagePath = storagePath; - this.loadFromFile(); + constructor(private readonly storagePath: string) { + this.values = this.loadValues(); } - private loadFromFile() { - if (!this.storagePath) { - this.localStore = {}; + get(key: string): string | null { + return this.values.get(key) ?? null; + } + + set(key: string, value: string): void { + const nextValues = new Map(this.values); + nextValues.set(key, value); + this.writeValues(nextValues); + this.values = nextValues; + } + + delete(key: string): void { + if (!this.values.has(key)) { return; } + const nextValues = new Map(this.values); + nextValues.delete(key); + this.writeValues(nextValues); + this.values = nextValues; + } + + private loadValues(): Map { + let contents: string; try { - const loaded = JSON.parse(nodeFs.readFileSync(this.storagePath, "utf-8")) as Record; - this.localStore = { ...loaded }; + contents = nodeFs.readFileSync(this.storagePath, "utf-8"); + } catch (error) { + if (isMissingFileError(error)) { + return new Map(); + } + throw sanitiseValueStoreError(error); + } + + let document: unknown; + try { + document = JSON.parse(contents); } catch { - this.localStore = {}; + throw new ValueStoreError("invalid-data"); } + if (document === null || typeof document !== "object" || Array.isArray(document)) { + throw new ValueStoreError("invalid-data"); + } + + const values = new Map(); + for (const [key, value] of Object.entries(document)) { + if (typeof value !== "string") { + throw new ValueStoreError("invalid-data"); + } + values.set(key, value); + } + return values; } - private flushToFile() { - if (!this.storagePath) { - return; - } + private writeValues(values: ReadonlyMap): void { nodeFs.mkdirSync(nodePath.dirname(this.storagePath), { recursive: true }); - nodeFs.writeFileSync(this.storagePath, JSON.stringify(this.localStore, null, 2), "utf-8"); - } - - getItem(key: string): string | null { - return this.localStore[key] ?? null; - } - - setItem(key: string, value: string) { - this.localStore[key] = value; - this.flushToFile(); - } - - removeItem(key: string) { - if (!(key in this.localStore)) { - return; - } - delete this.localStore[key]; - this.flushToFile(); - } - - clear() { - this.localStore = {}; - this.flushToFile(); + nodeFs.writeFileSync(this.storagePath, JSON.stringify(Object.fromEntries(values), null, 2), "utf-8"); } } -const persistentNodeLocalStorage = new PersistentNodeLocalStorage(); - -function createNodeLocalStorageShim(): LocalStorageShape { - return { - getItem(key: string) { - return persistentNodeLocalStorage.getItem(key); - }, - setItem(key: string, value: string) { - persistentNodeLocalStorage.setItem(key, value); - }, - removeItem(key: string) { - persistentNodeLocalStorage.removeItem(key); - }, - clear() { - persistentNodeLocalStorage.clear(); - }, - }; -} - -export function ensureGlobalNodeLocalStorage() { - const g = compatGlobal as unknown as Record; - if ( - !("localStorage" in g) || - typeof (g.localStorage as Record | undefined)?.getItem !== "function" - ) { - g.localStorage = createNodeLocalStorageShim(); +function isMissingFileError(error: unknown): boolean { + if (typeof error !== "object" || error === null) { + return false; + } + try { + const code = Object.getOwnPropertyDescriptor(error, "code"); + return code !== undefined && Object.prototype.hasOwnProperty.call(code, "value") && code.value === "ENOENT"; + } catch { + return false; } } -export function configureNodeLocalStorage(storagePath: string) { - persistentNodeLocalStorage.configure(storagePath); - ensureGlobalNodeLocalStorage(); -} - -export function getNodeLocalStorageItem(key: string): string { - return persistentNodeLocalStorage.getItem(key) ?? ""; -} - -export function setNodeLocalStorageItem(key: string, value: string) { - persistentNodeLocalStorage.setItem(key, value); -} - -export function deleteNodeLocalStorageItem(key: string) { - persistentNodeLocalStorage.removeItem(key); -} - -export function clearNodeLocalStorage() { - persistentNodeLocalStorage.clear(); +export function createNodeLocalStorage(storagePath: string): NodeLocalStorage { + return createDeletableValueStore(new JsonFileValueBackend(storagePath)); } diff --git a/src/apps/cli/services/NodeLocalStorage.unit.spec.ts b/src/apps/cli/services/NodeLocalStorage.unit.spec.ts index fde5aae4..db35b755 100644 --- a/src/apps/cli/services/NodeLocalStorage.unit.spec.ts +++ b/src/apps/cli/services/NodeLocalStorage.unit.spec.ts @@ -1,58 +1,127 @@ import { fs, os, path } from "@vrtmrz/livesync-commonlib/node"; +import { ValueStoreError } from "@vrtmrz/value-store"; import { afterEach, describe, expect, it } from "vitest"; -import { - clearNodeLocalStorage, - configureNodeLocalStorage, - ensureGlobalNodeLocalStorage, - getNodeLocalStorageItem, - setNodeLocalStorageItem, -} from "./NodeLocalStorage"; +import { createNodeLocalStorage } from "./NodeLocalStorage"; describe("NodeLocalStorage", () => { const tempDirs: string[] = []; afterEach(() => { - clearNodeLocalStorage(); for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true, force: true }); } }); - it("persists values to the configured file", () => { - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); - tempDirs.push(tempDir); - const storagePath = path.join(tempDir, "runtime", "local-storage.json"); - - configureNodeLocalStorage(storagePath); - setNodeLocalStorageItem("checkpoint", "42"); - - const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record; - expect(saved.checkpoint).toBe("42"); - }); - - it("reloads persisted values when configured again", () => { + it("loads and preserves legacy flat string data, including empty values and exact keys", async () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); tempDirs.push(tempDir); const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const specialKey = "__proto__"; + const specialValue = "line one\n雪"; fs.mkdirSync(path.dirname(storagePath), { recursive: true }); - fs.writeFileSync(storagePath, JSON.stringify({ persisted: "value" }, null, 2), "utf-8"); + fs.writeFileSync(storagePath, JSON.stringify({ [specialKey]: specialValue, empty: "" }), "utf-8"); - configureNodeLocalStorage(storagePath); + const store = createNodeLocalStorage(storagePath); - expect(getNodeLocalStorageItem("persisted")).toBe("value"); + await expect(store.getValue(specialKey)).resolves.toBe(specialValue); + await expect(store.getValue("empty", "default")).resolves.toBe(""); + const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record; + expect(saved[specialKey]).toBe(specialValue); + expect(saved.empty).toBe(""); }); - it("installs a global localStorage shim backed by the same store", () => { + it("persists strings and objects, reloads them, and deletes keys", async () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); tempDirs.push(tempDir); const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const store = createNodeLocalStorage(storagePath); + const object = { enabled: false, entries: ["value", null, 0] }; - configureNodeLocalStorage(storagePath); - ensureGlobalNodeLocalStorage(); + await store.setValue("empty", ""); + await store.setValue("text", "stored value"); + await store.setObject("object", object); - globalThis.localStorage.setItem("shared", "state"); + const reloaded = createNodeLocalStorage(storagePath); + await expect(reloaded.getValue("empty", "default")).resolves.toBe(""); + await expect(reloaded.getValue("text")).resolves.toBe("stored value"); + await expect(reloaded.getObject("object")).resolves.toEqual(object); - expect(getNodeLocalStorageItem("shared")).toBe("state"); + await reloaded.delete("text"); + await expect(reloaded.getValue("text")).resolves.toBeNull(); + const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record; + expect(saved.text).toBeUndefined(); + expect(saved.object).toBe(JSON.stringify(object)); + }); + + it("treats a missing file as empty and creates its parent on the first write", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const store = createNodeLocalStorage(storagePath); + + await expect(store.getValue("absent")).resolves.toBeNull(); + expect(fs.existsSync(storagePath)).toBe(false); + await store.setValue("created", "value"); + expect(fs.existsSync(storagePath)).toBe(true); + }); + + it("keeps separate store instances isolated by their file paths", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const first = createNodeLocalStorage(path.join(tempDir, "first", "local-storage.json")); + const second = createNodeLocalStorage(path.join(tempDir, "second", "local-storage.json")); + + await first.setValue("shared-key", "first"); + await second.setValue("shared-key", "second"); + + await expect(first.getValue("shared-key")).resolves.toBe("first"); + await expect(second.getValue("shared-key")).resolves.toBe("second"); + }); + + it("surfaces malformed and unreadable files as sanitised value-store errors", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const malformedPath = path.join(tempDir, "malformed.json"); + fs.writeFileSync(malformedPath, "{ malformed", "utf-8"); + + let malformedError: unknown; + try { + createNodeLocalStorage(malformedPath); + } catch (error) { + malformedError = error; + } + expect(malformedError).toBeInstanceOf(ValueStoreError); + expect(malformedError).toMatchObject({ code: "invalid-data" }); + expect(fs.readFileSync(malformedPath, "utf-8")).toBe("{ malformed"); + + const unreadablePath = path.join(tempDir, "unreadable.json"); + fs.mkdirSync(unreadablePath); + expect(() => createNodeLocalStorage(unreadablePath)).toThrowError(ValueStoreError); + }); + + it("does not change its visible values when a write fails", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const store = createNodeLocalStorage(storagePath); + await store.setValue("retained", "before failure"); + + fs.rmSync(storagePath); + fs.mkdirSync(storagePath); + + await expect(store.setValue("new", "write fails")).rejects.toBeInstanceOf(ValueStoreError); + await expect(store.getValue("retained")).resolves.toBe("before failure"); + await expect(store.getValue("new")).resolves.toBeNull(); + }); + + it("does not install or modify a global localStorage value", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const before = Object.getOwnPropertyDescriptor(globalThis, "localStorage"); + + createNodeLocalStorage(path.join(tempDir, "runtime", "local-storage.json")); + + expect(Object.getOwnPropertyDescriptor(globalThis, "localStorage")).toEqual(before); }); }); diff --git a/src/apps/cli/services/NodeServiceHub.ts b/src/apps/cli/services/NodeServiceHub.ts index bed9cb16..484861c2 100644 --- a/src/apps/cli/services/NodeServiceHub.ts +++ b/src/apps/cli/services/NodeServiceHub.ts @@ -1,6 +1,5 @@ import type { AppLifecycleServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService"; import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; -import { ConfigServiceBrowserCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/browser/ConfigServiceBrowserCompat"; import type { ComponentHasResult, SvelteDialogManager, @@ -11,6 +10,7 @@ import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService"; import { InjectableDatabaseEventService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableDatabaseEventService"; import { InjectableFileProcessingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableFileProcessingService"; +import { InjectableSettingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableSettingService"; import { PathServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectablePathService"; import { InjectableRemoteService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableRemoteService"; import { InjectableReplicationService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableReplicationService"; @@ -21,7 +21,7 @@ import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/ import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import { HeadlessAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/headless/HeadlessAPIService"; import { NodeKeyValueDBService } from "./NodeKeyValueDBService"; -import { NodeSettingService } from "./NodeSettingService"; +import { createNodeLocalStorage } from "./NodeLocalStorage"; import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { path as nodePath } from "@vrtmrz/livesync-commonlib/node"; @@ -96,11 +96,11 @@ export class NodeServiceHub extends InjectableServ const conflict = new InjectableConflictService(context); const fileProcessing = new InjectableFileProcessingService(context); - const setting = new NodeSettingService( - context, - { APIService: API, onDisplayLanguageChanged: setLang }, - localStoragePath - ); + const setting = new InjectableSettingService(context, { + APIService: API, + onDisplayLanguageChanged: setLang, + deviceValues: createNodeLocalStorage(localStoragePath), + }); const appLifecycle = new NodeAppLifecycleService(context, { settingService: setting, @@ -132,10 +132,7 @@ export class NodeServiceHub extends InjectableServ setting, }); - const config = new ConfigServiceBrowserCompat(context, { - settingService: setting, - APIService: API, - }); + const config = setting.config; const replicator = new InjectableReplicatorService(context, { settingService: setting, diff --git a/src/apps/cli/services/NodeSettingService.ts b/src/apps/cli/services/NodeSettingService.ts deleted file mode 100644 index 771cb41c..00000000 --- a/src/apps/cli/services/NodeSettingService.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { EVENT_SETTING_SAVED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; -import { EVENT_REQUEST_RELOAD_SETTING_TAB } from "@/common/events"; -import { handlers } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; -import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; -import { SettingService, type SettingServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService"; -import { - configureNodeLocalStorage, - deleteNodeLocalStorageItem, - getNodeLocalStorageItem, - setNodeLocalStorageItem, -} from "./NodeLocalStorage"; - -export class NodeSettingService extends SettingService { - constructor(context: T, dependencies: SettingServiceDependencies, storagePath: string) { - super(context, dependencies); - configureNodeLocalStorage(storagePath); - this.onSettingSaved.addHandler((settings) => { - this.context.events.emitEvent(EVENT_SETTING_SAVED, settings); - return Promise.resolve(true); - }); - this.onSettingLoaded.addHandler((settings) => { - this.context.events.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB); - return Promise.resolve(true); - }); - } - - protected setItem(key: string, value: string) { - setNodeLocalStorageItem(key, value); - } - - protected getItem(key: string): string { - return getNodeLocalStorageItem(key); - } - - protected deleteItem(key: string): void { - deleteNodeLocalStorageItem(key); - } - - public saveData = handlers<{ saveData: (data: ObsidianLiveSyncSettings) => Promise }>().binder("saveData"); - public loadData = handlers<{ loadData: () => Promise }>().binder("loadData"); -} diff --git a/src/apps/webapp/WebAppRuntime.ts b/src/apps/webapp/WebAppRuntime.ts index 48405a4c..fbee662b 100644 --- a/src/apps/webapp/WebAppRuntime.ts +++ b/src/apps/webapp/WebAppRuntime.ts @@ -21,6 +21,8 @@ import { useCheckRemoteSize } from "@vrtmrz/livesync-commonlib/compat/serviceFea import { useRemoteConfiguration } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/remoteConfig"; import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature"; import type { UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult"; +import type { ValueStore, ValueDeleter } from "@vrtmrz/value-store"; +import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { createLiveSyncBrowserServiceHub, @@ -56,6 +58,7 @@ const DEFAULT_SETTINGS: Partial = { export type WebAppRuntimeStatusKind = "info" | "warning" | "error" | "success"; export interface WebAppRuntimeOptions { + deviceValues?: ValueStore & ValueDeleter; reportStatus?: (kind: WebAppRuntimeStatusKind, message: string) => void; scheduleReload?: (delayMilliseconds: number) => void; } @@ -71,7 +74,10 @@ export class WebAppRuntime { private paneHost: P2PReplicatorPaneHost | null = null; private restartScheduled = false; - constructor(rootHandle: FileSystemDirectoryHandle, options: WebAppRuntimeOptions = {}) { + constructor( + rootHandle: FileSystemDirectoryHandle, + private readonly options: WebAppRuntimeOptions = {} + ) { this.rootHandle = rootHandle; this.reportStatus = options.reportStatus ?? (() => {}); this.scheduleReload = @@ -89,6 +95,8 @@ export class WebAppRuntime { private createServiceOptions(): LiveSyncBrowserServiceHubOptions { return { + deviceValues: + this.options.deviceValues ?? createWebStorageValueStore(compatGlobal.localStorage, { prefix: "" }), getSystemVaultName: () => this.rootHandle.name || "livesync-webapp", settings: { save: async (data) => { @@ -219,7 +227,9 @@ export class WebAppRuntime { useCheckRemoteSize(core); useRemoteConfiguration(core); this.p2p = useP2PReplicatorFeature(core, undefined, undefined, { - prepareP2PSettings: useP2PSettingsPreparation(core.services.API.webCompatFetch.bind(core.services.API)), + prepareP2PSettings: useP2PSettingsPreparation( + core.services.API.webCompatFetch.bind(core.services.API) + ), }); this.paneHost = { services: core.services, diff --git a/src/apps/webpeer/src/WebPeerRuntime.ts b/src/apps/webpeer/src/WebPeerRuntime.ts index b4f69758..fd29947a 100644 --- a/src/apps/webpeer/src/WebPeerRuntime.ts +++ b/src/apps/webpeer/src/WebPeerRuntime.ts @@ -7,6 +7,8 @@ import { P2PLogCollector } from "@vrtmrz/livesync-commonlib/compat/replication/t import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature"; import { ServiceContext, type LiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; import type { P2PServiceViews } from "@vrtmrz/livesync-commonlib/p2p"; +import type { ValueStore, ValueDeleter } from "@vrtmrz/value-store"; +import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage"; import { unique } from "octagonal-wheels/collection"; import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase"; @@ -20,6 +22,7 @@ import { translateLiveSyncMessage } from "@/common/translation"; import { WEBPEER_STORE_NAME, createWebPeerPersistence } from "./WebPeerPersistence"; export interface WebPeerRuntimeOptions { + deviceValues?: ValueStore & ValueDeleter; context?: ServiceContext; store?: SimpleStore; deviceName?: string; @@ -62,6 +65,7 @@ export class WebPeerRuntime { this.context = options.context ?? new ServiceContext({ translate: translateLiveSyncMessage }); this.services = createLiveSyncBrowserServiceHub({ context: this.context, + deviceValues: options.deviceValues ?? createWebStorageValueStore(compatGlobal.localStorage, { prefix: "" }), getSystemVaultName: () => options.systemVaultName ?? WEBPEER_STORE_NAME, settings: persistence.settings, restart: { @@ -103,7 +107,7 @@ export class WebPeerRuntime { await this.services.setting.loadSettings(); const deviceName = this.options.deviceName?.trim(); if (deviceName) { - this.services.config.setSmallConfig(SETTING_KEY_P2P_DEVICE_NAME, deviceName); + await this.services.config.setValue(SETTING_KEY_P2P_DEVICE_NAME, deviceName); } const opened = await this.services.database.openDatabase({ replicator: this.services.replicator, @@ -192,7 +196,7 @@ export class WebPeerRuntime { ); } - getDeviceName(): string { - return this.services.config.getSmallConfig(SETTING_KEY_P2P_DEVICE_NAME) || this.services.vault.getVaultName(); + async getDeviceName(): Promise { + return (await this.services.config.getValue(SETTING_KEY_P2P_DEVICE_NAME)) || this.services.vault.getVaultName(); } } diff --git a/src/common/replicatorProviders.ts b/src/common/replicatorProviders.ts index 845d8c2e..f3cc40b0 100644 --- a/src/common/replicatorProviders.ts +++ b/src/common/replicatorProviders.ts @@ -28,6 +28,8 @@ import { getObjectStorageReplicatorConfigurationIdentity, } from "./replicatorConfigurationIdentity"; import { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, createCouchDBConnectionProbeFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, @@ -115,6 +117,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getCouchDBReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncCouchDBReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createCouchDBJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability(createCouchDBConnectionProbeFactory(host)), [REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK]: supportedCapability( createCouchDBPreferredTweakProbeFactory(host) @@ -141,6 +146,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getObjectStorageReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncJournalReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createObjectStorageJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability( createObjectStorageConnectionProbeFactory(host) ), diff --git a/src/common/replicatorProviders.unit.spec.ts b/src/common/replicatorProviders.unit.spec.ts index 54a2c441..97371072 100644 --- a/src/common/replicatorProviders.unit.spec.ts +++ b/src/common/replicatorProviders.unit.spec.ts @@ -44,7 +44,16 @@ describe("central Replicator provider definitions", () => { it("keeps the retained remote-resource catalogue bounded", () => { expect .soft(Object.values(REMOTE_RESOURCE_KINDS).sort()) - .toEqual(["connection", "preferred-tweak", "security-seed", "synchronisation-information"].sort()); + .toEqual( + [ + "connection", + "initial-fetch-fast-forward", + "joined-devices", + "preferred-tweak", + "security-seed", + "synchronisation-information", + ].sort() + ); }); it("lets Journal prepare its own fresh Security Seed while CouchDB uses central preparation", () => { @@ -106,8 +115,15 @@ describe("central Replicator provider definitions", () => { const couchAdministration = definitions.get(REMOTE_COUCHDB)?.centralRemoteAdministration; const objectAdministration = definitions.get(REMOTE_MINIO)?.centralRemoteAdministration; - expect(Object.keys(couchResources ?? {}).sort()).toEqual(Object.values(REMOTE_RESOURCE_KINDS).sort()); - expect(Object.keys(objectResources ?? {}).sort()).toEqual(Object.values(REMOTE_RESOURCE_KINDS).sort()); + const requiredResources = Object.values(REMOTE_RESOURCE_KINDS).filter( + (kind) => kind !== REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD + ); + expect(Object.keys(couchResources ?? {}).sort()).toEqual(requiredResources.sort()); + expect(Object.keys(objectResources ?? {}).sort()).toEqual(requiredResources.sort()); + expect(couchResources?.[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); + expect(objectResources?.[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); + expect(couchResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]).toBeUndefined(); + expect(objectResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]).toBeUndefined(); expect(couchResources?.[REMOTE_RESOURCE_KINDS.CONNECTION].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(couchResources?.[REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(couchResources?.[REMOTE_RESOURCE_KINDS.SECURITY_SEED].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); diff --git a/src/common/replicatorResources.unit.spec.ts b/src/common/replicatorResources.unit.spec.ts index 2215a48a..0dffd7d0 100644 --- a/src/common/replicatorResources.unit.spec.ts +++ b/src/common/replicatorResources.unit.spec.ts @@ -12,6 +12,8 @@ const mocks = vi.hoisted(() => ({ getRemoteStatus: ReturnType; getRemotePreferredTweakValues: ReturnType; getReplicationPBKDF2Salt: ReturnType; + getJoinedDevices: ReturnType; + forgetJoinedDevice: ReturnType; closeReplication: ReturnType; }>, objectStorage: [] as Array<{ @@ -20,6 +22,8 @@ const mocks = vi.hoisted(() => ({ getRemoteStatus: ReturnType; getRemotePreferredTweakValues: ReturnType; getReplicationPBKDF2Salt: ReturnType; + getJoinedDevices: ReturnType; + forgetJoinedDevice: ReturnType; closeReplication: ReturnType; }>, checkSyncInfo: vi.fn(async () => true), @@ -42,6 +46,8 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicato getRemoteStatus = vi.fn(); getRemotePreferredTweakValues = vi.fn(); getReplicationPBKDF2Salt = vi.fn(); + getJoinedDevices = vi.fn(); + forgetJoinedDevice = vi.fn(); closeReplication = vi.fn(); constructor(host: unknown) { @@ -58,6 +64,8 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe getRemoteStatus = vi.fn(); getRemotePreferredTweakValues = vi.fn(); getReplicationPBKDF2Salt = vi.fn(); + getJoinedDevices = vi.fn(); + forgetJoinedDevice = vi.fn(); closeReplication = vi.fn(); constructor(host: unknown) { @@ -69,10 +77,12 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe import { createCouchDBConnectionProbeFactory, + createCouchDBJoinedDevicesResourceFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, createCouchDBSynchronisationInformationResourceFactory, createObjectStorageConnectionProbeFactory, + createObjectStorageJoinedDevicesResourceFactory, createObjectStoragePreferredTweakProbeFactory, createObjectStorageSecuritySeedResourceFactory, } from "./replicatorResources"; @@ -226,6 +236,65 @@ describe("replicator probe factories", () => { expect(preferredReplicator.getRemotePreferredTweakValues).toHaveBeenCalledWith(snapshot); }); + it.each([ + { + kind: "CouchDB", + remoteType: REMOTE_COUCHDB, + createFactory: createCouchDBJoinedDevicesResourceFactory, + instances: () => mocks.couchDB, + }, + { + kind: "Object Storage", + remoteType: REMOTE_MINIO, + createFactory: createObjectStorageJoinedDevicesResourceFactory, + instances: () => mocks.objectStorage, + }, + ])( + "keeps $kind joined-device operations on an owned settings snapshot", + async ({ remoteType, createFactory, instances }) => { + const host = { name: "host" }; + const source = createSettings({ remoteType }); + const snapshot = { ...source }; + const factory = createFactory(host as never); + const resource = await factory(source); + const nextResource = await factory(source); + const [replicator, nextReplicator] = instances(); + expect(instances()).toHaveLength(2); + expect(replicator.host).toBe(host); + expect(replicator.getJoinedDevices).not.toHaveBeenCalled(); + expect(replicator.forgetJoinedDevice).not.toHaveBeenCalled(); + const devices = { + milestonePresent: true, + locked: false, + cleaned: false, + devices: [{ id: "other", accepted: true }], + }; + replicator.getJoinedDevices.mockResolvedValue(devices); + replicator.forgetJoinedDevice.mockResolvedValue("conflict"); + + source.couchDB_DBNAME = "changed-vault"; + source.bucket = "changed-bucket"; + await expect(resource.read()).resolves.toBe(devices); + await expect(resource.forget("other")).resolves.toBe("conflict"); + expect(replicator.getJoinedDevices).toHaveBeenCalledWith(snapshot); + expect(replicator.getJoinedDevices.mock.calls[0][0]).not.toBe(source); + expect(replicator.forgetJoinedDevice).toHaveBeenCalledWith(snapshot, "other"); + + const failure = new Error("remote unavailable"); + replicator.getJoinedDevices.mockRejectedValue(failure); + await expect(resource.read()).rejects.toBe(failure); + replicator.forgetJoinedDevice.mockRejectedValue(failure); + await expect(resource.forget("other")).rejects.toBe(failure); + const disposal = resource.dispose(); + expect(resource.dispose()).toBe(disposal); + await disposal; + expect(replicator.closeReplication).toHaveBeenCalledOnce(); + expect(nextReplicator.closeReplication).not.toHaveBeenCalled(); + await nextResource.dispose(); + expect(nextReplicator.closeReplication).toHaveBeenCalledOnce(); + } + ); + it("shares one successful asynchronous disposal promise for every probe kind", async () => { const couchProbe = await createCouchDBPreferredTweakProbeFactory({} as never)(createSettings()); const objectProbe = await createObjectStoragePreferredTweakProbeFactory({} as never)( diff --git a/src/common/replicatorResources/index.ts b/src/common/replicatorResources/index.ts index 9888e85c..f2f359f1 100644 --- a/src/common/replicatorResources/index.ts +++ b/src/common/replicatorResources/index.ts @@ -14,3 +14,8 @@ export { type SecuritySeedResourceHost, } from "./securitySeed"; export { createCouchDBSynchronisationInformationResourceFactory } from "./synchronisationInformation"; + +export { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, +} from "./joinedDevices"; diff --git a/src/common/replicatorResources/joinedDevices.ts b/src/common/replicatorResources/joinedDevices.ts new file mode 100644 index 00000000..a9acf1c1 --- /dev/null +++ b/src/common/replicatorResources/joinedDevices.ts @@ -0,0 +1,40 @@ +import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { + JoinedDevicesResource, + JoinedDevicesResourceFactory, + JoinedDevicesSnapshot, + ForgetJoinedDeviceResult, +} from "@vrtmrz/livesync-commonlib/replication"; +import { + LiveSyncCouchDBReplicator, + type LiveSyncCouchDBReplicatorEnv, +} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; +import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator"; +import { createReplicatorDisposer, snapshotRemoteSettings, type ResourceReplicator } from "./shared"; + +interface JoinedDevicesReplicator extends ResourceReplicator { + getJoinedDevices(setting: RemoteDBSettings): Promise; + forgetJoinedDevice(setting: RemoteDBSettings, deviceId: string): Promise; +} + +function createResource(replicator: JoinedDevicesReplicator, snapshot: RemoteDBSettings): JoinedDevicesResource { + return { + read: () => replicator.getJoinedDevices(snapshot), + forget: (deviceId) => replicator.forgetJoinedDevice(snapshot, deviceId), + dispose: createReplicatorDisposer(replicator), + }; +} + +export function createCouchDBJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncCouchDBReplicator(host), snapshotRemoteSettings(setting))); +} + +export function createObjectStorageJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncJournalReplicator(host), snapshotRemoteSettings(setting))); +} diff --git a/src/common/turnSettingsPrivacy.unit.spec.ts b/src/common/turnSettingsPrivacy.unit.spec.ts index c0243c79..451db93b 100644 --- a/src/common/turnSettingsPrivacy.unit.spec.ts +++ b/src/common/turnSettingsPrivacy.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { describe, expect, it } from "vitest"; import { DEFAULT_SETTINGS, @@ -18,17 +19,7 @@ import { } from "./turnSettingsPrivacy"; class MemorySettingService extends SettingService { - readonly items = new Map(); saved?: ObsidianLiveSyncSettings; - protected setItem(key: string, value: string) { - this.items.set(key, value); - } - protected getItem(key: string) { - return this.items.get(key) ?? ""; - } - protected deleteItem(key: string) { - this.items.delete(key); - } protected saveData(settings: ObsidianLiveSyncSettings) { this.saved = structuredClone(settings); return Promise.resolve(); @@ -70,6 +61,7 @@ describe("managed TURN settings privacy", () => { const originalURI = ConnectionStringParser.serialize({ type: "p2p", settings: current }); current.remoteConfigurations.managed.uri = originalURI; const service = new MemorySettingService(new ServiceContext(), { + deviceValues: new MemoryValueStore(), APIService: { getSystemVaultName: () => "test-vault", getAppID: () => "test-app", diff --git a/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte b/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte index f8bf13ab..39ad12fc 100644 --- a/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte +++ b/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte @@ -1,5 +1,6 @@
+ {#each Object.values(storageErrors) as storageError} +

{storageError}

+ {/each}

{_msg("Peer to Peer Replicator")}

{_msg("P2P.Note.Summary")} @@ -345,6 +440,7 @@ type="text" placeholder="iphone-16" bind:value={eDeviceName} + disabled={!deviceNameReady} autocomplete="off" /> @@ -383,7 +479,7 @@ --> - diff --git a/src/features/ReviewHarness/ReviewHarnessView.ts b/src/features/ReviewHarness/ReviewHarnessView.ts index b116e9cf..2d1ae0f0 100644 --- a/src/features/ReviewHarness/ReviewHarnessView.ts +++ b/src/features/ReviewHarness/ReviewHarnessView.ts @@ -1,4 +1,5 @@ import { ItemView, Notice, Setting, type WorkspaceLeaf } from "@/deps"; +import { LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger"; import { REVIEW_HARNESS_SCENARIOS, type ReviewHarnessScenarioId, @@ -101,7 +102,7 @@ export class ReviewHarnessView extends ItemView { true ); this.addActionButton(actions, "Restart and return", "review-harness-restart", () => - this.controller.prepareCompatibilityReviewRestart() + this.controller.prepareCompatibilityReviewRestart().catch((error) => Logger(error, LOG_LEVEL_NOTICE)) ); } } diff --git a/src/features/ReviewHarness/reviewHarnessController.ts b/src/features/ReviewHarness/reviewHarnessController.ts index 2bde2f58..1cec69aa 100644 --- a/src/features/ReviewHarness/reviewHarnessController.ts +++ b/src/features/ReviewHarness/reviewHarnessController.ts @@ -23,9 +23,9 @@ export interface ReviewHarnessRuntime { runVaultRoundTrip(): Promise; runIdBenchmark(): Promise; runFileContentHashBenchmark(): Promise; - readContinuation(): string | null; - writeContinuation(value: string): void; - deleteContinuation(): void; + readContinuation(): Promise; + writeContinuation(value: string): Promise; + deleteContinuation(): Promise; restart(): void; reportError(error: unknown): void; copyText(value: string): Promise; @@ -68,6 +68,7 @@ export class ReviewHarnessController { private current: ReviewHarnessScenarioId | null = null; private resumedRequestId: string | null = null; private continuationError: string | null = null; + private continuationConsumption: Promise | undefined; constructor(private readonly runtime: ReviewHarnessRuntime) {} @@ -102,10 +103,20 @@ export class ReviewHarnessController { }; } - consumeContinuation(): void { - const serialised = this.runtime.readContinuation(); + consumeContinuation(): Promise { + if (this.continuationConsumption) return this.continuationConsumption; + const consumption = this.consumeContinuationFromStore(); + const trackedConsumption = consumption.finally(() => { + if (this.continuationConsumption === trackedConsumption) this.continuationConsumption = undefined; + }); + this.continuationConsumption = trackedConsumption; + return trackedConsumption; + } + + private async consumeContinuationFromStore(): Promise { + const serialised = await this.runtime.readContinuation(); if (!serialised) return; - this.runtime.deleteContinuation(); + await this.runtime.deleteContinuation(); const parsed = parsePendingReviewRun(serialised); if (!parsed.pendingRun) { this.continuationError = parsed.error ?? "The Review Harness continuation was invalid."; @@ -232,7 +243,7 @@ export class ReviewHarnessController { this.record("scenario-failed", id); } - prepareCompatibilityReviewRestart(): void { + async prepareCompatibilityReviewRestart(): Promise { const requestedAt = this.runtime.now().toISOString(); const pending = { formatVersion: 1, @@ -241,7 +252,7 @@ export class ReviewHarnessController { stage: "awaiting-restart", requestedAt, } as const; - this.runtime.writeContinuation(JSON.stringify(pending)); + await this.runtime.writeContinuation(JSON.stringify(pending)); this.results["compatibility-review"] = { status: "waiting-for-user", detail: "Restart requested. The one-shot continuation will be removed before the review resumes.", diff --git a/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts index cce28869..2b00cd23 100644 --- a/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts +++ b/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts @@ -90,14 +90,14 @@ function createRuntime(): ReviewHarnessRuntime & { detail: "File content hash measurements completed.", observations: ["SHA-256, 4 KiB: median=1.00 ms; range=1.00–1.00 ms"], })), - readContinuation() { + async readContinuation() { return this.continuation; }, - writeContinuation(value) { + async writeContinuation(value) { this.events.push("write"); this.continuation = value; }, - deleteContinuation() { + async deleteContinuation() { this.events.push("delete"); this.continuation = null; }, @@ -251,7 +251,7 @@ describe("ReviewHarnessController", () => { expect(controller.snapshot().running).toBe(false); }); - it("deletes a one-shot continuation before exposing the resumed guided step", () => { + it("deletes a one-shot continuation before exposing the resumed guided step", async () => { const runtime = createRuntime(); runtime.continuation = JSON.stringify({ formatVersion: 1, @@ -262,7 +262,7 @@ describe("ReviewHarnessController", () => { }); const controller = new ReviewHarnessController(runtime); - controller.consumeContinuation(); + await controller.consumeContinuation(); expect(runtime.events).toEqual(["delete"]); expect(controller.snapshot().results["compatibility-review"]).toMatchObject({ @@ -271,7 +271,7 @@ describe("ReviewHarnessController", () => { expect(controller.snapshot().resumedRequestId).toBe("compatibility-review-2026-07-18T11:59:00.000Z"); }); - it("does not copy rejected continuation values into the report", () => { + it("does not copy rejected continuation values into the report", async () => { const runtime = createRuntime(); runtime.continuation = JSON.stringify({ formatVersion: 1, @@ -282,7 +282,7 @@ describe("ReviewHarnessController", () => { }); const controller = new ReviewHarnessController(runtime); - controller.consumeContinuation(); + await controller.consumeContinuation(); expect(controller.snapshot().continuationError).toContain("private-vault"); expect(controller.snapshot().results["compatibility-review"].detail).toBe( @@ -292,7 +292,7 @@ describe("ReviewHarnessController", () => { expect(controller.createReport()).not.toContain("secret-note.md"); }); - it("does not accept an injected request ID which could enter the report transcript", () => { + it("does not accept an injected request ID which could enter the report transcript", async () => { const runtime = createRuntime(); runtime.continuation = JSON.stringify({ formatVersion: 1, @@ -303,18 +303,18 @@ describe("ReviewHarnessController", () => { }); const controller = new ReviewHarnessController(runtime); - controller.consumeContinuation(); + await controller.consumeContinuation(); expect(controller.snapshot().results["compatibility-review"].status).toBe("failed"); expect(controller.createReport()).not.toContain("private-vault"); expect(controller.createReport()).not.toContain("secret-note.md"); }); - it("persists the fixed continuation before requesting restart", () => { + it("persists the fixed continuation before requesting restart", async () => { const runtime = createRuntime(); const controller = new ReviewHarnessController(runtime); - controller.prepareCompatibilityReviewRestart(); + await controller.prepareCompatibilityReviewRestart(); expect(runtime.events).toEqual(["write", "restart"]); expect(JSON.parse(runtime.continuation!)).toMatchObject({ @@ -324,6 +324,135 @@ describe("ReviewHarnessController", () => { }); }); + it("waits for the continuation save before requesting restart", async () => { + const runtime = createRuntime(); + let markSaveStarted!: () => void; + const saveStarted = new Promise((resolve) => { + markSaveStarted = resolve; + }); + let finishSave!: () => void; + const saving = new Promise((resolve) => { + finishSave = resolve; + }); + runtime.writeContinuation = vi.fn(async (value) => { + runtime.events.push("write-started"); + markSaveStarted(); + await saving; + runtime.continuation = value; + runtime.events.push("write-saved"); + }); + const controller = new ReviewHarnessController(runtime); + + const preparing = controller.prepareCompatibilityReviewRestart(); + await saveStarted; + + expect(runtime.events).toEqual(["write-started"]); + expect(runtime.restart).not.toHaveBeenCalled(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("idle"); + + finishSave(); + await preparing; + + expect(runtime.events).toEqual(["write-started", "write-saved", "restart"]); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + + it("keeps the guided review retryable when saving its continuation fails", async () => { + const runtime = createRuntime(); + const controller = new ReviewHarnessController(runtime); + await controller.runScenario("compatibility-review"); + runtime.writeContinuation = vi.fn(async () => { + runtime.events.push("write-failed"); + throw new Error("continuation save failed"); + }); + + await expect(controller.prepareCompatibilityReviewRestart()).rejects.toThrow("continuation save failed"); + + expect(runtime.events).toEqual(["write-failed"]); + expect(runtime.restart).not.toHaveBeenCalled(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + + it("waits for one continuation deletion before exposing it and coalesces concurrent consumption", async () => { + const runtime = createRuntime(); + runtime.continuation = JSON.stringify({ + formatVersion: 1, + requestId: "compatibility-review-2026-07-18T11:59:00.000Z", + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt: "2026-07-18T11:59:00.000Z", + }); + let markDeleteStarted!: () => void; + const deleteStarted = new Promise((resolve) => { + markDeleteStarted = resolve; + }); + let finishDelete!: () => void; + const deleting = new Promise((resolve) => { + finishDelete = resolve; + }); + runtime.readContinuation = vi.fn(async () => { + runtime.events.push("read"); + return runtime.continuation; + }); + runtime.deleteContinuation = vi.fn(async () => { + runtime.events.push("delete-started"); + markDeleteStarted(); + await deleting; + runtime.continuation = null; + runtime.events.push("delete-finished"); + }); + const controller = new ReviewHarnessController(runtime); + + const firstConsumption = controller.consumeContinuation(); + const duplicateConsumption = controller.consumeContinuation(); + await deleteStarted; + + expect(duplicateConsumption).toBe(firstConsumption); + expect(runtime.events).toEqual(["read", "delete-started"]); + expect(controller.snapshot().resumedRequestId).toBeNull(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("idle"); + + finishDelete(); + await firstConsumption; + + expect(runtime.events).toEqual(["read", "delete-started", "delete-finished"]); + expect(runtime.readContinuation).toHaveBeenCalledOnce(); + expect(runtime.deleteContinuation).toHaveBeenCalledOnce(); + expect(controller.snapshot().resumedRequestId).toBe("compatibility-review-2026-07-18T11:59:00.000Z"); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + + it("keeps a failed continuation deletion available for retry", async () => { + const runtime = createRuntime(); + runtime.continuation = JSON.stringify({ + formatVersion: 1, + requestId: "compatibility-review-2026-07-18T11:59:00.000Z", + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt: "2026-07-18T11:59:00.000Z", + }); + runtime.deleteContinuation = vi + .fn() + .mockRejectedValueOnce(new Error("continuation delete failed")) + .mockImplementation(async () => { + runtime.events.push("delete"); + runtime.continuation = null; + }); + const controller = new ReviewHarnessController(runtime); + + await expect(controller.consumeContinuation()).rejects.toThrow("continuation delete failed"); + + expect(runtime.continuation).not.toBeNull(); + expect(controller.snapshot().resumedRequestId).toBeNull(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("idle"); + + await controller.consumeContinuation(); + + expect(runtime.continuation).toBeNull(); + expect(controller.snapshot().resumedRequestId).toBe("compatibility-review-2026-07-18T11:59:00.000Z"); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + it("passes only after the actual compatibility review clears its pause", async () => { const runtime = createRuntime(); const controller = new ReviewHarnessController(runtime); diff --git a/src/modules/features/ModuleObsidianSettingTab.ts b/src/modules/features/ModuleObsidianSettingTab.ts index 727eee81..ccb51bbd 100644 --- a/src/modules/features/ModuleObsidianSettingTab.ts +++ b/src/modules/features/ModuleObsidianSettingTab.ts @@ -8,13 +8,13 @@ import { openObsidianSettings } from "@/common/obsidianSettings.ts"; export class ModuleObsidianSettingDialogue extends AbstractObsidianModule { settingTab!: ObsidianLiveSyncSettingTab; - _everyOnloadAfterLoadSettings(): Promise { + async _everyOnloadAfterLoadSettings(): Promise { this.settingTab = new ObsidianLiveSyncSettingTab(this.app, this.plugin); - this.settingTab.reloadAllSettings(true); + await this.settingTab.loadSettings(); this.plugin.addSettingTab(this.settingTab); eventHub.onEvent(EVENT_REQUEST_OPEN_SETTINGS, () => this.openSetting()); - return Promise.resolve(true); + return true; } openSetting() { diff --git a/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts b/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts index b1840ea0..87ddc9eb 100644 --- a/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts +++ b/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const settingTabState = vi.hoisted(() => ({ callOrder: [] as string[], - reloadAllSettings: vi.fn<(skipUpdate?: boolean) => void>(), + loadSettings: vi.fn<() => Promise>(), })); const eventHubState = vi.hoisted(() => ({ @@ -11,9 +11,9 @@ const eventHubState = vi.hoisted(() => ({ vi.mock("./SettingDialogue/ObsidianLiveSyncSettingTab.ts", () => ({ ObsidianLiveSyncSettingTab: class ObsidianLiveSyncSettingTab { - reloadAllSettings(skipUpdate?: boolean) { - settingTabState.callOrder.push(`reload:${String(skipUpdate)}`); - settingTabState.reloadAllSettings(skipUpdate); + async loadSettings() { + await settingTabState.loadSettings(); + settingTabState.callOrder.push("settings-loaded"); } }, })); @@ -65,7 +65,7 @@ function createModuleHarness() { describe("ModuleObsidianSettingDialogue startup lifecycle", () => { beforeEach(() => { settingTabState.callOrder.length = 0; - settingTabState.reloadAllSettings.mockClear(); + settingTabState.loadSettings.mockReset().mockResolvedValue(undefined); eventHubState.onEvent.mockClear(); }); @@ -79,13 +79,29 @@ describe("ModuleObsidianSettingDialogue startup lifecycle", () => { }); it("seeds the setting editor without requesting a render before registration", async () => { - const { initialisationHandler, settingsLoadedHandler } = createModuleHarness(); + const { initialisationHandler, settingsLoadedHandler, plugin } = createModuleHarness(); + let finishRead!: () => void; + settingTabState.loadSettings.mockReturnValueOnce( + new Promise((resolve) => { + finishRead = resolve; + }) + ); const handler = settingsLoadedHandler() ?? initialisationHandler(); expect(handler).toBeTypeOf("function"); - await handler!(); + const loading = handler!(); + expect(plugin.addSettingTab).not.toHaveBeenCalled(); + finishRead(); + await loading; - expect(settingTabState.reloadAllSettings).toHaveBeenCalledWith(true); - expect(settingTabState.callOrder).toEqual(["reload:true", "add-setting-tab"]); + expect(settingTabState.loadSettings).toHaveBeenCalledOnce(); + expect(settingTabState.callOrder).toEqual(["settings-loaded", "add-setting-tab"]); + }); + it("does not register an editor when reading device values fails", async () => { + const { settingsLoadedHandler, plugin } = createModuleHarness(); + const failure = new Error("device storage unavailable"); + settingTabState.loadSettings.mockRejectedValueOnce(failure); + await expect(settingsLoadedHandler()!()).rejects.toBe(failure); + expect(plugin.addSettingTab).not.toHaveBeenCalled(); }); }); diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts index d45582bb..da894f79 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { SettingDefinitionGroup, SettingDefinitionItem, SettingDefinitionPage } from "obsidian"; @@ -158,6 +159,7 @@ function createSettingsTab(options: SettingsTabOptions = {}): ObsidianLiveSyncSe }, services: { setting: { + deviceValues: new MemoryValueStore(), getDeviceAndVaultName: vi.fn(() => ""), saveSettingData: vi.fn(async () => undefined), }, @@ -457,7 +459,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { expect(runtime.pageCleanup).toHaveBeenCalledOnce(); }); - it("rebuilds the catalogue when an externally loaded setting changes page visibility", () => { + it("rebuilds the catalogue when an externally loaded setting changes page visibility", async () => { const tab = createSettingsTab(); const changeLog = findPage(tab, "Change Log"); if (!changeLog.page) { @@ -466,7 +468,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { changeLog.page().display(); tab.core.settings.usePowerUserMode = !tab.editingSettings.usePowerUserMode; - tab.requestReload(); + await tab.requestReload(); expect(tab.update).toHaveBeenCalledOnce(); }); @@ -480,7 +482,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { expect(tab.update).toHaveBeenCalledOnce(); }); - it("rebuilds translated catalogue names when the display language changes externally", () => { + it("rebuilds translated catalogue names when the display language changes externally", async () => { const tab = createSettingsTab(); const changeLog = findPage(tab, "Change Log"); if (!changeLog.page) { @@ -489,12 +491,12 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { changeLog.page().display(); tab.core.settings.displayLanguage = "ja"; - tab.requestReload(); + await tab.requestReload(); expect(tab.update).toHaveBeenCalledOnce(); }); - it("rebuilds the catalogue after accepting an external page-visibility setting over a dirty value", () => { + it("rebuilds the catalogue after accepting an external page-visibility setting over a dirty value", async () => { const tab = createSettingsTab(); const changeLog = findPage(tab, "Change Log"); if (!changeLog.page) { @@ -505,7 +507,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { tab.editingSettings.usePowerUserMode = true; tab.core.settings.usePowerUserMode = true; - tab.requestReload(); + await tab.requestReload(); const configureAnchor = vi.mocked(tab.core.confirm.askInPopup).mock.calls[0]?.[2]; expect(configureAnchor).toBeTypeOf("function"); let acceptExternalSetting: (() => void) | undefined; diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 424213f8..a3c72a31 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -52,7 +52,6 @@ import { type PageFunctions, type UpdateFunction, } from "./SettingPane.ts"; -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { JournalSyncCore } from "@vrtmrz/livesync-commonlib/compat/replication/journal/JournalSyncCore"; import { MinioStorageAdapter } from "@vrtmrz/livesync-commonlib/compat/replication/journal/objectstore/MinioStorageAdapter"; import { closeObsidianSettings } from "@/common/obsidianSettings.ts"; @@ -100,6 +99,9 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { } selectedScreen = ""; + private localConfigurationPassphrase = ""; + private localSettingsReadVersion = 0; + _editingSettings?: AllSettings; // Buffered Settings for editing get editingSettings(): AllSettings { @@ -162,8 +164,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { async saveLocalSetting(key: keyof typeof OnDialogSettingsDefault) { if (key == "deviceAndVaultName") { this.services.setting.setDeviceAndVaultName(this.editingSettings?.[key] ?? ""); - this.services.setting.saveDeviceAndVaultName(); - return await Promise.resolve(); + await this.services.setting.saveDeviceAndVaultName(); } } /** @@ -225,6 +226,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { appliedConfigurationEncryption.configPassphrase ); + if (appliedConfigurationEncryption.configPassphraseStore === "LOCALSTORAGE") { + this.localSettingsReadVersion++; + this.localConfigurationPassphrase = appliedConfigurationEncryption.configPassphrase; + } this.copySettingValue(this.initialSettings, appliedConfigurationEncryption, "configPassphraseStore"); this.copySettingValue(this.initialSettings, appliedConfigurationEncryption, "configPassphrase"); } @@ -285,9 +290,21 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { } } + private async readLocalSettings(): Promise { + const version = ++this.localSettingsReadVersion; + const passphrase = await this.services.setting.deviceValues.getValue("ls-setting-passphrase", ""); + if (version !== this.localSettingsReadVersion) return false; + this.localConfigurationPassphrase = passphrase; + return true; + } + + async loadSettings(): Promise { + if (await this.readLocalSettings()) this.reloadAllSettings(true); + } + reloadAllLocalSettings() { const ret = { ...OnDialogSettingsDefault }; - ret.configPassphrase = compatGlobal.localStorage.getItem("ls-setting-passphrase") || ""; + ret.configPassphrase = this.localConfigurationPassphrase; ret.preset = ""; ret.deviceAndVaultName = this.services.setting.getDeviceAndVaultName(); return ret; @@ -402,7 +419,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { this.plugin = plugin; Setting.env = this; eventHub.onEvent(EVENT_REQUEST_RELOAD_SETTING_TAB, () => { - this.requestReload(); + void this.requestReload().catch((error: unknown) => Logger(error, LOG_LEVEL_NOTICE)); }); this.addOnSaved("displayLanguage", () => this.requestCatalogueRefresh()); this.addOnSaved("showStatusOnEditor", () => eventHub.emitEvent(EVENT_ON_UNRESOLVED_ERROR)); @@ -545,7 +562,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { ); } - requestReload() { + async requestReload(): Promise { + if (!(await this.readLocalSettings())) return; const nativeTabIsShown = this.supportsDeclarativeSettings() && this.containerEl !== undefined && this.containerEl.isShown(); if (this.isShown || nativeTabIsShown) { diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts index 1349edfc..df8f6b23 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { REMOTE_RESOURCE_KINDS } from "@vrtmrz/livesync-commonlib/replication"; -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +import { MemoryValueStore } from "@vrtmrz/value-store"; const settingsInitialisationMocks = vi.hoisted(() => ({ applySettingsWithInitialisationChoice: vi.fn(), @@ -76,8 +76,6 @@ beforeEach(() => { settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockReset(); loggerMocks.Logger.mockClear(); vi.mocked($msg).mockClear(); - vi.mocked(compatGlobal.localStorage.getItem).mockReset().mockReturnValue(null); - vi.mocked(compatGlobal.localStorage.setItem).mockReset(); }); describe("ObsidianLiveSyncSettingTab passphrase verification", () => { @@ -280,6 +278,7 @@ describe("ObsidianLiveSyncSettingTab pending-setting initialisation", () => { }, services: { setting: { + deviceValues: new MemoryValueStore(), saveSettingData, getDeviceAndVaultName: vi.fn(() => ""), suspendAllSync, @@ -665,6 +664,7 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { }, services: { setting: { + deviceValues: new MemoryValueStore(), saveSettingData, getDeviceAndVaultName: vi.fn(() => ""), }, @@ -692,6 +692,54 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { return { tab, saveSettingData }; } + it("awaits the stored passphrase before initialising the editing buffer", async () => { + const { tab } = createSettingsTab(); + tab.resetEditingSettings(); + let resolveRead!: (value: string) => void; + vi.spyOn(tab.services.setting.deviceValues, "getValue").mockReturnValueOnce( + new Promise((resolve) => { + resolveRead = resolve; + }) + ); + const loading = tab.loadSettings(); + expect(tab.initialSettings).toBeUndefined(); + resolveRead("stored passphrase"); + await loading; + expect(tab.editingSettings.configPassphrase).toBe("stored passphrase"); + expect(tab.initialSettings!.configPassphrase).toBe("stored passphrase"); + }); + + it("preserves the loaded settings when device storage cannot be read", async () => { + const { tab } = createSettingsTab(); + await tab.services.setting.deviceValues.setValue("ls-setting-passphrase", "stored passphrase"); + await tab.loadSettings(); + const failure = new Error("device storage unavailable"); + vi.spyOn(tab.services.setting.deviceValues, "getValue").mockRejectedValueOnce(failure); + await expect(tab.requestReload()).rejects.toBe(failure); + expect(tab.editingSettings.configPassphrase).toBe("stored passphrase"); + expect(tab.initialSettings!.configPassphrase).toBe("stored passphrase"); + }); + + it("does not let a delayed read overwrite a successfully saved passphrase", async () => { + const { tab } = createSettingsTab(); + await tab.loadSettings(); + let resolveRead!: (value: string) => void; + vi.spyOn(tab.services.setting.deviceValues, "getValue").mockReturnValueOnce( + new Promise((resolve) => { + resolveRead = resolve; + }) + ); + const loading = tab.loadSettings(); + Object.assign(tab.services.setting, { changeConfigurationEncryption: vi.fn(async () => undefined) }); + tab.editingSettings.configPassphraseStore = "LOCALSTORAGE"; + tab.editingSettings.configPassphrase = "new passphrase"; + await tab.saveSettings(["configPassphraseStore", "configPassphrase"]); + resolveRead("old passphrase"); + await loading; + tab.reloadAllSettings(); + expect(tab.initialSettings!.configPassphrase).toBe("new passphrase"); + }); + it("loads the imperative fallback without a SettingPage runtime export", () => { const { tab } = createSettingsTab(); @@ -779,7 +827,7 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { it("saves a custom configuration passphrase edit through the setting service", async () => { const { tab } = createSettingsTab(); const changeConfigurationEncryption = vi.fn(async () => { - vi.mocked(compatGlobal.localStorage.getItem).mockReturnValue("new custom passphrase"); + await tab.services.setting.deviceValues.setValue("ls-setting-passphrase", "new custom passphrase"); }); Object.assign(tab.core.services.setting, { changeConfigurationEncryption }); const store = "LOCALSTORAGE"; @@ -787,13 +835,13 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { tab.initialSettings!.configPassphraseStore = store; tab.editingSettings.configPassphrase = "new custom passphrase"; tab.initialSettings!.configPassphrase = "old custom passphrase"; - vi.mocked(compatGlobal.localStorage.setItem).mockClear(); + const setValue = vi.spyOn(tab.services.setting.deviceValues, "setValue"); await tab.saveSettings(["configPassphrase"]); expect(changeConfigurationEncryption).toHaveBeenCalledOnce(); expect(changeConfigurationEncryption).toHaveBeenCalledWith(store, "new custom passphrase"); - expect(compatGlobal.localStorage.setItem).not.toHaveBeenCalled(); + expect(setValue).toHaveBeenCalledOnce(); expect(tab.initialSettings!.configPassphrase).toBe("new custom passphrase"); }); @@ -811,7 +859,6 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { tab.initialSettings!.configPassphraseStore = "LOCALSTORAGE"; tab.initialSettings!.configPassphrase = "previous passphrase"; tab.editingSettings.configPassphrase = "passphrase A"; - vi.mocked(compatGlobal.localStorage.getItem).mockReturnValue("passphrase A"); const apply = tab.saveSettings(["configPassphrase"]); expect(changeConfigurationEncryption).toHaveBeenCalledWith("LOCALSTORAGE", "passphrase A"); @@ -840,13 +887,13 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { tab.editingSettings.configPassphraseStore = editedStore; tab.initialSettings!.configPassphrase = "old custom passphrase"; tab.editingSettings.configPassphrase = "new custom passphrase"; - vi.mocked(compatGlobal.localStorage.setItem).mockClear(); + const setValue = vi.spyOn(tab.services.setting.deviceValues, "setValue"); await expect(tab.saveSettings(["configPassphraseStore", "configPassphrase"])).rejects.toBe(failure); expect(changeConfigurationEncryption).toHaveBeenCalledOnce(); expect(changeConfigurationEncryption).toHaveBeenCalledWith(editedStore, "new custom passphrase"); - expect(compatGlobal.localStorage.setItem).not.toHaveBeenCalled(); + expect(setValue).not.toHaveBeenCalled(); expect(tab.core.settings.configPassphraseStore).toBe(savedStore); expect(tab.initialSettings!.configPassphraseStore).toBe(savedStore); expect(tab.initialSettings!.configPassphrase).toBe("old custom passphrase"); diff --git a/src/modules/features/SetupManager.unit.spec.ts b/src/modules/features/SetupManager.unit.spec.ts index c8dfe4ec..68320e2c 100644 --- a/src/modules/features/SetupManager.unit.spec.ts +++ b/src/modules/features/SetupManager.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_SETTINGS, @@ -37,11 +38,6 @@ import { decodeSettingsFromQRCodeData } from "@vrtmrz/livesync-commonlib/compat/ import { SetupManager, UserMode } from "./SetupManager"; class TestSettingService extends SettingService { - protected setItem(_key: string, _value: string): void {} - protected getItem(_key: string): string { - return ""; - } - protected deleteItem(_key: string): void {} protected saveData(_setting: ObsidianLiveSyncSettings): Promise { return Promise.resolve(); } @@ -65,6 +61,7 @@ function createLegacyRemoteSetting(): ObsidianLiveSyncSettings { function createSetupManager() { const setting = new TestSettingService(new ServiceContext(), { + deviceValues: new MemoryValueStore(), APIService: { getSystemVaultName: vi.fn(() => "vault"), getAppID: vi.fn(() => "app"), diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte index 2f89b845..210633b5 100644 --- a/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte +++ b/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte @@ -52,26 +52,48 @@ const context = getDialogContext(); let error = $state(""); let connectionPathResetNotice = $state(false); + let peerNameReady = $state(false); + let mounted = false; const hasValidTurnServer = $derived(hasP2PTurnConfiguration(syncSetting)); type Props = GuestDialogProps; const { setResult, getInitialData }: Props = $props(); let connectionProbe: SetupRemoteP2PInitialData["connectionProbe"] | undefined; onMount(() => { + mounted = true; const initialData = getInitialData?.(); connectionProbe = initialData?.connectionProbe; const initialSettings = initialData?.settings; if (initialSettings) { syncSetting = pickP2PSyncSettings(initialSettings); } - const initialPeerName = (initialSettings?.P2P_DevicePeerName ?? "").trim(); + const savedPeerName = initialSettings?.P2P_DevicePeerName; + const initialPeerName = typeof savedPeerName === "string" ? savedPeerName.trim() : ""; if (initialPeerName !== "") { - return; - } - const cachedPeerName = context.services.config.getSmallConfig(SETTING_KEY_P2P_DEVICE_NAME); - if (cachedPeerName) { - syncSetting.P2P_DevicePeerName = cachedPeerName as string; + peerNameReady = true; + return () => { + mounted = false; + }; } + void context.services.config.getValue(SETTING_KEY_P2P_DEVICE_NAME).then( + (cachedPeerName) => { + if (mounted) { + if (typeof cachedPeerName === "string" && cachedPeerName) { + syncSetting.P2P_DevicePeerName = cachedPeerName; + } + peerNameReady = true; + } + }, + (loadError: unknown) => { + if (mounted) { + error = loadError instanceof Error ? loadError.message : String(loadError); + peerNameReady = true; + } + } + ); + return () => { + mounted = false; + }; }); function generateSetting() { const connSetting: P2PSyncSetting = { @@ -193,6 +215,7 @@ } async function checkAndCommit() { + if (!peerNameReady) return; error = ""; try { error = (await checkConnection()) || ""; @@ -207,6 +230,7 @@ } } function commit() { + if (!peerNameReady) return; error = validateManagedTurnSettings(syncSetting) ?? ""; if (error) return; const setting = pickP2PSyncSettings(generateSetting()); @@ -217,6 +241,7 @@ } const canProceed = $derived.by(() => { return ( + peerNameReady && syncSetting.P2P_relays.trim() !== "" && syncSetting.P2P_roomID.trim() !== "" && syncSetting.P2P_passphrase.trim() !== "" && @@ -291,6 +316,7 @@ autocorrect="off" autocapitalize="off" spellcheck="false" + disabled={!peerNameReady} bind:value={syncSetting.P2P_DevicePeerName} /> @@ -385,7 +411,11 @@ disabled={!canProceed} commit={() => checkAndCommit()} /> - commit()} /> + commit()} + /> cancel()} /> {/if} diff --git a/src/modules/services/ObsidianServiceHub.ts b/src/modules/services/ObsidianServiceHub.ts index 3613ea27..b361986f 100644 --- a/src/modules/services/ObsidianServiceHub.ts +++ b/src/modules/services/ObsidianServiceHub.ts @@ -1,4 +1,6 @@ import { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; +import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage"; import { ObsidianServiceContext } from "@/modules/services/ObsidianServiceContext"; import type { ServiceInstances } from "@vrtmrz/livesync-commonlib/compat/services/ServiceHub"; import type ObsidianLiveSyncPlugin from "@/main"; @@ -11,7 +13,6 @@ import { ObsidianTweakValueService, ObsidianTestService, ObsidianDatabaseEventService, - ObsidianConfigService, ObsidianKeyValueDBService, ObsidianControlService, } from "./ObsidianServices"; @@ -46,6 +47,7 @@ export class ObsidianServiceHub extends InjectableServiceHub( - task: () => T | PromiseLike, - options?: ActivityOptions - ): Promise { + async runBoundedLocalApplicationActivity(task: () => T | PromiseLike, options?: ActivityOptions): Promise { this.boundedLocalApplicationActivityCount.value++; try { return this.dependencies.activityRunner @@ -84,7 +80,6 @@ export class ObsidianConflictService extends InjectableConflictService {} // InjectableTestService export class ObsidianTestService extends InjectableTestService {} -export class ObsidianConfigService extends ConfigServiceBrowserCompat {} export class ObsidianKeyValueDBService extends KeyValueDBService {} diff --git a/src/modules/services/ObsidianSettingService.ts b/src/modules/services/ObsidianSettingService.ts index a934ebd1..87b93b59 100644 --- a/src/modules/services/ObsidianSettingService.ts +++ b/src/modules/services/ObsidianSettingService.ts @@ -1,4 +1,3 @@ -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { EVENT_REQUEST_RELOAD_SETTING_TAB, @@ -27,19 +26,6 @@ export class ObsidianSettingService extends Se return Promise.resolve(true); }); } - protected setItem(key: string, value: string) { - // TODO: Implement nativeLocalStorage. - return compatGlobal.localStorage.setItem(key, value); - } - protected getItem(key: string): string { - // TODO: Implement nativeLocalStorage. - return compatGlobal.localStorage.getItem(key) ?? ""; - } - protected deleteItem(key: string): void { - // TODO: Implement nativeLocalStorage. - compatGlobal.localStorage.removeItem(key); - } - protected override async saveData(data: ObsidianLiveSyncSettings): Promise { return await this.context.liveSyncPlugin.saveData(data); } diff --git a/src/modules/services/ObsidianSettingService.unit.spec.ts b/src/modules/services/ObsidianSettingService.unit.spec.ts index 9175c931..6df8404d 100644 --- a/src/modules/services/ObsidianSettingService.unit.spec.ts +++ b/src/modules/services/ObsidianSettingService.unit.spec.ts @@ -1,6 +1,23 @@ import { describe, expect, it } from "vitest"; +import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { IAPIService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import { MemoryValueStore } from "@vrtmrz/value-store"; +import type { ObsidianServiceContext } from "@/modules/services/ObsidianServiceContext"; import { normaliseObsidianSettingsData } from "./ObsidianSettingService.ts"; +import { ObsidianSettingService } from "./ObsidianSettingService.ts"; + +function createSettingService( + vaultName: string, + deviceValues: MemoryValueStore +): ObsidianSettingService { + const context = createServiceContext() as unknown as ObsidianServiceContext; + const APIService = { + addLog: () => undefined, + getSystemVaultName: () => vaultName, + } as unknown as IAPIService; + return new ObsidianSettingService(context, { APIService, deviceValues }); +} describe("normaliseObsidianSettingsData", () => { it("maps Obsidian's missing data value to Commonlib's new-Vault input", () => { @@ -12,4 +29,26 @@ describe("normaliseObsidianSettingsData", () => { expect(normaliseObsidianSettingsData(settings)).toBe(settings); }); + + it("keeps injected device values independent from each Vault-scoped configuration store", async () => { + const firstValues = new MemoryValueStore(); + const secondValues = new MemoryValueStore(); + const firstSetting = createSettingService("first-vault", firstValues); + const secondSetting = createSettingService("second-vault", secondValues); + + await firstSetting.deviceValues.setValue("shared-key", "device value"); + await firstSetting.config.setValue("shared-key", "first config"); + await secondSetting.config.setValue("shared-key", "second config"); + + expect(firstSetting.deviceValues).toBe(firstValues); + expect(secondSetting.deviceValues).toBe(secondValues); + expect(firstSetting.deviceValues).not.toBe(secondSetting.deviceValues); + expect(firstSetting.config).not.toBe(firstValues); + expect(firstSetting.config).not.toBe(secondSetting.config); + await expect(firstSetting.deviceValues.getValue("shared-key")).resolves.toBe("device value"); + await expect(firstSetting.config.getValue("shared-key")).resolves.toBe("first config"); + await expect(secondSetting.config.getValue("shared-key")).resolves.toBe("second config"); + await expect(firstValues.getValue("first-vault-shared-key")).resolves.toBe("first config"); + await expect(secondValues.getValue("second-vault-shared-key")).resolves.toBe("second config"); + }); }); diff --git a/src/modules/services/ObsidianUIService.ts b/src/modules/services/ObsidianUIService.ts index c5abf23f..08e6e572 100644 --- a/src/modules/services/ObsidianUIService.ts +++ b/src/modules/services/ObsidianUIService.ts @@ -1,4 +1,4 @@ -import type { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService"; +import type { IConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import type { AppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService"; import type { ReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/base/ReplicatorService"; import { UIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/base/UIService"; @@ -8,7 +8,7 @@ import DialogToCopy from "@/modules/services/LiveSyncUI/dialogues/DialogueToCopy import type { IAPIService, IControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; export type ObsidianUIServiceDependencies = { appLifecycle: AppLifecycleService; - config: ConfigService; + config: IConfigService; replicator: ReplicatorService; APIService: IAPIService; control: IControlService; diff --git a/src/serviceFeatures/compatibilityReview.ts b/src/serviceFeatures/compatibilityReview.ts index b92aeec6..7c79f98f 100644 --- a/src/serviceFeatures/compatibilityReview.ts +++ b/src/serviceFeatures/compatibilityReview.ts @@ -44,17 +44,17 @@ export class CompatibilityReviewController { return this._initialised; } - private readAcknowledgedVersion(): string | null { + private async readAcknowledgedVersion(): Promise { const setting = this.core.services.setting; - const currentMarker = setting.getSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY); + const currentMarker = await this.core.services.config.getValue(DATABASE_COMPATIBILITY_VERSION_KEY, ""); if (currentMarker) return currentMarker; const legacyKey = legacyDatabaseCompatibilityVersionKey(this.core.services.vault.getVaultName()); - const legacyMarker = setting.getDeviceLocalConfig(legacyKey); + const legacyMarker = await setting.deviceValues.getValue(legacyKey, ""); if (!legacyMarker) return null; - setting.setSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY, legacyMarker); - setting.deleteDeviceLocalConfig(legacyKey); + await this.core.services.config.setValue(DATABASE_COMPATIBILITY_VERSION_KEY, legacyMarker); + await setting.deviceValues.delete(legacyKey); return legacyMarker; } @@ -75,7 +75,7 @@ export class CompatibilityReviewController { return true; } - const acknowledgedVersion = this.readAcknowledgedVersion(); + const acknowledgedVersion = await this.readAcknowledgedVersion(); const evaluation = evaluateCompatibilityPause({ acknowledgedVersion, currentVersion: this.currentVersion, @@ -85,7 +85,7 @@ export class CompatibilityReviewController { this.pause = evaluation.pause; if (evaluation.initialiseAcknowledgedVersion) { - setting.setSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); + await this.core.services.config.setValue(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); this._initialised = true; return true; } @@ -115,9 +115,9 @@ export class CompatibilityReviewController { throw error; } - setting.setSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); + await this.core.services.config.setValue(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); const legacyKey = legacyDatabaseCompatibilityVersionKey(this.core.services.vault.getVaultName()); - setting.deleteDeviceLocalConfig(legacyKey); + await setting.deviceValues.delete(legacyKey); this.pause = undefined; this.ui.clearReminder(); await this.core.services.control.applySettings(); diff --git a/src/serviceFeatures/compatibilityReview.unit.spec.ts b/src/serviceFeatures/compatibilityReview.unit.spec.ts index 760718cc..44a1b05b 100644 --- a/src/serviceFeatures/compatibilityReview.unit.spec.ts +++ b/src/serviceFeatures/compatibilityReview.unit.spec.ts @@ -33,13 +33,32 @@ function createFixture( } = {} ) { const local = new Map(); + const deviceValues = new Map(); if (options.marker !== undefined && options.marker !== null) { local.set(DATABASE_COMPATIBILITY_VERSION_KEY, options.marker); } const legacyKey = legacyDatabaseCompatibilityVersionKey("Test Vault"); if (options.legacyMarker !== undefined && options.legacyMarker !== null) { - local.set(legacyKey, options.legacyMarker); + deviceValues.set(legacyKey, options.legacyMarker); } + const config = { + getValue: vi.fn(async (key: string, defaultValue = "") => local.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + local.set(key, value); + }), + delete: vi.fn(async (key: string) => { + local.delete(key); + }), + }; + const deviceValueStore = { + getValue: vi.fn(async (key: string, defaultValue = "") => deviceValues.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + deviceValues.set(key, value); + }), + delete: vi.fn(async (key: string) => { + deviceValues.delete(key); + }), + }; const settings = { versionUpFlash: options.versionUpFlash ?? "", isConfigured: options.isConfigured ?? true, @@ -49,15 +68,13 @@ function createFixture( const setting = { currentSettings: () => settings, getSettingsMigrationState: () => migrationState(options.migration), - getSmallConfig: (key: string) => local.get(key) ?? "", - setSmallConfig: (key: string, value: string) => local.set(key, value), - getDeviceLocalConfig: (key: string) => local.get(key) ?? null, - deleteDeviceLocalConfig: (key: string) => local.delete(key), + deviceValues: deviceValueStore, saveSettingData, }; const core = { services: { setting, + config, vault: { getVaultName: () => "Test Vault" }, control: { applySettings }, }, @@ -69,7 +86,18 @@ function createFixture( clearReminder: vi.fn(), }; const controller = new CompatibilityReviewController(core, ui, 12); - return { controller, ui, local, legacyKey, settings, saveSettingData, applySettings }; + return { + controller, + ui, + local, + deviceValues, + config, + deviceValueStore, + legacyKey, + settings, + saveSettingData, + applySettings, + }; } describe("compatibility review controller", () => { @@ -196,10 +224,96 @@ describe("compatibility review controller", () => { await fixture.controller.initialise(); expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); - expect(fixture.local.has(fixture.legacyKey)).toBe(false); + expect(fixture.deviceValues.has(fixture.legacyKey)).toBe(false); expect(fixture.controller.pendingPause).toBeDefined(); }); + it("saves the scoped marker before deleting the legacy device value", async () => { + const fixture = createFixture({ legacyMarker: "11" }); + let finishSave!: () => void; + const saving = new Promise((resolve) => { + finishSave = resolve; + }); + fixture.config.setValue.mockImplementation(async (key, value) => { + await saving; + fixture.local.set(key, value); + }); + + const initialising = fixture.controller.initialise(); + await Promise.resolve(); + await Promise.resolve(); + + expect(fixture.deviceValues.get(fixture.legacyKey)).toBe("11"); + expect(fixture.deviceValueStore.delete).not.toHaveBeenCalled(); + expect(fixture.controller.initialised).toBe(false); + + finishSave(); + await initialising; + + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); + expect(fixture.deviceValues.has(fixture.legacyKey)).toBe(false); + expect(fixture.controller.initialised).toBe(true); + }); + + it("does not treat a failed marker read as an absent value", async () => { + const fixture = createFixture({ legacyMarker: "11" }); + fixture.config.getValue.mockRejectedValueOnce(new Error("marker read failed")); + + await expect(fixture.controller.initialise()).rejects.toThrow("marker read failed"); + + expect(fixture.deviceValueStore.getValue).not.toHaveBeenCalled(); + expect(fixture.deviceValues.get(fixture.legacyKey)).toBe("11"); + expect(fixture.controller.initialised).toBe(false); + }); + + it("keeps the compatibility pause blocked until its async marker write succeeds", async () => { + const fixture = createFixture({ marker: "11" }); + vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); + await fixture.controller.initialise(); + let finishSave!: () => void; + const saving = new Promise((resolve) => { + finishSave = resolve; + }); + fixture.config.setValue.mockImplementation(async (key, value) => { + await saving; + fixture.local.set(key, value); + }); + + const reviewing = fixture.controller.openReview(); + await Promise.resolve(); + await Promise.resolve(); + + expect(fixture.controller.pendingPause).toBeDefined(); + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); + expect(fixture.applySettings).not.toHaveBeenCalled(); + + finishSave(); + await reviewing; + + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("12"); + expect(fixture.controller.pendingPause).toBeUndefined(); + expect(fixture.applySettings).toHaveBeenCalledOnce(); + }); + + it("keeps a storage-failed acknowledgement blocked and retryable", async () => { + const fixture = createFixture({ marker: "11" }); + vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); + await fixture.controller.initialise(); + fixture.config.setValue.mockRejectedValueOnce(new Error("marker write failed")); + + await expect(fixture.controller.openReview()).rejects.toThrow("marker write failed"); + + expect(fixture.controller.pendingPause).toBeDefined(); + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); + expect(fixture.applySettings).not.toHaveBeenCalled(); + + await fixture.controller.openReview(); + + expect(fixture.controller.pendingPause).toBeUndefined(); + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("12"); + expect(fixture.applySettings).toHaveBeenCalledOnce(); + }); + it("restores the runtime gate if persisting an acknowledgement fails", async () => { const fixture = createFixture({ marker: "11" }); vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); diff --git a/src/serviceFeatures/redFlag.simpleFetch.ts b/src/serviceFeatures/redFlag.simpleFetch.ts index 4ce84e29..950e1618 100644 --- a/src/serviceFeatures/redFlag.simpleFetch.ts +++ b/src/serviceFeatures/redFlag.simpleFetch.ts @@ -63,28 +63,27 @@ function buildSimpleFetchResult(stage1: string, stage2?: string) { return undefined; } -function rememberSimpleFetchMode(host: NecessaryServices<"setting", never>, stage1: string, stage2?: string) { - host.services.setting.setSmallConfig(SIMPLE_FETCH_MODE_KEY, JSON.stringify({ stage1, stage2 })); +async function rememberSimpleFetchMode(host: NecessaryServices<"setting", never>, stage1: string, stage2?: string) { + await host.services.setting.config.setValue(SIMPLE_FETCH_MODE_KEY, JSON.stringify({ stage1, stage2 })); } -function getRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { - const saved = host.services.setting.getSmallConfig(SIMPLE_FETCH_MODE_KEY); +async function getRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { + const saved = await host.services.setting.config.getValue(SIMPLE_FETCH_MODE_KEY, ""); if (!saved) return undefined; + let remembered: ReturnType = undefined; try { const { stage1, stage2 } = JSON.parse(saved) as { stage1?: string; stage2?: string }; - if (stage1) { - const remembered = buildSimpleFetchResult(stage1, stage2); - if (remembered) return remembered; - } + if (stage1) remembered = buildSimpleFetchResult(stage1, stage2); } catch { // Clear below; the saved choice is optional and can be rebuilt by asking again. } - host.services.setting.deleteSmallConfig(SIMPLE_FETCH_MODE_KEY); + if (remembered) return remembered; + await host.services.setting.config.delete(SIMPLE_FETCH_MODE_KEY); return undefined; } -function clearRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { - host.services.setting.deleteSmallConfig(SIMPLE_FETCH_MODE_KEY); +async function clearRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { + await host.services.setting.config.delete(SIMPLE_FETCH_MODE_KEY); } async function confirmTranslatedChoice( @@ -104,7 +103,7 @@ async function confirmTranslatedChoice( export async function askSimpleFetchMode( host: NecessaryServices<"UI" | "setting", never> ): Promise<{ mode: string; options: Partial } | "cancelled" | "aborted"> { - const remembered = getRememberedSimpleFetchMode(host); + const remembered = await getRememberedSimpleFetchMode(host); if (remembered) return remembered; const msg = $msg( @@ -169,7 +168,7 @@ Firstly, how shall we handle the data retrieved from this remote source? if (stage2 === STAGE2_ABORT) { return "aborted"; } - rememberSimpleFetchMode(host, stage1, stage2); + await rememberSimpleFetchMode(host, stage1, stage2); return buildSimpleFetchResult(stage1, stage2)!; } @@ -198,7 +197,7 @@ Firstly, how shall we handle the data retrieved from this remote source? if (stage2 === STAGE2_ABORT) { return "aborted"; } - rememberSimpleFetchMode(host, stage1, stage2); + await rememberSimpleFetchMode(host, stage1, stage2); return buildSimpleFetchResult(stage1, stage2)!; } @@ -234,18 +233,18 @@ export async function askAndPerformFastSetupOnScheduledFetchAll( "Remediation mode is active, so the detailed fetch flow is used instead of Simple Fetch.", LOG_LEVEL_NOTICE ); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); return undefined; } const result = await askSimpleFetchMode(host); if (result === "cancelled") { log("Fetch cancelled by user.", LOG_LEVEL_NOTICE); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); return await cancelScheduledInitialisation(host, cleanupFlag); } if (result === "aborted") { log("Fetch exited by user.", LOG_LEVEL_NOTICE); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); host.services.appLifecycle.performRestart(); return false; } @@ -256,7 +255,7 @@ export async function askAndPerformFastSetupOnScheduledFetchAll( const settings = host.services.setting.currentSettings(); if (!(await adjustSettingToRemoteIfNeeded(host, log, { preventFetchingConfig: false }, settings))) { log("Fetch initialisation cancelled by user.", LOG_LEVEL_NOTICE); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); return await cancelScheduledInitialisation(host, cleanupFlag); } @@ -295,7 +294,7 @@ export async function askAndPerformFastSetupOnScheduledFetchAll( } await host.serviceModules.rebuilder.finishRebuild(); await cleanupFlag(); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); log("Simple fetch and scan operation completed.", LOG_LEVEL_NOTICE); return true; }; diff --git a/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts b/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts index 73b3d9c0..bc583d8f 100644 --- a/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts +++ b/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts @@ -10,19 +10,26 @@ const KEEP_LOCAL = "Keep local files even if not on remote"; function createHost(savedChoice?: string) { const smallConfig = new Map(); if (savedChoice) smallConfig.set("simple-fetch-mode", savedChoice); + const config = { + getValue: vi.fn(async (key: string, defaultValue = "") => smallConfig.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + smallConfig.set(key, value); + }), + delete: vi.fn(async (key: string) => { + smallConfig.delete(key); + }), + }; const confirmWithMessage = vi.fn<(title: string, message: string, buttons: string[], defaultAction: string) => Promise>(); const host = { services: { UI: { confirm: { confirmWithMessage } }, setting: { - getSmallConfig: (key: string) => smallConfig.get(key) ?? "", - setSmallConfig: (key: string, value: string) => smallConfig.set(key, value), - deleteSmallConfig: (key: string) => smallConfig.delete(key), + config, }, }, } as unknown as NecessaryServices<"UI" | "setting", never>; - return { host, confirmWithMessage, smallConfig }; + return { host, confirmWithMessage, smallConfig, config }; } afterEach(() => setLang("def")); @@ -50,6 +57,75 @@ describe("Fast Setup translated choices", () => { }); }); + it("waits for the saved choice before returning it to the fetch workflow", async () => { + const { host, confirmWithMessage, smallConfig, config } = createHost(); + let markSaveStarted!: () => void; + const saveStarted = new Promise((resolve) => { + markSaveStarted = resolve; + }); + let finishSave!: () => void; + const save = new Promise((resolve) => { + finishSave = resolve; + }); + config.setValue.mockImplementation(async (key, value) => { + markSaveStarted(); + await save; + smallConfig.set(key, value); + }); + confirmWithMessage.mockImplementation(async (_title, _message, buttons) => buttons[1]); + + const asking = askSimpleFetchMode(host); + await saveStarted; + + expect(smallConfig.has("simple-fetch-mode")).toBe(false); + expect(confirmWithMessage).toHaveBeenCalledTimes(2); + + finishSave(); + await expect(asking).resolves.toMatchObject({ mode: "remote-only" }); + expect(JSON.parse(smallConfig.get("simple-fetch-mode")!)).toEqual({ + stage1: REMOTE_WINS, + stage2: KEEP_LOCAL, + }); + }); + + it("propagates a stored-choice read failure instead of treating it as missing", async () => { + const { host, confirmWithMessage, config } = createHost(); + config.getValue.mockRejectedValueOnce(new Error("choice read failed")); + + await expect(askSimpleFetchMode(host)).rejects.toThrow("choice read failed"); + + expect(confirmWithMessage).not.toHaveBeenCalled(); + }); + + it("waits for an invalid optional choice to be deleted before asking again", async () => { + const { host, confirmWithMessage, smallConfig, config } = createHost("not-json"); + let markDeleteStarted!: () => void; + const deleteStarted = new Promise((resolve) => { + markDeleteStarted = resolve; + }); + let finishDelete!: () => void; + const deletion = new Promise((resolve) => { + finishDelete = resolve; + }); + config.delete.mockImplementation(async (key) => { + markDeleteStarted(); + await deletion; + smallConfig.delete(key); + }); + confirmWithMessage.mockImplementation(async (_title, _message, buttons) => buttons[3]); + + const asking = askSimpleFetchMode(host); + await deleteStarted; + + expect(confirmWithMessage).not.toHaveBeenCalled(); + expect(smallConfig.has("simple-fetch-mode")).toBe(true); + + finishDelete(); + await expect(asking).resolves.toBe("cancelled"); + expect(smallConfig.has("simple-fetch-mode")).toBe(false); + expect(confirmWithMessage).toHaveBeenCalledOnce(); + }); + it("restores choices saved by the previous English implementation while displaying Spanish", async () => { const savedChoice = JSON.stringify({ stage1: REMOTE_WINS, stage2: KEEP_LOCAL }); const { host, confirmWithMessage, smallConfig } = createHost(savedChoice); diff --git a/src/serviceFeatures/redFlag.unit.spec.ts b/src/serviceFeatures/redFlag.unit.spec.ts index 9c1a2bf8..b21cb99a 100644 --- a/src/serviceFeatures/redFlag.unit.spec.ts +++ b/src/serviceFeatures/redFlag.unit.spec.ts @@ -107,13 +107,15 @@ const createSettingServiceMock = () => { }), suspendAllSync: vi.fn(() => Promise.resolve()), suspendExtraSync: vi.fn(() => Promise.resolve()), - getSmallConfig: vi.fn((key: string) => smallConfig.get(key) ?? ""), - setSmallConfig: vi.fn((key: string, value: string) => { - smallConfig.set(key, value); - }), - deleteSmallConfig: vi.fn((key: string) => { - smallConfig.delete(key); - }), + config: { + getValue: vi.fn(async (key: string, defaultValue = "") => smallConfig.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + smallConfig.set(key, value); + }), + delete: vi.fn(async (key: string) => { + smallConfig.delete(key); + }), + }, }; }; @@ -785,7 +787,7 @@ describe("Red Flag Feature", () => { ); expect(host.mocks.ui.confirm.confirmWithMessage).not.toHaveBeenCalled(); - expect(host.mocks.setting.deleteSmallConfig).toHaveBeenCalledWith("simple-fetch-mode"); + expect(host.mocks.setting.config.delete).toHaveBeenCalledWith("simple-fetch-mode"); expect(cleanupFlag).not.toHaveBeenCalled(); }); @@ -830,7 +832,7 @@ describe("Red Flag Feature", () => { }); expect(host.mocks.rebuilder.finishRebuild).not.toHaveBeenCalled(); expect(cleanupFlag).not.toHaveBeenCalled(); - expect(host.mocks.setting.deleteSmallConfig).not.toHaveBeenCalledWith("simple-fetch-mode"); + expect(host.mocks.setting.config.delete).not.toHaveBeenCalledWith("simple-fetch-mode"); }); it("re-suspends both reflection directions when finalisation fails after releasing them", async () => { @@ -892,7 +894,7 @@ describe("Red Flag Feature", () => { await askAndPerformFastSetupOnScheduledFetchAll(host as any, log, cleanupFlag); - expect(host.mocks.setting.deleteSmallConfig).toHaveBeenCalledWith("simple-fetch-mode"); + expect(host.mocks.setting.config.delete).toHaveBeenCalledWith("simple-fetch-mode"); }); it("should return false and cleanup when quick flow is cancelled", async () => { diff --git a/src/serviceFeatures/useReviewHarness.ts b/src/serviceFeatures/useReviewHarness.ts index 99e2cd5a..64efb03e 100644 --- a/src/serviceFeatures/useReviewHarness.ts +++ b/src/serviceFeatures/useReviewHarness.ts @@ -69,9 +69,9 @@ export function useReviewHarness( await createFileContentHashBenchmarkOperations(activeWindow.performance), activeWindow.performance ), - readContinuation: () => services.setting.getSmallConfig(REVIEW_HARNESS_STATE_KEY), - writeContinuation: (value) => services.setting.setSmallConfig(REVIEW_HARNESS_STATE_KEY, value), - deleteContinuation: () => services.setting.deleteSmallConfig(REVIEW_HARNESS_STATE_KEY), + readContinuation: () => services.config.getValue(REVIEW_HARNESS_STATE_KEY, ""), + writeContinuation: (value) => services.config.setValue(REVIEW_HARNESS_STATE_KEY, value), + deleteContinuation: () => services.config.delete(REVIEW_HARNESS_STATE_KEY), restart: () => services.appLifecycle.performRestart(), reportError: (error) => services.API.addLog(error, LOG_LEVEL_NOTICE), copyText: async (value) => { @@ -94,14 +94,14 @@ export function useReviewHarness( let registered = false; let openAfterLayout = false; - services.appLifecycle.onSettingLoaded.addHandler(() => { + services.appLifecycle.onSettingLoaded.addHandler(async () => { if (!continuationConsumed) { + await controller.consumeContinuation(); continuationConsumed = true; - controller.consumeContinuation(); } const snapshot = controller.snapshot(); openAfterLayout = snapshot.resumedRequestId !== null || snapshot.continuationError !== null; - if (!services.setting.currentSettings().enableDebugTools || registered) return Promise.resolve(true); + if (!services.setting.currentSettings().enableDebugTools || registered) return true; registered = true; services.API.registerWindow( @@ -115,7 +115,7 @@ export function useReviewHarness( void services.API.showWindow(VIEW_TYPE_REVIEW_HARNESS); }, }); - return Promise.resolve(true); + return true; }); services.appLifecycle.onLayoutReady.addHandler(() => { diff --git a/src/serviceFeatures/useReviewHarness.unit.spec.ts b/src/serviceFeatures/useReviewHarness.unit.spec.ts index 35281081..c73070a7 100644 --- a/src/serviceFeatures/useReviewHarness.unit.spec.ts +++ b/src/serviceFeatures/useReviewHarness.unit.spec.ts @@ -31,6 +31,15 @@ function createFixture(options: { enableDebugTools?: boolean; continuation?: str const layoutReadyHandlers: Array<() => Promise> = []; const local = new Map(); if (options.continuation) local.set(REVIEW_HARNESS_STATE_KEY, options.continuation); + const config = { + getValue: vi.fn(async (key: string, defaultValue = "") => local.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + local.set(key, value); + }), + delete: vi.fn(async (key: string) => { + local.delete(key); + }), + }; const settings = { enableDebugTools: options.enableDebugTools ?? true, liveSync: false, @@ -63,10 +72,8 @@ function createFixture(options: { enableDebugTools?: boolean; continuation?: str requiresSyncReview: true, reviewReasons: [], }), - getSmallConfig: (key: string) => local.get(key) ?? "", - setSmallConfig: (key: string, value: string) => local.set(key, value), - deleteSmallConfig: (key: string) => local.delete(key), }, + config, appLifecycle: { onSettingLoaded: { addHandler: vi.fn((handler: () => Promise) => settingLoadedHandlers.push(handler)), @@ -106,6 +113,7 @@ function createFixture(options: { enableDebugTools?: boolean; continuation?: str api, settings, local, + config, settingLoadedHandlers, layoutReadyHandlers, compatibilityReview, @@ -144,6 +152,74 @@ describe("Review Harness composition", () => { expect(fixture.api.showWindow).toHaveBeenCalledWith(VIEW_TYPE_REVIEW_HARNESS); }); + it("holds the setting-loaded lifecycle until continuation deletion has completed", async () => { + const requestedAt = "2026-07-18T11:59:00.000Z"; + const continuation = JSON.stringify({ + formatVersion: 1, + requestId: `compatibility-review-${requestedAt}`, + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt, + }); + const fixture = createFixture({ continuation }); + let markDeleteStarted!: () => void; + const deleteStarted = new Promise((resolve) => { + markDeleteStarted = resolve; + }); + let finishDelete!: () => void; + const deleting = new Promise((resolve) => { + finishDelete = resolve; + }); + fixture.config.delete.mockImplementation(async (key) => { + markDeleteStarted(); + await deleting; + fixture.local.delete(key); + }); + + const loading = fixture.settingLoadedHandlers[0](); + await deleteStarted; + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(true); + expect(fixture.controller.snapshot().resumedRequestId).toBeNull(); + expect(fixture.api.registerWindow).not.toHaveBeenCalled(); + + finishDelete(); + await loading; + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(false); + expect(fixture.controller.snapshot().resumedRequestId).toBe(`compatibility-review-${requestedAt}`); + expect(fixture.api.registerWindow).toHaveBeenCalledOnce(); + }); + + it("can retry one-shot continuation consumption after a deletion failure", async () => { + const requestedAt = "2026-07-18T11:59:00.000Z"; + const continuation = JSON.stringify({ + formatVersion: 1, + requestId: `compatibility-review-${requestedAt}`, + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt, + }); + const fixture = createFixture({ continuation }); + fixture.config.delete + .mockRejectedValueOnce(new Error("continuation delete failed")) + .mockImplementation(async (key) => { + fixture.local.delete(key); + }); + + await expect(fixture.settingLoadedHandlers[0]()).rejects.toThrow("continuation delete failed"); + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(true); + expect(fixture.controller.snapshot().resumedRequestId).toBeNull(); + expect(fixture.api.registerWindow).not.toHaveBeenCalled(); + + await fixture.settingLoadedHandlers[0](); + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(false); + expect(fixture.controller.snapshot().resumedRequestId).toBe(`compatibility-review-${requestedAt}`); + expect(fixture.api.registerWindow).toHaveBeenCalledOnce(); + }); + it("uses the actual compatibility controller as the guided review boundary", async () => { const fixture = createFixture(); diff --git a/test/apps/webapp/WebAppRuntime.unit.spec.ts b/test/apps/webapp/WebAppRuntime.unit.spec.ts index 34d2cffa..e5aae6c3 100644 --- a/test/apps/webapp/WebAppRuntime.unit.spec.ts +++ b/test/apps/webapp/WebAppRuntime.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { LOG_LEVEL_INFO, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { LiveSyncBrowserServiceHubOptions } from "@/apps/browser/createLiveSyncBrowserServiceHub"; @@ -168,7 +169,7 @@ describe("WebAppRuntime lifecycle", () => { it("owns core start, directory scan, status reporting, and shutdown", async () => { const reportStatus = vi.fn(); - const runtime = new WebAppRuntime(createRootHandle(), { reportStatus }); + const runtime = new WebAppRuntime(createRootHandle(), { reportStatus, deviceValues: new MemoryValueStore() }); await runtime.start(); @@ -193,7 +194,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("finalises application readiness after importing local files for optional P2P", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.currentSettings.mockReturnValue(unconfiguredSettings); @@ -239,7 +240,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not publish readiness after a partial local-file import", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.collectFilesOnStorage.mockResolvedValue({ @@ -269,7 +270,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not publish readiness when database initialisation rejects manual preparation", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.onDatabaseInitialised.mockResolvedValue(false); @@ -282,7 +283,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not publish readiness when pending file events cannot be committed", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.commitPendingFileEvents.mockResolvedValue(false); @@ -295,7 +296,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not repeat readiness finalisation after a successful manual scan", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); @@ -311,7 +312,7 @@ describe("WebAppRuntime lifecycle", () => { it("rejects a failed core start after cleaning up the partial runtime", async () => { runtimeMocks.onLoad.mockResolvedValue(false); const reportStatus = vi.fn(); - const runtime = new WebAppRuntime(createRootHandle(), { reportStatus }); + const runtime = new WebAppRuntime(createRootHandle(), { reportStatus, deviceValues: new MemoryValueStore() }); await expect(runtime.start()).rejects.toThrow("Failed to initialise LiveSync"); @@ -325,7 +326,7 @@ describe("WebAppRuntime lifecycle", () => { it("shuts down once before scheduling a host reload", async () => { const scheduleReload = vi.fn(); - const runtime = new WebAppRuntime(createRootHandle(), { scheduleReload }); + const runtime = new WebAppRuntime(createRootHandle(), { scheduleReload, deviceValues: new MemoryValueStore() }); await runtime.start(); runtimeMocks.options?.restart?.schedule(); diff --git a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts index 2d8f678c..693b1e3d 100644 --- a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts +++ b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { DEFAULT_SETTINGS, SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase"; @@ -27,6 +28,7 @@ describe("WebPeer runtime composition", () => { translate: (key) => `webpeer:${key}`, }); const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), context, store: createMemoryStore(), }); @@ -43,6 +45,7 @@ describe("WebPeer runtime composition", () => { it("loads settings and opens the local database before announcing layout readiness", async () => { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); const loadSettings = vi.spyOn(runtime.services.setting, "loadSettings").mockResolvedValue(undefined); @@ -68,6 +71,7 @@ describe("WebPeer runtime composition", () => { vi.useFakeTimers(); try { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); vi.spyOn(runtime.services.setting, "loadSettings").mockResolvedValue(undefined); @@ -77,9 +81,7 @@ describe("WebPeer runtime composition", () => { P2P_Enabled: true, }); vi.spyOn(runtime.services.database, "openDatabase").mockResolvedValue(true); - const onResumed = vi - .spyOn(runtime.services.appLifecycle, "onResumed") - .mockResolvedValue(true); + const onResumed = vi.spyOn(runtime.services.appLifecycle, "onResumed").mockResolvedValue(true); const open = vi.spyOn(runtime.p2p.transportLifecycle, "connect").mockResolvedValue(undefined); await runtime.start(); @@ -94,9 +96,11 @@ describe("WebPeer runtime composition", () => { it("rejects start when the browser-local database cannot be opened", async () => { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); vi.spyOn(runtime.services.setting, "loadSettings").mockResolvedValue(undefined); + vi.spyOn(runtime.services.setting, "currentSettings").mockReturnValue({ ...DEFAULT_SETTINGS }); vi.spyOn(runtime.services.database, "openDatabase").mockResolvedValue(false); await expect(runtime.start()).rejects.toThrow("WebPeer local database could not be opened"); @@ -104,6 +108,7 @@ describe("WebPeer runtime composition", () => { it("isolates a specialised runtime and applies its device name before opening the database", async () => { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), deviceName: " p2p-check-browser-desktop-abc ", systemVaultName: "p2p-check-vault", @@ -114,15 +119,39 @@ describe("WebPeer runtime composition", () => { P2P_AutoStart: false, P2P_Enabled: false, }); - const setSmallConfig = vi.spyOn(runtime.services.config, "setSmallConfig").mockImplementation(() => undefined); + let finishWrite!: () => void; + const setValue = vi.spyOn(runtime.services.config, "setValue").mockImplementation( + () => + new Promise((resolve) => { + finishWrite = resolve; + }) + ); const openDatabase = vi.spyOn(runtime.services.database, "openDatabase").mockImplementation(async () => { - expect(setSmallConfig).toHaveBeenCalledWith(SETTING_KEY_P2P_DEVICE_NAME, "p2p-check-browser-desktop-abc"); + expect(setValue).toHaveBeenCalledWith(SETTING_KEY_P2P_DEVICE_NAME, "p2p-check-browser-desktop-abc"); return true; }); - await runtime.start(); + const starting = runtime.start(); + await vi.waitFor(() => expect(setValue).toHaveBeenCalledOnce()); + expect(openDatabase).not.toHaveBeenCalled(); + finishWrite(); + await starting; expect(runtime.services.API.getSystemVaultName()).toBe("p2p-check-vault"); expect(openDatabase).toHaveBeenCalledOnce(); }); + + it("reads the current device name after the pane updates the store", async () => { + const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), + store: createMemoryStore(), + systemVaultName: "webpeer-vault", + }); + vi.spyOn(runtime.services.setting, "currentSettings").mockReturnValue({ ...DEFAULT_SETTINGS }); + expect(await runtime.getDeviceName()).toBe("webpeer-vault"); + await runtime.services.config.setValue(SETTING_KEY_P2P_DEVICE_NAME, "first-name"); + expect(await runtime.getDeviceName()).toBe("first-name"); + await runtime.services.config.setValue(SETTING_KEY_P2P_DEVICE_NAME, "renamed-device"); + expect(await runtime.getDeviceName()).toBe("renamed-device"); + }); }); diff --git a/test/contracts/serviceContext.ts b/test/contracts/serviceContext.ts index ddfb6ce4..ec712e09 100644 --- a/test/contracts/serviceContext.ts +++ b/test/contracts/serviceContext.ts @@ -17,13 +17,12 @@ export const SERVICE_CONTEXT_MEMBERS = [ "vault", "test", "UI", - "config", "keyValueDB", "control", -] as const satisfies readonly Exclude[]; +] as const satisfies readonly Exclude[]; export type ServiceContextMember = (typeof SERVICE_CONTEXT_MEMBERS)[number]; -type MissingServiceContextMember = Exclude, ServiceContextMember>; +type MissingServiceContextMember = Exclude, ServiceContextMember>; const serviceContextMembersAreExhaustive: [MissingServiceContextMember] extends [never] ? true : never = true; void serviceContextMembersAreExhaustive; diff --git a/test/e2e-obsidian/runner/liveSyncWorkflow.ts b/test/e2e-obsidian/runner/liveSyncWorkflow.ts index cde2d049..e256fc10 100644 --- a/test/e2e-obsidian/runner/liveSyncWorkflow.ts +++ b/test/e2e-obsidian/runner/liveSyncWorkflow.ts @@ -111,7 +111,7 @@ export async function readE2eCompatibilityMarker( return await evalObsidianJson( cliBinary, [ - "(()=>{", + "(async()=>{", "const core=app.plugins.plugins['obsidian-livesync'].core;", "const setting=core.services.setting;", "const settings=setting.currentSettings();", @@ -122,7 +122,7 @@ export async function readE2eCompatibilityMarker( "return JSON.stringify({", "vaultName,additionalSuffix,expectedStorageKey,", "rawStorageValue:localStorage.getItem(expectedStorageKey),", - "serviceValue:setting.getSmallConfig(markerKey),", + "serviceValue:await setting.config.getValue(markerKey,''),", "versionUpFlash:settings.versionUpFlash,", "});", "})()", diff --git a/test/e2e-obsidian/runner/upgradeWorkflow.ts b/test/e2e-obsidian/runner/upgradeWorkflow.ts index 76b6c7aa..1af10d32 100644 --- a/test/e2e-obsidian/runner/upgradeWorkflow.ts +++ b/test/e2e-obsidian/runner/upgradeWorkflow.ts @@ -288,7 +288,7 @@ export async function readRuntimeUpgradeState( "localDatabaseDocumentCount:databaseInfo.doc_count,", "nodeId:nodeInfo?.nodeid??replicator?.nodeid??'',", "legacyCompatibilityMarker:localStorage.getItem(`obsidian-live-sync-ver${vaultName}`),", - "compatibilityMarker:setting.getSmallConfig('database-compatibility-version')??'',", + "compatibilityMarker:await (setting.config?.getValue?.('database-compatibility-version','')??Promise.resolve(setting.getSmallConfig?.('database-compatibility-version')??'')),", "compatibilityStorageEntries:Object.fromEntries(Array.from({length:localStorage.length},(_,index)=>localStorage.key(index))", ".filter((key)=>key!==null)", ".filter(key=>key.startsWith('obsidian-live-sync-ver')||key.endsWith('-database-compatibility-version'))", @@ -335,7 +335,7 @@ export async function readRuntimeSettingsUpgradeState( "const migrationState=setting.getSettingsMigrationState?.();", "return JSON.stringify({", "pluginVersion:app.plugins.manifests['obsidian-livesync']?.version??'unknown',", - "compatibilityMarker:setting.getSmallConfig?.('database-compatibility-version')??'',", + "compatibilityMarker:await (setting.config?.getValue?.('database-compatibility-version','')??Promise.resolve(setting.getSmallConfig?.('database-compatibility-version')??'')),", "migrationState,", "settings:{", "isConfigured:settings.isConfigured,settingVersion:settings.settingVersion,", diff --git a/test/e2e-obsidian/scripts/dialog-mounts.ts b/test/e2e-obsidian/scripts/dialog-mounts.ts index 5f3e6307..2456d9cd 100644 --- a/test/e2e-obsidian/scripts/dialog-mounts.ts +++ b/test/e2e-obsidian/scripts/dialog-mounts.ts @@ -1,7 +1,9 @@ import { readFile } from "node:fs/promises"; import { join } from "node:path"; +import type { Page } from "playwright"; import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/setup-uri"; import { decryptString } from "@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption"; +import { SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { $msg } from "../../../src/common/translation.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { createE2eCouchDbPluginData, waitForLiveSyncCoreReady } from "../runner/liveSyncWorkflow.ts"; @@ -13,18 +15,34 @@ import { captureObsidianPage, obsidianRemoteDebuggingPort, openLiveSyncSettings, - withObsidianPage, + withObsidianPage as withObsidianSessionPage, } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; const dialogRunStateKey = "__livesyncE2EDialogMount"; +const savedP2PDeviceName = "e2e-p2p-saved-device"; const repairRunStateKey = "__livesyncE2ETroubleshootingRepair"; const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_DIALOG_TIMEOUT_MS ?? 10000); +async function withObsidianPage(port: number, operation: (page: Page) => Promise): Promise { + return await withObsidianSessionPage(port, async (page) => { + for (const candidate of page.context().pages()) { + const hasPlugin = await candidate + .evaluate( + () => (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"] !== undefined + ) + .catch(() => false); + if (hasPlugin) return await operation(candidate); + } + throw new Error("The LiveSync Vault renderer is unavailable."); + }); +} + type DialogueMode = "desktop" | "mobile"; type DialogueRunState = { done: boolean; + deviceNameRead?: boolean; error?: string; expected?: unknown; kind: string; @@ -46,6 +64,14 @@ type LiveSyncTestPlugin = { createAllChunks(force: boolean): Promise; }; modules: SetupManagerHandle[]; + services: { + setting: { + config: { + getValue(key: string, defaultValue?: string): Promise; + setValue(key: string, value: string): Promise; + }; + }; + }; settings: Record; }; }; @@ -91,39 +117,63 @@ async function openRemoteSelectionDialogue(): Promise { async function openP2PRemoteSelectionDialogueForInspection(): Promise { await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - await page.evaluate((stateKey) => { - const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; - if (plugin === undefined) throw new Error("Self-hosted LiveSync is not loaded"); - const manager = plugin.core.modules.find((module) => module.constructor.name === "SetupManager"); - if ( - typeof manager?.onSelectServer !== "function" || - typeof manager.onConfirmApplySettingsFromWizard !== "function" - ) { - throw new Error("Could not find the P2P setup workflow"); + await page.evaluate( + async ({ stateKey, deviceNameKey, deviceName }) => { + const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; + if (plugin === undefined) throw new Error("Self-hosted LiveSync is not loaded"); + const initialPeerName = plugin.core.settings.P2P_DevicePeerName; + if (typeof initialPeerName === "string" && initialPeerName.trim() !== "") { + throw new Error("The P2P setup fixture unexpectedly has an initial device name."); + } + const manager = plugin.core.modules.find((module) => module.constructor.name === "SetupManager"); + if ( + typeof manager?.onSelectServer !== "function" || + typeof manager.onConfirmApplySettingsFromWizard !== "function" + ) { + throw new Error("Could not find the P2P setup workflow"); + } + const config = plugin.core.services.setting.config; + await config.setValue(deviceNameKey, deviceName); + const originalGetValue = config.getValue; + const originalConfirm = manager.onConfirmApplySettingsFromWizard; + const state: DialogueRunState = { + kind: "p2p-compatibility-settings", + done: false, + deviceNameRead: false, + }; + (globalThis as unknown as Record)[stateKey] = state; + config.getValue = async (key, defaultValue) => { + const value = await originalGetValue.call(config, key, defaultValue); + if (key === deviceNameKey && value === deviceName) state.deviceNameRead = true; + return value; + }; + manager.onConfirmApplySettingsFromWizard = async (settings: unknown) => { + state.expected = settings; + return true; + }; + void manager + .onSelectServer(plugin.core.settings, "unknown") + .then( + (result) => { + state.result = result; + state.done = true; + }, + (error: unknown) => { + state.error = error instanceof Error ? error.message : String(error); + state.done = true; + } + ) + .finally(() => { + manager.onConfirmApplySettingsFromWizard = originalConfirm; + config.getValue = originalGetValue; + }); + }, + { + stateKey: dialogRunStateKey, + deviceNameKey: SETTING_KEY_P2P_DEVICE_NAME, + deviceName: savedP2PDeviceName, } - const originalConfirm = manager.onConfirmApplySettingsFromWizard; - const state: DialogueRunState = { kind: "p2p-compatibility-settings", done: false }; - (globalThis as unknown as Record)[stateKey] = state; - manager.onConfirmApplySettingsFromWizard = async (settings: unknown) => { - state.expected = settings; - return true; - }; - void manager - .onSelectServer(plugin.core.settings, "unknown") - .then( - (result) => { - state.result = result; - state.done = true; - }, - (error: unknown) => { - state.error = error instanceof Error ? error.message : String(error); - state.done = true; - } - ) - .finally(() => { - manager.onConfirmApplySettingsFromWizard = originalConfirm; - }); - }, dialogRunStateKey); + ); }); } @@ -388,7 +438,7 @@ async function verifyRemoteSelectionDialogue(mode: DialogueMode): Promise !element.isConnected, previousDialogue, { timeout: uiTimeoutMs }); await modal.waitFor({ state: "visible", timeout: uiTimeoutMs }); } @@ -495,9 +545,21 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { has: page.locator(".modal-title").filter({ hasText: "P2P Configuration" }), }); await modal.waitFor({ state: "visible", timeout: uiTimeoutMs }); + const peerName = modal.locator('input[name="p2p-device-peer-id"]'); + await peerName.waitFor({ state: "visible", timeout: uiTimeoutMs }); + await page.waitForFunction( + (expectedName) => { + const input = document.querySelector('input[name="p2p-device-peer-id"]'); + return input instanceof HTMLInputElement && !input.disabled && input.value === expectedName; + }, + savedP2PDeviceName, + { timeout: uiTimeoutMs } + ); + if ((await peerName.inputValue()) !== savedP2PDeviceName) { + throw new Error("The saved P2P device name was not loaded before submitting the setup dialogue."); + } await modal.locator('input[name="p2p-room-id"]').fill("e2e-p2p-compatibility"); await modal.locator('input[name="p2p-password"]').fill("e2e-passphrase"); - await modal.locator('input[name="p2p-device-peer-id"]').fill("e2e-device"); const compatibility = modal.locator("details").filter({ has: page.locator("summary").filter({ hasText: "Connection compatibility" }), @@ -565,8 +627,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { { timeout: uiTimeoutMs } ); const resetNotice = compatibility.locator(".sls-info-note-notice").filter({ - hasText: - "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.", + hasText: "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.", }); await resetNotice.waitFor({ state: "visible", timeout: uiTimeoutMs }); await resetNotice @@ -594,10 +655,16 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { }); const state = await assertDialogueRunCompleted(); + if (state.deviceNameRead !== true) { + throw new Error("The saved P2P device name was not read through setting.config."); + } if (state.result !== true || typeof state.expected !== "object" || state.expected === null) { throw new Error("The P2P compatibility settings were not passed to the setup workflow."); } const settings = state.expected as Record; + if (settings.P2P_DevicePeerName !== savedP2PDeviceName) { + throw new Error("The saved P2P device name was not passed through the setup workflow."); + } if (settings.P2P_maxWirePayloadBytes !== 800 || settings.P2P_connectionPath !== "relay") { throw new Error( `The P2P compatibility choices did not round-trip through the setup dialogue: ${JSON.stringify({ @@ -672,15 +739,14 @@ async function verifyGenerateSetupUriDialogue(mode: DialogueMode): Promise - (globalThis as ObsidianTestGlobal).app?.commands?.executeCommandById(commandId) === true, + (commandId) => (globalThis as ObsidianTestGlobal).app?.commands?.executeCommandById(commandId) === true, "obsidian-livesync:livesync-copysetupuri" ); }); @@ -714,7 +780,8 @@ async function verifyGenerateSetupUriDialogue(mode: DialogueMode): Promise { if (!(await toggle.evaluate((element) => element.classList.contains("is-enabled")))) { throw new Error("The automatic compatible-setting policy was displayed as disabled while still undefined."); } + await settingsNavigator.close(); }); } @@ -1094,9 +1162,10 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { ); await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - const liveSyncSettings = page.locator(".sls-setting"); + const settingsNavigator = await openLiveSyncSettings(page, uiTimeoutMs); + const liveSyncSettings = await settingsNavigator.openPage("Hatch"); const logSetting = liveSyncSettings.locator(".setting-item").filter({ - has: page.getByText("Write logs into the file", { exact: true }), + has: settingsNavigator.page.getByText("Write logs into the file", { exact: true }), }); await logSetting.locator(".checkbox-container").click({ timeout: uiTimeoutMs }); await page.waitForFunction( @@ -1130,10 +1199,10 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { // Saving a toggle refreshes the settings pane. Resolve the visible control again so the // second action does not target the detached pre-save element. - const refreshedLogToggle = page - .locator(".sls-setting:visible .setting-item:visible") + const refreshedLogToggle = liveSyncSettings + .locator(".setting-item:visible") .filter({ - has: page.getByText("Write logs into the file", { exact: true }), + has: settingsNavigator.page.getByText("Write logs into the file", { exact: true }), }) .locator(".checkbox-container:visible") .last(); @@ -1179,13 +1248,9 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { } }; }, repairRunStateKey); - await page - .locator(".sls-setting:visible") - .last() - .getByRole("button", { name: "Recreate current chunks", exact: true }) - .click({ - timeout: uiTimeoutMs, - }); + await liveSyncSettings.getByRole("button", { name: "Recreate current chunks", exact: true }).click({ + timeout: uiTimeoutMs, + }); await page.waitForFunction( (stateKey) => (globalThis as unknown as Record)[stateKey]?.done === true, @@ -1200,13 +1265,9 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { throw new Error(`Recreate missing chunks failed: ${repairState.error}`); } - await page - .locator(".sls-setting:visible") - .last() - .getByRole("button", { name: "Begin inspection", exact: true }) - .click({ - timeout: uiTimeoutMs, - }); + await liveSyncSettings.getByRole("button", { name: "Begin inspection", exact: true }).click({ + timeout: uiTimeoutMs, + }); await page .locator(".notice") .filter({ hasText: /^done$/u }) @@ -1214,6 +1275,7 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { state: "visible", timeout: uiTimeoutMs, }); + await settingsNavigator.close(); }); return screenshotPath; diff --git a/test/e2e-obsidian/scripts/review-harness.ts b/test/e2e-obsidian/scripts/review-harness.ts index 49e22c67..64631db9 100644 --- a/test/e2e-obsidian/scripts/review-harness.ts +++ b/test/e2e-obsidian/scripts/review-harness.ts @@ -372,13 +372,21 @@ async function restartAndResumeHarness(): Promise { await harness .locator('[data-testid="review-harness-resumed"]') .waitFor({ state: "visible", timeout: uiTimeoutMs }); - const continuationRemoved = await page.evaluate((stateKey) => { + const continuationRemoved = await page.evaluate(async (stateKey) => { const plugin = (globalThis as ReviewHarnessTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; if (typeof plugin !== "object" || plugin === null || !("core" in plugin)) { throw new Error("Self-hosted LiveSync is unavailable after restart."); } - const core = (plugin as { core: { services: { setting: { getSmallConfig(key: string): string } } } }).core; - return core.services.setting.getSmallConfig(stateKey) === ""; + const core = ( + plugin as { + core: { + services: { + setting: { config: { getValue(key: string, defaultValue: string): Promise } }; + }; + }; + } + ).core; + return (await core.services.setting.config.getValue(stateKey, "")) === ""; }, REVIEW_HARNESS_STATE_KEY); if (!continuationRemoved) throw new Error("The one-shot continuation was not removed before use."); await assertNoHorizontalOverflow(page, harness, { label: "resumed Review Harness" }); diff --git a/test/e2e-obsidian/scripts/settings-ui.ts b/test/e2e-obsidian/scripts/settings-ui.ts index c4be1651..0b930234 100644 --- a/test/e2e-obsidian/scripts/settings-ui.ts +++ b/test/e2e-obsidian/scripts/settings-ui.ts @@ -58,8 +58,9 @@ type LiveSyncTestPlugin = { usePowerUserMode: boolean; useEdgeCaseMode: boolean; hashCacheMaxCount: number; + customChunkSize: number; }; - getSmallConfig(key: string): string | null; + config: { getValue(key: string, defaultValue: string): Promise }; }; }; }; @@ -484,12 +485,12 @@ async function verifyCompatibilityReview(): Promise { }); await withObsidianPage(port, async (page) => { - const markerBeforeAcknowledgement = await page.evaluate(() => { + const markerBeforeAcknowledgement = await page.evaluate(async () => { const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; if (plugin === undefined) throw new Error("Self-hosted LiveSync is unavailable"); - return plugin.core.services.setting.getSmallConfig("database-compatibility-version"); + return await plugin.core.services.setting.config.getValue("database-compatibility-version", ""); }); - if (markerBeforeAcknowledgement !== null && markerBeforeAcknowledgement !== "") { + if (markerBeforeAcknowledgement !== "") { throw new Error( `The database version was marked as acknowledged before review: ${markerBeforeAcknowledgement}` ); @@ -579,12 +580,12 @@ async function verifyCompatibilityReview(): Promise { .click(); await summary.waitFor({ state: "hidden", timeout: uiTimeoutMs }); await page.waitForFunction( - (expectedVersion) => { + async (expectedVersion) => { const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; if (plugin === undefined) return false; const setting = plugin.core.services.setting; return ( - setting.getSmallConfig("database-compatibility-version") === expectedVersion && + (await setting.config.getValue("database-compatibility-version", "")) === expectedVersion && setting.currentSettings().versionUpFlash === "" ); }, @@ -611,10 +612,16 @@ async function verifyConfigDoctorFollowsCompatibilityReview(): Promise { state: "visible", timeout: uiTimeoutMs, }); - await doctor.getByText("Enhance chunk size", { exact: true }).waitFor({ - state: "visible", - timeout: uiTimeoutMs, + const chunkSize = await page.evaluate(() => { + const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; + return plugin?.core.services.setting.currentSettings().customChunkSize; }); + if (chunkSize !== 0) { + throw new Error("Compatibility review must retain the fixture's zero chunk-size value."); + } + if ((await doctor.getByText("Enhance chunk size", { exact: true }).count()) !== 0) { + throw new Error("Config Doctor must accept the recommended zero chunk-size value."); + } if ((await doctor.getByText("Data Compression", { exact: true }).count()) !== 0) { throw new Error("Config Doctor still treats supported Data Compression as a problem."); } @@ -978,6 +985,7 @@ function createSettingsPluginData(settingsOnlyRun: boolean): Record Date: Fri, 23 Oct 2026 14:40:01 +0000 Subject: [PATCH 11/28] Close WebPeer runtimes after composition tests --- test/apps/webpeer/WebPeerRuntime.unit.spec.ts | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts index 693b1e3d..73362b39 100644 --- a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts +++ b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts @@ -3,9 +3,9 @@ import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { DEFAULT_SETTINGS, SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase"; import { EVENT_LAYOUT_READY } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { WebPeerRuntime } from "@/apps/webpeer/src/WebPeerRuntime"; +import { WebPeerRuntime, type WebPeerRuntimeOptions } from "@/apps/webpeer/src/WebPeerRuntime"; function createMemoryStore(): SimpleStore { const values = new Map(); @@ -23,11 +23,30 @@ function createMemoryStore(): SimpleStore { } describe("WebPeer runtime composition", () => { + const runtimes = new Set(); + + function createRuntime(options: WebPeerRuntimeOptions): WebPeerRuntime { + const runtime = new WebPeerRuntime(options); + runtimes.add(runtime); + return runtime; + } + + afterEach(async () => { + try { + for (const runtime of runtimes) { + await runtime.shutdown(); + } + } finally { + runtimes.clear(); + vi.restoreAllMocks(); + } + }); + it("preserves one context and live P2P result across the service graph and pane host", () => { const context = createServiceContext({ translate: (key) => `webpeer:${key}`, }); - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), context, store: createMemoryStore(), @@ -44,7 +63,7 @@ describe("WebPeer runtime composition", () => { }); it("loads settings and opens the local database before announcing layout readiness", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); @@ -70,7 +89,7 @@ describe("WebPeer runtime composition", () => { it("delegates automatic P2P startup to the resumed lifecycle handler", async () => { vi.useFakeTimers(); try { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); @@ -95,7 +114,7 @@ describe("WebPeer runtime composition", () => { }); it("rejects start when the browser-local database cannot be opened", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); @@ -107,7 +126,7 @@ describe("WebPeer runtime composition", () => { }); it("isolates a specialised runtime and applies its device name before opening the database", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), deviceName: " p2p-check-browser-desktop-abc ", @@ -142,7 +161,7 @@ describe("WebPeer runtime composition", () => { }); it("reads the current device name after the pane updates the store", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), systemVaultName: "webpeer-vault", From fc2b35e0dc8538a639ba7efd8bcfd97001fc769d Mon Sep 17 00:00:00 2001 From: s-qin Date: Fri, 23 Oct 2026 23:54:13 +0800 Subject: [PATCH 12/28] perf(startup): inspect storage sizes without loading healthy files --- .../startupLifecycle/incompleteDocuments.ts | 33 ++++++++++++----- .../incompleteDocuments.unit.spec.ts | 36 +++++++++++++++++++ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts index 9836d682..0b4ee3de 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts @@ -31,7 +31,7 @@ export interface IncompleteDocumentsDependencies { readonly localDatabase: Pick; readonly getPath: StartupPathReader; readonly isTargetFile: (path: string) => Promise; - readonly storageAccess: Pick; + readonly storageAccess: Pick; readonly fileHandler: Pick; readonly keyValueDB: Pick; readonly noticeGroups: NoticeGroups; @@ -94,16 +94,11 @@ export async function checkIncompleteDocuments( } const isConflicted = metaDoc?._conflicts && metaDoc._conflicts.length > 0; - let storageFileContent; - try { - storageFileContent = await dependencies.storageAccess.readHiddenFileBinary(path); - } catch (e) { - Logger(`Failed to read file ${path}: Possibly unprocessed or missing`); - Logger(e, LOG_LEVEL_VERBOSE); + const storageStat = await dependencies.storageAccess.statHidden(path); + if (!storageStat || storageStat.type !== "file") { continue; } - // const storageFileBlob = createBlob(storageFileContent); - const sizeOnStorage = storageFileContent.byteLength; + let sizeOnStorage = storageStat.size; const recordedSize = doc.size; const docBlob = readAsBlob(doc); const actualSize = docBlob.size; @@ -113,6 +108,15 @@ export async function checkIncompleteDocuments( sizeOnStorage !== recordedSize || isConflicted ) { + let storageFileContent; + try { + storageFileContent = await dependencies.storageAccess.readHiddenFileBinary(path); + } catch (e) { + Logger(`Failed to read file ${path}: Possibly unprocessed or missing`); + Logger(e, LOG_LEVEL_VERBOSE); + continue; + } + sizeOnStorage = storageFileContent.byteLength; const contentMatched = await isDocContentSame(doc.data, storageFileContent); errorFiles.push({ path, @@ -125,6 +129,17 @@ export async function checkIncompleteDocuments( Logger( `Size mismatch for ${path}: ${recordedSize} (DB Recorded) , ${actualSize} (DB Stored) , ${sizeOnStorage} (Storage Stored), ${contentMatched ? "Content Matched" : "Content Mismatched"} ${isConflicted ? "Conflicted" : "Not Conflicted"}` ); + } else { + const currentStat = await dependencies.storageAccess.statHidden(path); + if ( + !currentStat || + currentStat.type !== "file" || + currentStat.size !== storageStat.size || + currentStat.mtime !== storageStat.mtime || + currentStat.ctime !== storageStat.ctime + ) { + throw new Error("Storage changed during the incomplete document check"); + } } } if (errorFiles.length == 0) { diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts index f40305c4..2f30f12b 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts @@ -99,6 +99,7 @@ function createDependencies(findAllNormalDocs: FindAllNormalDocs = noDocuments, isTargetFile: vi.fn(async () => true), storageAccess: { readHiddenFileBinary, + statHidden: vi.fn(async (path: string) => getFixture(path)?.stub.stat ?? null), getFileStub, }, fileHandler: { storeFileToDB }, @@ -157,6 +158,41 @@ describe("checkIncompleteDocuments", () => { expect(noticeGroups.finish).toHaveBeenCalledWith("startup-integrity-check"); }); + it("does not read healthy storage contents only to determine their size", async () => { + const healthy = documentFixture("healthy.md", { storageContent: "abc" }); + const fixture = createDependencies(documentsFrom([healthy]), [healthy]); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).not.toHaveBeenCalled(); + expect(fixture.askSelectStringDialogue).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).toHaveBeenCalledWith("checkIncompleteDocs", true); + }); + + it("does not mark a scan complete when storage changes during the size check", async () => { + const healthy = documentFixture("changing.md", { storageContent: "abc" }); + const fixture = createDependencies(documentsFrom([healthy]), [healthy]); + vi.mocked(fixture.dependencies.storageAccess.statHidden) + .mockResolvedValueOnce(healthy.stub.stat) + .mockResolvedValueOnce({ ...healthy.stub.stat, mtime: 3 }); + + await expect(checkIncompleteDocuments(fixture.dependencies)).rejects.toThrow("Storage changed"); + + expect(fixture.readHiddenFileBinary).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); + }); + + it("still reads conflicted storage for the existing repair dialogue", async () => { + const conflicted = documentFixture("conflicted.md", { storageContent: "abc", conflicts: ["2-other"] }); + const fixture = createDependencies(documentsFrom([conflicted]), [conflicted]); + fixture.askSelectStringDialogue.mockImplementation(selectButton(0)); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("conflicted.md"); + expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); + }); + it("repairs a recoverable document when FIX is selected", async () => { const recoverable = documentFixture("recoverable.md"); const fixture = createDependencies(documentsFrom([recoverable]), [recoverable]); From 39ef429d7bcefdc062320b43200b40c683b9f54c Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 16:07:35 +0000 Subject: [PATCH 13/28] Clarify device review guidance and follow dialogue windows --- docs/settings.md | 2 +- .../messages/LiveSyncProvisionalMessages.ts | 13 ++-- .../SettingDialogue/PaneRemoteConfig.ts | 2 +- .../dialogues/JoinedDevicesDialog.svelte | 2 +- src/serviceFeatures/useJoinedDevices.ts | 22 ++++-- test/e2e-obsidian/scripts/joined-devices.ts | 76 ++++++++++++++----- updates.md | 4 +- 7 files changed, 81 insertions(+), 40 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index c2b33e09..1a81032c 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -236,7 +236,7 @@ The active connection type. This is automatically projected to the legacy config ### Joined devices -Open **Joined devices** to read the device records on the active CouchDB or Object Storage connection. The same dialogue is available from the command palette and version Notices. **Refresh** reads the remote again. This device appears first, including when its record is absent from the remote. Opening or refreshing this dialogue does not create a missing CouchDB database. +Choose **Review devices** under **Joined devices** to read the device records on the active CouchDB or Object Storage connection. The same **Review devices** action is available from the command palette and version Notices. **Refresh** reads the remote again. This device appears first, including when its record is absent from the remote. Opening or refreshing this dialogue does not create a missing CouchDB database. The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 44d4c2bc..054bb7ba 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -26,8 +26,8 @@ export const liveSyncProvisionalEnglishMessages = { "This remote does not currently have a record of joined devices.", "This list shows device information recorded by the remote. It does not show whether a device is online now.": "This list shows device information recorded by the remote. It does not show whether a device is online now.", - "This device is locked out. Use the existing recovery process to join again.": - "This device is locked out. Use the existing recovery process to join again.", + "This device has been locked out. Use the existing recovery process to join again.": + "This device has been locked out. Use the existing recovery process to join again.", "This device": "This device", "This remote has been cleaned. Use the existing recovery process to join devices again as needed.": "This remote has been cleaned. Use the existing recovery process to join devices again as needed.", @@ -42,10 +42,10 @@ export const liveSyncProvisionalEnglishMessages = { Refresh: "Refresh", "Plug-in version": "Plug-in version", Platform: "Platform", - "Other joined devices last reported an older plug-in version: ${versions}.": - "Other joined devices last reported an older plug-in version: ${versions}.", - "Other joined devices last reported a newer plug-in version: ${versions}.": - "Other joined devices last reported a newer plug-in version: ${versions}.", + "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.": + "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", + "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.": + "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", Open: "Open", "Not recorded": "Not recorded", "Not accepted": "Not accepted", @@ -58,6 +58,7 @@ export const liveSyncProvisionalEnglishMessages = { "Joined devices are available after database startup with an active CouchDB or Object Storage connection.": "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", "Joined devices": "Joined devices", + "Review devices": "Review devices", "Forgetting…": "Forgetting…", "Forgetting the record does not delete Vault files or revoke credentials.": "Forgetting the record does not delete Vault files or revoke credentials.", diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts index 41fd1df8..81a68312 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts @@ -106,7 +106,7 @@ export function paneRemoteConfig( .setName($msg("Joined devices")) .setDesc($msg("Review device records for the active CouchDB or Object Storage connection.")) .addButton((button) => - button.setButtonText($msg("Open")).onClick(async () => { + button.setButtonText($msg("Review devices")).onClick(async () => { const result = await this.services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); if (result.status === "unavailable" || result.value === "unavailable") { this.services.API.addLog( diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte index 31b30356..91c54b0f 100644 --- a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -176,7 +176,7 @@ {#if device.self && device.lockedOut}

{translate( - "This device is locked out. Use the existing recovery process to join again." + "This device has been locked out. Use the existing recovery process to join again." )}

{/if} diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts index 37d50b0f..c5fcc759 100644 --- a/src/serviceFeatures/useJoinedDevices.ts +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -62,14 +62,20 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi notices.setItem(noticeGroup, direction, { message: direction === "newer" - ? $msg("Other joined devices last reported a newer plug-in version: ${versions}.", { - versions: versions.join(", "), - }) - : $msg("Other joined devices last reported an older plug-in version: ${versions}.", { - versions: versions.join(", "), - }), + ? $msg( + "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", + { + versions: versions.join(", "), + } + ) + : $msg( + "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", + { + versions: versions.join(", "), + } + ), action: { - label: $msg("Joined devices"), + label: $msg("Review devices"), onSelect: () => { if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); @@ -190,7 +196,7 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi services.appLifecycle.onInitialise.addHandler(() => { services.API.addCommand({ id: "joined-devices", - name: $msg("Joined devices"), + name: $msg("Review devices"), icon: "monitor-smartphone", checkCallback: (checking) => { if (!available()) return false; diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index d5736139..15d496e8 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -1,3 +1,5 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; import { DEVICE_ID_PREFERRED, MILESTONE_DOCID, NODEINFO_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { evalObsidianJson } from "../runner/cli.ts"; import { @@ -27,12 +29,7 @@ import { } from "@vrtmrz/obsidian-test-session"; import { iPhoneSafeArea, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; -import { - captureObsidianDialogue, - openLiveSyncSettings, - waitForVisibleObsidianDialogue, - withObsidianPage, -} from "../runner/ui.ts"; +import { openLiveSyncSettings, waitForVisibleObsidianDialogue, withObsidianPage } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; import type { Locator, Page } from "playwright"; @@ -161,7 +158,9 @@ function createFixture(self: SelfDeviceInfo): DeviceFixture { async function openFromRemoteSettings(page: Page): Promise { const settings = await openLiveSyncSettings(page, uiTimeoutMs); const remoteConfiguration = await settings.openPage("Remote Configuration"); - await remoteConfiguration.getByRole("button", { name: "Open", exact: true }).click({ timeout: uiTimeoutMs }); + await remoteConfiguration + .getByRole("button", { name: "Review devices", exact: true }) + .click({ timeout: uiTimeoutMs }); const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await dialogue.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); return dialogue; @@ -185,7 +184,7 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa await selfCard.getByText("Locked out", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); await selfCard.getByText("Not accepted", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); await selfCard - .getByText("This device is locked out.", { exact: false }) + .getByText("This device has been locked out.", { exact: false }) .waitFor({ state: "visible", timeout: uiTimeoutMs }); await assertDetail(selfCard, "Platform", self.platform); await assertDetail(selfCard, "Plug-in version", self.pluginVersion); @@ -339,6 +338,27 @@ async function dismissNotices(page: Page): Promise { assertEqual(await notices.count(), 0, "Notices still obscured the dialogue screenshot."); } +async function captureJoinedDevices( + port: number, + filename: string, + assertReady: (page: Page) => Promise +): Promise { + const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e"; + await mkdir(outputDirectory, { recursive: true }); + await withObsidianPage(port, async (page) => { + try { + const dialogue = await assertReady(page); + await dialogue.page().screenshot({ path: join(outputDirectory, filename), fullPage: true }); + } catch (error) { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", 500).catch(() => undefined); + const failurePath = join(outputDirectory, filename.replace(/\.png$/u, ".failure.png")); + await (dialogue?.page() ?? page).screenshot({ path: failurePath, fullPage: true }).catch(() => undefined); + console.error(`Joined devices UI failure screenshot: ${failurePath}`); + throw error; + } + }); +} + async function main(): Promise { const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); @@ -381,24 +401,35 @@ async function main(): Promise { assert(Boolean(preparedMilestone._rev), "Remote preparation did not create a revisioned milestone document."); await putCouchDbDocument(couchDb, dbName, { ...fixture.milestone, _rev: preparedMilestone._rev }); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { - await page.setViewportSize({ width: 1280, height: 1100 }); + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { const dialogue = await openFromRemoteSettings(page); - await assertInitialDialogue(dialogue, self, page); - await page + const dialoguePage = dialogue.page(); + await dialoguePage.setViewportSize({ width: 1280, height: 1100 }); + await assertInitialDialogue(dialogue, self, dialoguePage); + await dialoguePage .locator(".notice") - .getByText("Other joined devices last reported a newer plug-in version: 999.0.0.", { exact: true }) + .getByText( + "A newer plug-in version was detected in other device records: 999.0.0. We recommend checking whether this device needs updating.", + { exact: true } + ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); + return dialogue; }); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await dialogue.getByRole("button", { name: "Close", exact: true }).click({ timeout: uiTimeoutMs }); - await page.locator(".notice").getByRole("button", { name: "Joined devices", exact: true }).first().click(); + await dialogue + .page() + .locator(".notice") + .getByRole("button", { name: "Review devices", exact: true }) + .first() + .click(); const reopened = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await reopened.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); - await dismissNotices(page); + await dismissNotices(reopened.page()); + return reopened; }); - await captureObsidianDialogue( + await captureJoinedDevices( session.remoteDebuggingPort, "joined-devices-forget-confirmation.png", async (page) => { @@ -406,12 +437,14 @@ async function main(): Promise { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await showForgetConfirmation(dialogue); await deviceCard(dialogue, targetId).locator(".forget-confirmation").scrollIntoViewIfNeeded(); + return dialogue; } ); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-after-forget.png", async (page) => { + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-after-forget.png", async (page) => { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await confirmForgetTarget(dialogue); await dialogue.locator(".modal-content").evaluate((element) => element.scrollTo(0, 0)); + return dialogue; }); await withObsidianPage(session.remoteDebuggingPort, async (page) => { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); @@ -423,7 +456,7 @@ async function main(): Promise { await setObsidianMobileTestMode(session.remoteDebuggingPort, true, uiTimeoutMs); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-mobile.png", async (page) => { + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-mobile.png", async (page) => { const opened = await page.evaluate((id) => { const app = ( window as typeof window & { @@ -435,13 +468,14 @@ async function main(): Promise { assert(opened, "The Joined devices command did not open the dialogue."); const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await dialogue.getByText(retainedId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); - await assertScrollableMobileDialogue(page, dialogue); + await assertScrollableMobileDialogue(dialogue.page(), dialogue); assertEqual( await dialogue.locator(".device-card").count(), 2, "The refreshed mobile list did not retain both remaining devices." ); - await dismissNotices(page); + await dismissNotices(dialogue.page()); + return dialogue; }); console.log(`Joined devices scenario passed against CouchDB database ${dbName}.`); diff --git a/updates.md b/updates.md index a17be86e..534eac87 100644 --- a/updates.md +++ b/updates.md @@ -27,9 +27,9 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi #### Added -- We can review participating devices through **Joined devices** in Remote Configuration, the command palette, or a version Notice. +- We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. - - Advisory Notices help us find devices which last reported newer or older plug-in versions. + - Advisory Notices highlight newer or older plug-in versions detected in device records and help us check which devices may need updating. - We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. From 4f7a6b7ae9ccabcc0b8248db6edbc7142fc04af6 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 16:32:12 +0000 Subject: [PATCH 14/28] Explain compatibility costs in device review --- docs/settings.md | 2 +- src/common/messages/LiveSyncProvisionalMessages.ts | 10 ++++++---- .../LiveSyncUI/dialogues/JoinedDevicesDialog.svelte | 5 +++++ src/serviceFeatures/useJoinedDevices.ts | 4 ++-- test/e2e-obsidian/scripts/joined-devices.ts | 9 ++++++++- updates.md | 2 +- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 1a81032c..330815e1 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -240,7 +240,7 @@ Choose **Review devices** under **Joined devices** to read the device records on The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. -Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices are advisory and do not stop synchronisation. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. +Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices recommend reviewing and aligning plug-in versions across devices; they are advisory and do not stop synchronisation. The dialogue explains that even compatible versions can reduce Chunk reuse and increase storage use when files are rewritten, while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. **Forget device** removes another device's participation records after confirmation. It preserves the remote lock state and every other device. It does not delete Vault files or revoke credentials. A device can register again on an unlocked remote; on a locked remote, the existing recovery process applies. The result is checked after saving to CouchDB; if it cannot be confirmed, refresh the list and try again. CouchDB stores these records together in a local document which does not reject simultaneous updates, so changes from other devices can be overwritten. Pausing synchronisation on the other devices avoids that overlap. Object Storage uses conditional updates; if the server cannot support them, the record is retained. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 054bb7ba..40dd3d34 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -42,10 +42,10 @@ export const liveSyncProvisionalEnglishMessages = { Refresh: "Refresh", "Plug-in version": "Plug-in version", Platform: "Platform", - "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.": - "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", - "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.": - "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", + "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": + "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", + "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": + "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", Open: "Open", "Not recorded": "Not recorded", "Not accepted": "Not accepted", @@ -59,6 +59,8 @@ export const liveSyncProvisionalEnglishMessages = { "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", "Joined devices": "Joined devices", "Review devices": "Review devices", + "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.": + "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", "Forgetting…": "Forgetting…", "Forgetting the record does not delete Vault files or revoke credentials.": "Forgetting the record does not delete Vault files or revoke credentials.", diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte index 91c54b0f..67b5a04f 100644 --- a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -2,6 +2,7 @@ import { onDestroy, onMount } from "svelte"; import type { GuestDialogProps } from "@/modules/services/LiveSyncUI/svelteDialog"; import DialogHeader from "@/modules/services/LiveSyncUI/components/DialogHeader.svelte"; + import InfoNote from "@/modules/services/LiveSyncUI/components/InfoNote.svelte"; import { $msg as translate } from "@/common/translation"; import type { JoinedDeviceRow, @@ -121,6 +122,10 @@ {translate("This list shows device information recorded by the remote. It does not show whether a device is online now.")}

+ + {#if view} {#if !view.milestonePresent}

{translate("This remote does not currently have a record of joined devices.")}

diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts index c5fcc759..424f5ce9 100644 --- a/src/serviceFeatures/useJoinedDevices.ts +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -63,13 +63,13 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi message: direction === "newer" ? $msg( - "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", + "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", { versions: versions.join(", "), } ) : $msg( - "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", + "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", { versions: versions.join(", "), } diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index 15d496e8..9cca9011 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -213,6 +213,13 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa { exact: true } ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dialogue + .locator(".sls-info-note") + .getByText( + "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", + { exact: true } + ) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); await dialogue .getByText("This remote is locked.", { exact: false }) .waitFor({ state: "visible", timeout: uiTimeoutMs }); @@ -409,7 +416,7 @@ async function main(): Promise { await dialoguePage .locator(".notice") .getByText( - "A newer plug-in version was detected in other device records: 999.0.0. We recommend checking whether this device needs updating.", + "A newer plug-in version was detected in other device records: 999.0.0. Reviewing and aligning the plug-in versions across devices is recommended.", { exact: true } ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); diff --git a/updates.md b/updates.md index 534eac87..161bb640 100644 --- a/updates.md +++ b/updates.md @@ -29,7 +29,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. - - Advisory Notices highlight newer or older plug-in versions detected in device records and help us check which devices may need updating. + - Advisory Notices highlight newer or older plug-in versions detected in device records and recommend reviewing and aligning our versions. The dialogue explains possible storage costs and P2P compatibility limits. - We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. From b40784e66f8ebfcb46af7b671218bab4dfa9d7b9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 16:41:21 +0000 Subject: [PATCH 15/28] Shorten device version compatibility guidance --- docs/settings.md | 2 +- src/common/messages/LiveSyncProvisionalMessages.ts | 4 ++-- .../services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte | 2 +- test/e2e-obsidian/scripts/joined-devices.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 330815e1..c65b2949 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -240,7 +240,7 @@ Choose **Review devices** under **Joined devices** to read the device records on The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. -Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices recommend reviewing and aligning plug-in versions across devices; they are advisory and do not stop synchronisation. The dialogue explains that even compatible versions can reduce Chunk reuse and increase storage use when files are rewritten, while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. +Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices recommend reviewing and aligning plug-in versions across devices; they are advisory and do not stop synchronisation. The dialogue explains that synchronisation between different versions is generally safe unless the connection is explicitly blocked, but may be less efficient or use more storage. P2P synchronisation may also be unavailable. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. **Forget device** removes another device's participation records after confirmation. It preserves the remote lock state and every other device. It does not delete Vault files or revoke credentials. A device can register again on an unlocked remote; on a locked remote, the existing recovery process applies. The result is checked after saving to CouchDB; if it cannot be confirmed, refresh the list and try again. CouchDB stores these records together in a local document which does not reject simultaneous updates, so changes from other devices can be overwritten. Pausing synchronisation on the other devices avoids that overlap. Object Storage uses conditional updates; if the server cannot support them, the record is retained. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 40dd3d34..c35a9c8d 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -59,8 +59,8 @@ export const liveSyncProvisionalEnglishMessages = { "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", "Joined devices": "Joined devices", "Review devices": "Review devices", - "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.": - "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", + "Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.": + "Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.", "Forgetting…": "Forgetting…", "Forgetting the record does not delete Vault files or revoke credentials.": "Forgetting the record does not delete Vault files or revoke credentials.", diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte index 67b5a04f..7ae1b3ac 100644 --- a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -123,7 +123,7 @@

{#if view} diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index 9cca9011..b53ef035 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -216,7 +216,7 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa await dialogue .locator(".sls-info-note") .getByText( - "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", + "Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.", { exact: true } ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); From 1a54d21e712497b64879957ecd6b86608a021076 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 17:37:42 +0000 Subject: [PATCH 16/28] test: cover Commonlib 0.2 consumer workflows --- package.json | 1 + .../useJoinedDevices.unit.spec.ts | 56 +++++++ test/e2e-obsidian/README.md | 8 +- test/e2e-obsidian/scripts/dialog-mounts.ts | 85 +++++++--- .../scripts/hidden-file-snippet-sync.ts | 49 ++++-- test/e2e-obsidian/scripts/joined-devices.ts | 154 +++++++++++++----- test/e2e-obsidian/scripts/local-suite.ts | 3 + test/e2e-obsidian/scripts/run-focused.ts | 1 + 8 files changed, 276 insertions(+), 81 deletions(-) diff --git a/package.json b/package.json index cc9316c3..7a758866 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", "test:e2e:obsidian:joined-devices": "tsx test/e2e-obsidian/scripts/joined-devices.ts", + "test:e2e:obsidian:joined-devices-object-storage": "tsx test/e2e-obsidian/scripts/joined-devices.ts --object-storage", "test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts", "test:e2e:obsidian:database-diagnostics-couchdb": "tsx test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", diff --git a/src/serviceFeatures/useJoinedDevices.unit.spec.ts b/src/serviceFeatures/useJoinedDevices.unit.spec.ts index 7806508a..7021f87c 100644 --- a/src/serviceFeatures/useJoinedDevices.unit.spec.ts +++ b/src/serviceFeatures/useJoinedDevices.unit.spec.ts @@ -137,6 +137,62 @@ describe("joined devices service feature", () => { data.close(); await pending; }); + it.each(["dialogue cancellation", "remote switch", "unload"] as const)( + "does not send a pending forget operation after %s", + async (invalidation) => { + const data = fixture(); + const resource = { read: data.read, forget: data.forget, dispose: data.dispose }; + let resolve!: (value: typeof resource) => void; + data.createResource.mockReturnValueOnce( + new Promise((done) => { + resolve = done; + }) + ); + const { options, pending } = await open(data); + const forgetting = options.forget("other"); + const rejected = expect(forgetting).rejects.toThrow(); + await vi.waitFor(() => expect(data.createResource).toHaveBeenCalledOnce()); + + if (invalidation === "dialogue cancellation") options.cancel(); + if (invalidation === "remote switch") data.settings.couchDB_DBNAME = "new-vault"; + if (invalidation === "unload") for (const unload of data.unload) await unload(); + resolve(resource); + + await rejected; + expect(data.forget).not.toHaveBeenCalled(); + expect(data.dispose).toHaveBeenCalledOnce(); + data.close(); + await pending; + } + ); + it("discards a completed forget result after the dialogue closes and releases its resource", async () => { + const data = fixture(); + let resolve!: (value: "forgotten") => void; + data.forget.mockReturnValueOnce( + new Promise((done) => { + resolve = done; + }) + ); + const { options, pending } = await open(data); + const forgetting = options.forget("other"); + const rejected = expect(forgetting).rejects.toThrow(); + await vi.waitFor(() => expect(data.forget).toHaveBeenCalledWith("other")); + data.close(); + await pending; + resolve("forgotten"); + await rejected; + expect(data.dispose).toHaveBeenCalledOnce(); + }); + it("propagates a failed forget operation after releasing its resource", async () => { + const data = fixture(); + const failure = new Error("conditional write failed"); + data.forget.mockRejectedValueOnce(failure); + const { options, pending } = await open(data); + await expect(options.forget("other")).rejects.toBe(failure); + expect(data.dispose).toHaveBeenCalledOnce(); + data.close(); + await pending; + }); it("deduplicates advisory Notices and fences their actions after configuration changes", async () => { const data = fixture(); const { options, pending } = await open(data); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 1438a780..aa52c0c5 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -123,9 +123,9 @@ The underlying `test:e2e:obsidian:` scripts remain available for an im `configuration-encryption` needs no remote service. It applies a custom configuration key through the settings interface, restarts the same isolated Vault, and applies a change to only the custom string. It then copies the encrypted settings into a fresh profile without that key and checks masked input, wrong-input retry, cancellation without a settings write or database initialisation, re-enabling for another attempt, access to the settings tab, and restoration after a further restart. Connection, E2EE, profile, and independent ID values must remain unchanged. These checks establish ordinary Apply and restart behaviour, without simulating process termination between the device-key and settings writes. It also checks changes from custom and ASK modes to Default through the settings interface, and recovery from a wrong saved device key, including cancellation, the validated replacement key, and another restart. -`test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. +`test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The P2P setup check delays the saved device-name read, requires the name field and both continuation controls to remain disabled, then verifies the saved name and selected connection settings reach the setup workflow. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. -`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and forgets one device through the inline confirmation. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. +`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. `test:e2e:obsidian:joined-devices-object-storage` runs the same assertions against a unique Object Storage prefix, including the conditional removal supported by the test S3 service. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and verifies that cancelling the inline confirmation leaves the remote unchanged before forgetting one device. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. `test:e2e:obsidian:settings-ui` starts with a pending compatibility review and verifies the dedicated pause summary, its detailed explanation, and the explicit resume action in a temporary real Obsidian session. It captures the desktop summary and the iPhone-sized summary and detail dialogues; the mobile checks cover viewport containment, horizontal overflow, safe-area containment, and the close control's touch target. It confirms that the acknowledged internal version advances only after the review is accepted, and checks that the Change Log contains no acknowledgement control. It then enables Advanced mode and persists one numeric Advanced setting. On Obsidian before 1.13, this covers the imperative settings fallback used by the `SettingSpec` proof. On Obsidian 1.13 or later, it verifies the task-oriented landing order, the separate Synchronisation group containing Remote Configuration and Sync Settings, and the General Settings group containing Appearance, Logging, and Extra menus. It reopens and cancels onboarding from Quick Setup, opens all 14 nested settings pages, searches globally for the Advanced control, captures the landing page, search result, Advanced page, and mobile landing page, and verifies that the value remains after the settings dialogue is closed and reopened. The mobile landing check confirms that Remote Configuration is reachable without initial scrolling. Finally, it selects the Synchronisation Settings page and verifies that the deletion panel still exposes the effective 'Keep empty folder' setting without presenting the legacy `trashInsteadDelete` control, whose value no longer changes Obsidian deletion behaviour. @@ -248,9 +248,9 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:document-history-restore` creates a normal note, records a logical deletion while retaining readable chunks, and restores the deleted content through the visible Document History dialogue. It requires the action itself to create and reflect a new non-deleted successor revision, reopens the history at that successor, and captures the file picker, readable deleted revision, restored Vault file, and new successor revision. This scenario owns the ordinary-history restoration boundary; conflict resolution remains with **Inspect conflicts and file/database differences**. -`test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. +`test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. That target-pattern case keeps a normal-file allow-list for Markdown and PDF files enabled on both devices, then requires exact hidden JSON content after the receiving device includes it. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. -`test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path. +`test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database, and also runs in the complete local suite. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path. `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. diff --git a/test/e2e-obsidian/scripts/dialog-mounts.ts b/test/e2e-obsidian/scripts/dialog-mounts.ts index 2456d9cd..60d7cf1e 100644 --- a/test/e2e-obsidian/scripts/dialog-mounts.ts +++ b/test/e2e-obsidian/scripts/dialog-mounts.ts @@ -15,6 +15,7 @@ import { captureObsidianPage, obsidianRemoteDebuggingPort, openLiveSyncSettings, + waitForVisibleObsidianDialogue, withObsidianPage as withObsidianSessionPage, } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; @@ -43,6 +44,8 @@ type DialogueMode = "desktop" | "mobile"; type DialogueRunState = { done: boolean; deviceNameRead?: boolean; + deviceNameReadPending?: boolean; + releaseDeviceNameRead?: () => void; error?: string; expected?: unknown; kind: string; @@ -140,11 +143,20 @@ async function openP2PRemoteSelectionDialogueForInspection(): Promise { kind: "p2p-compatibility-settings", done: false, deviceNameRead: false, + deviceNameReadPending: false, }; (globalThis as unknown as Record)[stateKey] = state; + const deviceNameReadGate = new Promise((resolve) => { + state.releaseDeviceNameRead = resolve; + }); config.getValue = async (key, defaultValue) => { const value = await originalGetValue.call(config, key, defaultValue); - if (key === deviceNameKey && value === deviceName) state.deviceNameRead = true; + if (key === deviceNameKey) { + state.deviceNameReadPending = true; + await deviceNameReadGate; + state.deviceNameReadPending = false; + if (value === deviceName) state.deviceNameRead = true; + } return value; }; manager.onConfirmApplySettingsFromWizard = async (settings: unknown) => { @@ -156,16 +168,17 @@ async function openP2PRemoteSelectionDialogueForInspection(): Promise { .then( (result) => { state.result = result; - state.done = true; }, (error: unknown) => { state.error = error instanceof Error ? error.message : String(error); - state.done = true; } ) .finally(() => { + state.releaseDeviceNameRead?.(); + state.releaseDeviceNameRead = undefined; manager.onConfirmApplySettingsFromWizard = originalConfirm; config.getValue = originalGetValue; + state.done = true; }); }, { @@ -177,6 +190,15 @@ async function openP2PRemoteSelectionDialogueForInspection(): Promise { }); } +async function releasePendingP2PDeviceNameRead(): Promise { + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + await page.evaluate((stateKey) => { + const state = (globalThis as unknown as Record)[stateKey]; + state?.releaseDeviceNameRead?.(); + }, dialogRunStateKey); + }); +} + async function openSetupUriDialogue(): Promise { const opened = await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { return await page.evaluate( @@ -523,9 +545,11 @@ async function verifyCouchDBSettingsDialogue(mode: DialogueMode): Promise { await openP2PRemoteSelectionDialogueForInspection(); await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - const remoteSelection = page.locator(".modal-container").filter({ - has: page.locator(".modal-title").filter({ hasText: "Choose a synchronisation remote" }), - }); + const remoteSelection = await waitForVisibleObsidianDialogue( + page, + "Choose a synchronisation remote", + uiTimeoutMs + ); await remoteSelection .locator("label") .filter({ hasText: "Peer-to-Peer (P2P)" }) @@ -541,13 +565,36 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { obsidianRemoteDebuggingPort(), "setup-p2p-compatibility-dialogue.png", async (page) => { - const modal = page.locator(".modal-container").filter({ - has: page.locator(".modal-title").filter({ hasText: "P2P Configuration" }), - }); + const modal = await waitForVisibleObsidianDialogue(page, "P2P Configuration", uiTimeoutMs); + const modalPage = modal.page(); await modal.waitFor({ state: "visible", timeout: uiTimeoutMs }); - const peerName = modal.locator('input[name="p2p-device-peer-id"]'); + const peerName = modal.getByLabel("Device Peer ID", { exact: true }); await peerName.waitFor({ state: "visible", timeout: uiTimeoutMs }); - await page.waitForFunction( + await withObsidianPage(obsidianRemoteDebuggingPort(), async (statePage) => { + await statePage.waitForFunction( + (stateKey) => + (globalThis as unknown as Record)[stateKey] + ?.deviceNameReadPending === true, + dialogRunStateKey, + { timeout: uiTimeoutMs } + ); + }); + const testAndContinue = modal.getByRole("button", { name: "Test Settings and Continue", exact: true }); + const continueAnyway = modal.getByRole("button", { name: "Continue anyway", exact: true }); + await testAndContinue.waitFor({ state: "visible", timeout: uiTimeoutMs }); + await continueAnyway.waitFor({ state: "visible", timeout: uiTimeoutMs }); + if ( + !(await peerName.isDisabled()) || + !(await testAndContinue.isDisabled()) || + !(await continueAnyway.isDisabled()) + ) { + throw new Error( + "The P2P device name and setup actions must remain disabled while the saved device name is loading." + ); + } + + await releasePendingP2PDeviceNameRead(); + await modalPage.waitForFunction( (expectedName) => { const input = document.querySelector('input[name="p2p-device-peer-id"]'); return input instanceof HTMLInputElement && !input.disabled && input.value === expectedName; @@ -562,7 +609,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { await modal.locator('input[name="p2p-password"]').fill("e2e-passphrase"); const compatibility = modal.locator("details").filter({ - has: page.locator("summary").filter({ hasText: "Connection compatibility" }), + has: modalPage.locator("summary").filter({ hasText: "Connection compatibility" }), }); await compatibility.waitFor({ state: "visible", timeout: uiTimeoutMs }); if (!(await compatibility.evaluate((element) => (element as HTMLDetailsElement).open))) { @@ -603,7 +650,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { } const advanced = modal.locator("details").filter({ - has: page.locator("summary").filter({ hasText: "Advanced Settings" }), + has: modalPage.locator("summary").filter({ hasText: "Advanced Settings" }), }); if (!(await advanced.evaluate((element) => (element as HTMLDetailsElement).open))) { await advanced.locator("summary").click({ timeout: uiTimeoutMs }); @@ -615,13 +662,13 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { if (relayOnlyElement === null || connectionPathElement === null) { throw new Error("P2P connection-path controls were removed whilst validating TURN settings."); } - await page.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { + await modalPage.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { timeout: uiTimeoutMs, }); await connectionPath.selectOption("relay"); await turnServers.fill(""); - await page.waitForFunction( + await modalPage.waitForFunction( (element) => (element as HTMLSelectElement).value === "automatic", connectionPathElement, { timeout: uiTimeoutMs } @@ -636,7 +683,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { .waitFor({ state: "visible", timeout: uiTimeoutMs }); await turnServers.fill("turn:turn.example.com:3478"); - await page.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { + await modalPage.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { timeout: uiTimeoutMs, }); await connectionPath.selectOption("relay"); @@ -644,12 +691,10 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { await advanced.locator("summary").click({ timeout: uiTimeoutMs }); await compatibility.scrollIntoViewIfNeeded(); } - ); + ).finally(releasePendingP2PDeviceNameRead); await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - const modal = page.locator(".modal-container").filter({ - has: page.locator(".modal-title").filter({ hasText: "P2P Configuration" }), - }); + const modal = await waitForVisibleObsidianDialogue(page, "P2P Configuration", uiTimeoutMs); await modal.getByRole("button", { name: "Continue anyway", exact: true }).click({ timeout: uiTimeoutMs }); await modal.waitFor({ state: "hidden", timeout: uiTimeoutMs }); }); diff --git a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts index 5079c021..19a92322 100644 --- a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts +++ b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts @@ -69,6 +69,7 @@ type RunnerContext = { cliBinary: string; couchDb: CouchDbConfig; dbName: string; + activeSessions: Set; }; async function writeVaultFile(vaultPath: string, path: string, content: string): Promise { @@ -348,6 +349,7 @@ async function startConfiguredSession( pluginData: createE2eCouchDbPluginData(couchDbSettings, hiddenFileSettings), localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), }); + context.activeSessions.add(session); await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); await assertE2eCompatibilityMarker(context.cliBinary, session.cliEnv); await configureCouchDb(context.cliBinary, session.cliEnv, couchDbSettings, hiddenFileSettings); @@ -355,6 +357,11 @@ async function startConfiguredSession( return session; } +async function stopSession(context: RunnerContext, session: ObsidianLiveSyncSession): Promise { + await session.app.stop(); + context.activeSessions.delete(session); +} + async function uploadHiddenFile( context: RunnerContext, session: ObsidianLiveSyncSession, @@ -430,12 +437,12 @@ async function runCreateRoundTrip( throw new Error("Remote Rebuild did not declare and encrypt internal Metadata for its accepted writer."); } console.log("Remote Rebuild declared the feature and preserved encrypted Hidden File Sync Metadata."); - await session.app.stop(); + await stopSession(context, session); session = await startConfiguredSession(context, vaultB); await pullAndApplyHiddenFiles(context, session); const received = await waitForPathContent(vaultB.path, snippetPath, (content) => content === snippetContent); - await session.app.stop(); + await stopSession(context, session); assertEqual(received, snippetContent, "Hidden snippet content did not round-trip to the second vault."); console.log(`Hidden create round-trip copied ${entry.id} to the second vault.`); @@ -450,12 +457,12 @@ async function runDeleteRoundTrip( let session = await startConfiguredSession(context, vaultA); await scanHiddenStorage(context.cliBinary, session.cliEnv); await pushLocalChanges(context.cliBinary, session.cliEnv); - await session.app.stop(); + await stopSession(context, session); session = await startConfiguredSession(context, vaultB); await pullAndApplyHiddenFiles(context, session); await waitForPathDeleted(vaultB.path, snippetPath); - await session.app.stop(); + await stopSession(context, session); console.log("Hidden delete round-trip removed the snippet from the second vault."); } @@ -476,14 +483,14 @@ async function runJsonConflictRoundTrip( const mergedOnB = await waitForPathContent(vaultB.path, mergeJsonPath, (content) => hasJsonValues(content, { fromA: true, fromB: true }) ); - await session.app.stop(); + await stopSession(context, session); session = await startConfiguredSession(context, vaultA); await pullAndApplyHiddenFiles(context, session); const mergedOnA = await waitForPathContent(vaultA.path, mergeJsonPath, (content) => hasJsonValues(content, { fromA: true, fromB: true }) ); - await session.app.stop(); + await stopSession(context, session); assertEqual(mergedOnA, mergedOnB, "Merged hidden JSON content was not consistent across both vaults."); console.log("Hidden JSON conflict was automatically merged and round-tripped."); @@ -503,7 +510,7 @@ async function runJsonManualConflictResolution(context: RunnerContext, vault: Te const merged = await waitForPathContent(vault.path, manualMergeJsonPath, (content) => hasJsonValues(content, { shared: "right", fromA: true, fromB: true }) ); - await session.app.stop(); + await stopSession(context, session); const parsed = JSON.parse(merged); assertEqual(parsed.shared, "right", "Manual JSON conflict resolution did not apply the selected merged result."); @@ -517,17 +524,19 @@ async function runTargetMismatch( vaultA: TemporaryVault, vaultB: TemporaryVault ): Promise { + const selectors = { syncOnlyRegEx: String.raw`\.(md|pdf)$` }; const targetContent = JSON.stringify({ onlyA: true, targetMismatch: true }, null, 4) + "\n"; await writeVaultFile(vaultA.path, targetPath, targetContent); - let session = await startConfiguredSession(context, vaultA); + let session = await startConfiguredSession(context, vaultA, selectors); try { await uploadHiddenFile(context, session, targetPath); } finally { - await session.app.stop(); + await stopSession(context, session); } session = await startConfiguredSession(context, vaultB, { + ...selectors, syncInternalFilesTargetPatterns: "snippets", }); try { @@ -538,10 +547,11 @@ async function runTargetMismatch( "Hidden file was applied on a device where it was not a target file." ); } finally { - await session.app.stop(); + await stopSession(context, session); } session = await startConfiguredSession(context, vaultB, { + ...selectors, syncInternalFilesTargetPatterns: "", }); let received = ""; @@ -549,11 +559,17 @@ async function runTargetMismatch( await pullAndApplyHiddenFiles(context, session, { resolveConflicts: false }); received = await waitForPathContent(vaultB.path, targetPath, (content) => content === targetContent); } finally { - await session.app.stop(); + await stopSession(context, session); } - assertEqual(received, targetContent, "Hidden file was not applied after it became a target file."); - console.log("Hidden target mismatch respected per-device target patterns, then applied after enabling the target."); + assertEqual( + received, + targetContent, + "The normal file allow-list blocked a selected hidden file from the other device." + ); + console.log( + "A normal Markdown/PDF allow-list preserved hidden target exclusion and exact remote restoration after enabling the target." + ); } async function setHiddenFileNoticeFixtures(port: number, itemIds: string[], includeRestart: boolean): Promise { @@ -828,7 +844,7 @@ async function runInitialisationNoticeGrouping(context: RunnerContext, vault: Te `Hidden File Sync showed one progress Notice before settings were saved and retained it throughout initialisation. Screenshot: ${screenshotPath}` ); } finally { - await session.app.stop(); + await stopSession(context, session); } } @@ -889,7 +905,7 @@ async function runConfigurationNoticeGrouping(context: RunnerContext, vault: Tem } finally { await clearHiddenFileNoticeFixtures(port).catch(() => undefined); await setObsidianMobileTestMode(port, false, timeoutMs).catch(() => undefined); - await session.app.stop(); + await stopSession(context, session); } } @@ -904,7 +920,7 @@ async function main(): Promise { const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "hidden-roundtrip"); const vaultA = await createTemporaryVault(); const vaultB = await createTemporaryVault(); - const context: RunnerContext = { binary, cliBinary: cli.binary, couchDb, dbName }; + const context: RunnerContext = { binary, cliBinary: cli.binary, couchDb, dbName, activeSessions: new Set() }; try { await assertCouchDbReachable(couchDb); @@ -923,6 +939,7 @@ async function main(): Promise { await runInitialisationNoticeGrouping(context, vaultB); await runConfigurationNoticeGrouping(context, vaultB); } finally { + for (const session of context.activeSessions) await stopSession(context, session); await vaultA.dispose(); await vaultB.dispose(); if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index b53ef035..e3d0ccd9 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -1,3 +1,4 @@ +import { PutObjectCommand } from "@aws-sdk/client-s3"; import { mkdir } from "node:fs/promises"; import { join } from "node:path"; import { DEVICE_ID_PREFERRED, MILESTONE_DOCID, NODEINFO_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; @@ -12,12 +13,22 @@ import { putCouchDbDocument, type CouchDbDocument, } from "../runner/couchdb.ts"; +import { + createObjectStorageClient, + deleteObjectStoragePrefix, + ensureObjectStorageBucket, + loadObjectStorageConfig, + makeUniqueBucketPrefix, + readObjectStorageJson, +} from "../runner/objectStorage.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { assertEqual, assertE2eCompatibilityUnpaused, configureCouchDb, + configureObjectStorage, createE2eCouchDbPluginData, + createE2eObjectStoragePluginData, prepareRemote, waitForLiveSyncCoreReady, } from "../runner/liveSyncWorkflow.ts"; @@ -72,6 +83,73 @@ type DeviceFixture = { preferredTweaks: Record; }; +type JoinedDevicesRemote = { + label: string; + pluginData: Record; + prepare(): Promise; + configure(cliBinary: string, env: NodeJS.ProcessEnv): Promise; + read(): Promise; + write(milestone: JoinedDevicesMilestone): Promise; + cleanup(): Promise; +}; + +async function createRemoteFixture(): Promise { + const args = process.argv.slice(2); + if (args.some((argument) => argument !== "--object-storage")) { + throw new Error("Usage: joined-devices.ts [--object-storage]"); + } + if (args.includes("--object-storage")) { + const config = await loadObjectStorageConfig(); + const bucketPrefix = makeUniqueBucketPrefix("joined-devices"); + const settings = { ...config, bucketPrefix }; + const key = `${bucketPrefix}_00000000-milestone.json`; + return { + label: "Object Storage", + pluginData: createE2eObjectStoragePluginData(settings), + prepare: () => ensureObjectStorageBucket(config), + configure: (cliBinary, env) => configureObjectStorage(cliBinary, env, settings), + read: () => readObjectStorageJson(config, key), + write: async (milestone) => { + const client = createObjectStorageClient(config); + try { + await client.send( + new PutObjectCommand({ + Bucket: config.bucket, + Key: key, + Body: JSON.stringify(milestone), + ContentType: "application/json", + }) + ); + } finally { + client.destroy(); + } + }, + cleanup: () => deleteObjectStoragePrefix(config, bucketPrefix), + }; + } + const config = await loadCouchDbConfig(); + const dbName = makeUniqueDatabaseName(config.dbPrefix, "joined-devices"); + const settings = { uri: config.uri, username: config.username, password: config.password, dbName }; + return { + label: "CouchDB", + pluginData: createE2eCouchDbPluginData(settings), + prepare: async () => { + await assertCouchDbReachable(config); + await createCouchDbDatabase(config, dbName); + }, + configure: (cliBinary, env) => configureCouchDb(cliBinary, env, settings), + read: async () => { + const milestone = await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID); + assert(Boolean(milestone._rev), "The remote milestone did not have a revision."); + return milestone as JoinedDevicesMilestone; + }, + write: async (milestone) => { + await putCouchDbDocument(config, dbName, milestone); + }, + cleanup: () => deleteCouchDbDatabase(config, dbName), + }; +} + function assert(condition: boolean, message: string): void { if (!condition) throw new Error(message); } @@ -228,13 +306,9 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa .waitFor({ state: "visible", timeout: uiTimeoutMs }); } -async function refreshRemotePlatform( - page: Page, - config: Awaited>, - dbName: string -): Promise { - const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; - await putCouchDbDocument(config, dbName, { +async function refreshRemotePlatform(page: Page, remote: JoinedDevicesRemote): Promise { + const milestone = await remote.read(); + await remote.write({ ...milestone, node_info: { ...milestone.node_info, @@ -258,6 +332,15 @@ async function showForgetConfirmation(dialogue: Locator): Promise { }); } +async function assertCancelledForget(dialogue: Locator, remote: JoinedDevicesRemote): Promise { + const before = await remote.read(); + await showForgetConfirmation(dialogue); + const confirmation = deviceCard(dialogue, targetId).locator(".forget-confirmation"); + await confirmation.getByRole("button", { name: "Cancel", exact: true }).click({ timeout: uiTimeoutMs }); + await confirmation.waitFor({ state: "detached", timeout: uiTimeoutMs }); + assertEqual(JSON.stringify(await remote.read()), JSON.stringify(before), "Cancelling Forget changed the remote."); +} + async function confirmForgetTarget(dialogue: Locator): Promise { const targetCard = deviceCard(dialogue, targetId); await targetCard.getByRole("button", { name: "Forget device", exact: true }).click({ timeout: uiTimeoutMs }); @@ -268,12 +351,8 @@ async function confirmForgetTarget(dialogue: Locator): Promise { await targetCard.waitFor({ state: "detached", timeout: uiTimeoutMs }); } -async function assertTargetForgotten( - config: Awaited>, - dbName: string, - fixture: DeviceFixture -): Promise { - const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; +async function assertTargetForgotten(remote: JoinedDevicesRemote, fixture: DeviceFixture): Promise { + const milestone = await remote.read(); assertEqual( JSON.stringify(milestone.accepted_nodes), JSON.stringify([retainedId]), @@ -371,42 +450,32 @@ async function main(): Promise { const cli = discoverObsidianCli(); if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); - const couchDb = await loadCouchDbConfig(); - const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "joined-devices"); + const remote = await createRemoteFixture(); const vault = await createTemporaryVault(); let session: ObsidianLiveSyncSession | undefined; try { - await assertCouchDbReachable(couchDb); - await createCouchDbDatabase(couchDb, dbName); + await remote.prepare(); session = await startObsidianLiveSyncSession({ binary, cliBinary: cli.binary, vault, startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), - pluginData: createE2eCouchDbPluginData({ - uri: couchDb.uri, - username: couchDb.username, - password: couchDb.password, - dbName, - }), + pluginData: remote.pluginData, }); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); await assertE2eCompatibilityUnpaused(cli.binary, session.cliEnv, session.remoteDebuggingPort); - await configureCouchDb(cli.binary, session.cliEnv, { - uri: couchDb.uri, - username: couchDb.username, - password: couchDb.password, - dbName, - }); + await remote.configure(cli.binary, session.cliEnv); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); await prepareRemote(cli.binary, session.cliEnv); const self = await readSelfDeviceInfo(cli.binary, session.cliEnv); const fixture = createFixture(self); - const preparedMilestone = await fetchCouchDbDocument(couchDb, dbName, MILESTONE_DOCID); - assert(Boolean(preparedMilestone._rev), "Remote preparation did not create a revisioned milestone document."); - await putCouchDbDocument(couchDb, dbName, { ...fixture.milestone, _rev: preparedMilestone._rev }); + const preparedMilestone = await remote.read(); + await remote.write({ + ...fixture.milestone, + ...(preparedMilestone._rev ? { _rev: preparedMilestone._rev } : {}), + }); await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { const dialogue = await openFromRemoteSettings(page); @@ -440,8 +509,9 @@ async function main(): Promise { session.remoteDebuggingPort, "joined-devices-forget-confirmation.png", async (page) => { - await refreshRemotePlatform(page, couchDb, dbName); + await refreshRemotePlatform(page, remote); const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await assertCancelledForget(dialogue, remote); await showForgetConfirmation(dialogue); await deviceCard(dialogue, targetId).locator(".forget-confirmation").scrollIntoViewIfNeeded(); return dialogue; @@ -459,7 +529,7 @@ async function main(): Promise { const settings = await openLiveSyncSettings(page, uiTimeoutMs); await settings.close(); }); - await assertTargetForgotten(couchDb, dbName, fixture); + await assertTargetForgotten(remote, fixture); await setObsidianMobileTestMode(session.remoteDebuggingPort, true, uiTimeoutMs); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); @@ -485,14 +555,16 @@ async function main(): Promise { return dialogue; }); - console.log(`Joined devices scenario passed against CouchDB database ${dbName}.`); + console.log(`Joined devices scenario passed against ${remote.label}.`); } finally { - if (session) await session.app.stop(); - await vault.dispose(); - if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { - await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => { - console.warn(error instanceof Error ? error.message : error); - }); + try { + if (session) await session.app.stop(); + } finally { + try { + await vault.dispose(); + } finally { + await remote.cleanup(); + } } } } diff --git a/test/e2e-obsidian/scripts/local-suite.ts b/test/e2e-obsidian/scripts/local-suite.ts index 8453c584..ee953160 100644 --- a/test/e2e-obsidian/scripts/local-suite.ts +++ b/test/e2e-obsidian/scripts/local-suite.ts @@ -26,6 +26,7 @@ const testSteps: Step[] = [ { name: "Review Harness", args: ["run", "test:e2e:obsidian:review-harness"] }, { name: "P2P status pane", args: ["run", "test:e2e:obsidian:p2p-pane"] }, { name: "vault reflection", args: ["run", "test:e2e:obsidian:vault-reflection"] }, + { name: "joined devices with CouchDB", args: ["run", "test:e2e:obsidian:joined-devices"] }, { name: "CouchDB upload", args: ["run", "test:e2e:obsidian:couchdb-upload"] }, { name: "manual CouchDB setup workflow", @@ -35,6 +36,7 @@ const testSteps: Step[] = [ name: "CLI to real Obsidian synchronisation", args: ["run", "test:e2e:obsidian:cli-to-obsidian-sync"], }, + { name: "joined devices with Object Storage", args: ["run", "test:e2e:obsidian:joined-devices-object-storage"] }, { name: "Object Storage upload", args: ["run", "test:e2e:obsidian:minio-upload"] }, { name: "Object Storage Setup URI workflow", @@ -56,6 +58,7 @@ const testSteps: Step[] = [ { name: "startup scan", args: ["run", "test:e2e:obsidian:startup-scan"] }, { name: "provisioned Setup URI workflow", args: ["run", "test:e2e:obsidian:setup-uri-workflow"] }, { name: "two-vault synchronisation", args: ["run", "test:e2e:obsidian:two-vault-sync"] }, + { name: "hidden file selectors", args: ["run", "test:e2e:obsidian:hidden-file-selectors"] }, { name: "hidden file snippet synchronisation", args: ["run", "test:e2e:obsidian:hidden-file-snippet-sync"] }, { name: "Customisation Sync", args: ["run", "test:e2e:obsidian:customisation-sync"] }, { name: "internal Metadata Doctor", args: ["run", "test:e2e:obsidian:internal-metadata-doctor"] }, diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 78d39fb8..31a8f339 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -9,6 +9,7 @@ const focusedScenarios = new Set([ "onboarding-invitation", "dialog-mounts", "joined-devices", + "joined-devices-object-storage", "database-diagnostics", "database-diagnostics-couchdb", "conflict-dialog-policy", From 916784c5c709c08cbf58e33668894eea441de329 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 20:18:29 +0000 Subject: [PATCH 17/28] fix: clarify device version notices --- .../messages/LiveSyncProvisionalMessages.ts | 10 +- src/serviceFeatures/useJoinedDevices.ts | 43 ++++---- .../useJoinedDevices.unit.spec.ts | 98 ++++++++++++++++++- test/e2e-obsidian/README.md | 2 +- test/e2e-obsidian/scripts/joined-devices.ts | 27 +++-- updates.md | 2 +- 6 files changed, 144 insertions(+), 38 deletions(-) diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index c35a9c8d..0747ac4f 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -42,10 +42,12 @@ export const liveSyncProvisionalEnglishMessages = { Refresh: "Refresh", "Plug-in version": "Plug-in version", Platform: "Platform", - "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": - "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", - "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": - "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", + "Older Self-hosted LiveSync versions detected: ${versions}.": + "Older Self-hosted LiveSync versions detected: ${versions}.", + "Newer Self-hosted LiveSync versions detected: ${versions}.": + "Newer Self-hosted LiveSync versions detected: ${versions}.", + "Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.": + "Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.", Open: "Open", "Not recorded": "Not recorded", "Not accepted": "Not accepted", diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts index 424f5ce9..53b7dbc9 100644 --- a/src/serviceFeatures/useJoinedDevices.ts +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -53,6 +53,7 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi const differences = differentDeviceVersions(snapshot, ownId, version); const seen = notified.get(identity) ?? new Set(); notified.set(identity, seen); + let updated = false; for (const direction of ["newer", "older"] as const) { const versions = differences[direction]; if (versions.length === 0) continue; @@ -62,29 +63,29 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi notices.setItem(noticeGroup, direction, { message: direction === "newer" - ? $msg( - "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", - { - versions: versions.join(", "), - } - ) - : $msg( - "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", - { - versions: versions.join(", "), - } - ), - action: { - label: $msg("Review devices"), - onSelect: () => { - if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { - void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); - } - }, - }, + ? $msg("Newer Self-hosted LiveSync versions detected: ${versions}.", { + versions: versions.join(", "), + }) + : $msg("Older Self-hosted LiveSync versions detected: ${versions}.", { + versions: versions.join(", "), + }), }); - notices.finish(noticeGroup, { durationMs: 15000 }); + updated = true; } + if (!updated) return; + notices.removeItem(noticeGroup, "review"); + notices.setItem(noticeGroup, "review", { + message: $msg("Reviewing and aligning Self-hosted LiveSync versions across devices is recommended."), + action: { + label: $msg("Review devices"), + onSelect: () => { + if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { + void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + } + }, + }, + }); + notices.finish(noticeGroup, { durationMs: 15000 }); }; const createOperations = (controller: AbortController): JoinedDevicesDialogOptions => { diff --git a/src/serviceFeatures/useJoinedDevices.unit.spec.ts b/src/serviceFeatures/useJoinedDevices.unit.spec.ts index 7021f87c..188afe5a 100644 --- a/src/serviceFeatures/useJoinedDevices.unit.spec.ts +++ b/src/serviceFeatures/useJoinedDevices.unit.spec.ts @@ -5,7 +5,10 @@ import { REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; import type { JoinedDevicesDialogOptions } from "./joinedDevicesDialogTypes"; vi.mock("@/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte", () => ({ default: {} })); -vi.mock("@/common/translation", () => ({ $msg: (message: string) => message })); +vi.mock("@/common/translation", () => ({ + $msg: (message: string, values: Record = {}) => + message.replace(/\$\{([^}]+)\}/g, (placeholder, key: string) => values[key] ?? placeholder), +})); import { useJoinedDevicesFeature } from "./useJoinedDevices"; import { OPEN_JOINED_DEVICES } from "./joinedDevicesCommand"; @@ -37,7 +40,7 @@ function fixture() { close = resolve; }) ); - const notices = { setItem: vi.fn(), finish: vi.fn(), hide: vi.fn() }; + const notices = { setItem: vi.fn(), removeItem: vi.fn(), finish: vi.fn(), hide: vi.fn() }; const palette: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = []; const database = { localDatabase: { get: vi.fn(async () => ({ nodeid: "self" })) } }; const host = { @@ -193,13 +196,100 @@ describe("joined devices service feature", () => { data.close(); await pending; }); + it("shares one review action across newer and older device versions", async () => { + const data = fixture(); + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [ + { id: "newer-a", accepted: true, pluginVersion: "2.0.0" }, + { id: "newer-b", accepted: true, pluginVersion: "3.0.0" }, + { id: "older-a", accepted: true, pluginVersion: "0.9.0" }, + { id: "older-b", accepted: true, pluginVersion: "0.8.0" }, + { id: "older-duplicate", accepted: true, pluginVersion: "0.8.0" }, + ], + }); + const { options, pending } = await open(data); + await options.refresh(); + const items = data.notices.setItem.mock.calls.map(([, , item]) => item); + expect(items.filter((item) => item.action)).toHaveLength(1); + const messages = items.map((item) => item.message).join("\n"); + expect(messages).toContain("Self-hosted LiveSync"); + expect(messages).toMatch(/Newer.*2\.0\.0, 3\.0\.0/); + expect(messages).toMatch(/Older.*0\.8\.0, 0\.9\.0/); + expect(messages.match(/0\.8\.0/g)).toHaveLength(1); + expect(messages.match(/Reviewing and aligning/g)).toHaveLength(1); + expect(items[items.length - 1]?.action?.label).toBe("Review devices"); + expect(data.notices.finish).toHaveBeenCalledExactlyOnceWith("joined-device-versions", { + durationMs: 15000, + }); + data.close(); + await pending; + }); + it("keeps the shared action last when another version direction arrives", async () => { + const data = fixture(); + const visible = new Map(); + data.notices.setItem.mockImplementation((_group, key, item) => visible.set(key, item)); + data.notices.removeItem.mockImplementation((_group, key) => visible.delete(key)); + const { options, pending } = await open(data); + await options.refresh(); + expect([...visible.keys()]).toEqual(["newer", "review"]); + + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [ + { id: "other", accepted: true, pluginVersion: "2.0.0" }, + { id: "older", accepted: true, pluginVersion: "0.9.0" }, + ], + }); + await options.refresh(); + expect([...visible.keys()]).toEqual(["newer", "older", "review"]); + expect([...visible.values()].filter((item) => item.action)).toHaveLength(1); + + visible.clear(); + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [ + { id: "other", accepted: true, pluginVersion: "2.0.0" }, + { id: "older", accepted: true, pluginVersion: "0.8.0" }, + ], + }); + await options.refresh(); + expect([...visible.keys()]).toEqual(["older", "review"]); + expect([...visible.values()].filter((item) => item.action)).toHaveLength(1); + data.close(); + await pending; + }); + it("does not show a review action when no different version is reported", async () => { + const data = fixture(); + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [{ id: "other", accepted: true, pluginVersion: "1.0.0" }], + }); + const { options, pending } = await open(data); + await options.refresh(); + expect(data.notices.setItem).not.toHaveBeenCalled(); + expect(data.notices.removeItem).not.toHaveBeenCalled(); + expect(data.notices.finish).not.toHaveBeenCalled(); + data.close(); + await pending; + }); it("deduplicates advisory Notices and fences their actions after configuration changes", async () => { const data = fixture(); const { options, pending } = await open(data); await options.refresh(); await options.refresh(); - expect(data.notices.setItem).toHaveBeenCalledTimes(1); - const action = data.notices.setItem.mock.calls[0][2].action; + expect(data.notices.setItem).toHaveBeenCalledTimes(2); + const actions = data.notices.setItem.mock.calls.map(([, , item]) => item.action).filter(Boolean); + expect(actions).toHaveLength(1); + const action = actions[0]; data.close(); await pending; data.settings.couchDB_DBNAME = "new-vault"; diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index aa52c0c5..5f04c8c8 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -125,7 +125,7 @@ The underlying `test:e2e:obsidian:` scripts remain available for an im `test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The P2P setup check delays the saved device-name read, requires the name field and both continuation controls to remain disabled, then verifies the saved name and selected connection settings reach the setup workflow. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. -`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. `test:e2e:obsidian:joined-devices-object-storage` runs the same assertions against a unique Object Storage prefix, including the conditional removal supported by the test S3 service. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and verifies that cancelling the inline confirmation leaves the remote unchanged before forgetting one device. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. +`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. `test:e2e:obsidian:joined-devices-object-storage` runs the same assertions against a unique Object Storage prefix, including the conditional removal supported by the test S3 service. It checks that newer and older Self-hosted LiveSync versions share one Notice with one recommendation and review action, that the current device appears first with current metadata, and that locked-out and other-device records are shown. It confirms that a manual refresh reads an updated remote record and that cancelling the inline confirmation leaves the remote unchanged before forgetting one device. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. `test:e2e:obsidian:settings-ui` starts with a pending compatibility review and verifies the dedicated pause summary, its detailed explanation, and the explicit resume action in a temporary real Obsidian session. It captures the desktop summary and the iPhone-sized summary and detail dialogues; the mobile checks cover viewport containment, horizontal overflow, safe-area containment, and the close control's touch target. It confirms that the acknowledged internal version advances only after the review is accepted, and checks that the Change Log contains no acknowledgement control. It then enables Advanced mode and persists one numeric Advanced setting. On Obsidian before 1.13, this covers the imperative settings fallback used by the `SettingSpec` proof. On Obsidian 1.13 or later, it verifies the task-oriented landing order, the separate Synchronisation group containing Remote Configuration and Sync Settings, and the General Settings group containing Appearance, Logging, and Extra menus. It reopens and cancels onboarding from Quick Setup, opens all 14 nested settings pages, searches globally for the Advanced control, captures the landing page, search result, Advanced page, and mobile landing page, and verifies that the value remains after the settings dialogue is closed and reopened. The mobile landing check confirms that Remote Configuration is reachable without initial scrolling. Finally, it selects the Synchronisation Settings page and verifies that the deletion panel still exposes the effective 'Keep empty folder' setting without presenting the legacy `trashInsteadDelete` control, whose value no longer changes Obsidian deletion behaviour. diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index e3d0ccd9..ee9ca83c 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -482,13 +482,27 @@ async function main(): Promise { const dialoguePage = dialogue.page(); await dialoguePage.setViewportSize({ width: 1280, height: 1100 }); await assertInitialDialogue(dialogue, self, dialoguePage); - await dialoguePage - .locator(".notice") - .getByText( - "A newer plug-in version was detected in other device records: 999.0.0. Reviewing and aligning the plug-in versions across devices is recommended.", - { exact: true } - ) + const notice = dialoguePage.locator(".notice").filter({ + hasText: "Newer Self-hosted LiveSync versions detected: 999.0.0.", + }); + await notice.waitFor({ state: "visible", timeout: uiTimeoutMs }); + await notice + .getByText("Older Self-hosted LiveSync versions detected: 0.1.0.", { exact: true }) .waitFor({ state: "visible", timeout: uiTimeoutMs }); + assertEqual( + await notice + .getByText("Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.", { + exact: true, + }) + .count(), + 1, + "The version Notice did not contain exactly one recommendation." + ); + assertEqual( + await notice.getByRole("button", { name: "Review devices", exact: true }).count(), + 1, + "The version Notice did not contain exactly one shared review action." + ); return dialogue; }); await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { @@ -498,7 +512,6 @@ async function main(): Promise { .page() .locator(".notice") .getByRole("button", { name: "Review devices", exact: true }) - .first() .click(); const reopened = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await reopened.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); diff --git a/updates.md b/updates.md index 161bb640..ce3527fd 100644 --- a/updates.md +++ b/updates.md @@ -29,7 +29,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. - - Advisory Notices highlight newer or older plug-in versions detected in device records and recommend reviewing and aligning our versions. The dialogue explains possible storage costs and P2P compatibility limits. + - An advisory Notice identifies Self-hosted LiveSync and lists newer and older versions separately, with one shared recommendation and **Review devices** action. The dialogue explains possible storage costs and P2P compatibility limits. - We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. From 73a14d4e4b0b414a1acfe170d01500abb3dca2e9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 24 Oct 2026 02:41:12 +0000 Subject: [PATCH 18/28] Releasing 1.0.40 --- docs/releases/1.0.md | 24 +++++++++++++++++ manifest.json | 2 +- package-lock.json | 10 +++---- package.json | 2 +- src/apps/cli/package.json | 2 +- src/apps/webapp/package.json | 2 +- src/apps/webpeer/package.json | 2 +- updates.md | 49 ++++++++++++----------------------- versions.json | 3 ++- 9 files changed, 52 insertions(+), 44 deletions(-) diff --git a/docs/releases/1.0.md b/docs/releases/1.0.md index 96ce8a6b..88bd667f 100644 --- a/docs/releases/1.0.md +++ b/docs/releases/1.0.md @@ -2,6 +2,30 @@ This document contains earlier published releases from the 1.0 line of the [current Self-hosted LiveSync release history](../../updates.md). Beta and release-candidate builds published before 1.0.0 are recorded in the [1.0 preview history](1.0-previews.md). Earlier release lines continue in the [0.25 history](0.25.md) and the [legacy history](legacy.md). +## 1.0.35 + +5th October, 2026 + +### Synchronisation + +#### Fixed + +- We can continue synchronising after rebuilding an encrypted CouchDB remote, including when Hidden File Sync and Customisation Sync are enabled. (#1234) + - This plug-in reads the rebuilt data without reusing HTTP responses encrypted with the previous Security Seed. + +### Customisation Sync + +#### Fixed + +- Customisation Sync now waits until this plug-in is ready before handling configuration file events, and applying settings while synchronisation is suspended no longer scans configuration files. (#1220) + - Resuming synchronisation scans once and restarts the periodic scan when enabled. + +### Acknowledgements + +Thank you for your contributions! + +- @kimjansheden (#1220) + ## 1.0.34 2nd October, 2026 diff --git a/manifest.json b/manifest.json index f12a57e2..267d35b5 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.39", + "version": "1.0.40", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index 4cf6a3e2..211e8c3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.39", + "version": "1.0.40", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.39", + "version": "1.0.40", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -11408,7 +11408,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.39-cli", + "version": "1.0.40-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11433,7 +11433,7 @@ }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.39-webapp", + "version": "1.0.40-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11445,7 +11445,7 @@ } }, "src/apps/webpeer": { - "version": "1.0.39-webpeer", + "version": "1.0.40-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, diff --git a/package.json b/package.json index 7a758866..cd383dde 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.39", + "version": "1.0.40", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 367e9fce..e8d7ca9c 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.39-cli", + "version": "1.0.40-cli", "main": "dist/index.cjs", "type": "module", "scripts": { diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index 43800d07..487af8bf 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.39-webapp", + "version": "1.0.40-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index dbe5c64e..22594247 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.39-webpeer", + "version": "1.0.40-webpeer", "type": "module", "scripts": { "dev": "vite", diff --git a/updates.md b/updates.md index ce3527fd..cefb2bce 100644 --- a/updates.md +++ b/updates.md @@ -12,20 +12,15 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased -### CLI +## 1.0.40 -- Unreadable or malformed device-local state now stops initialisation instead of being treated as empty. Existing valid state files keep their format. +24th October, 2026 -### Hidden File Sync - -#### Fixed - -- We can synchronise selected hidden files while the normal-file allow-list is set. (#1272) - - Normal-file exclusion patterns still apply to hidden files, and the Selector descriptions now explain their scope. +This has turned into a slightly larger set of changes overall, but I have finally been able to implement the features I felt we would need. ### Synchronisation -#### Added +#### New Feature - We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. @@ -45,8 +40,20 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi #### Changed - New setup uses the same Chunk-size recommendation of 0 across remote types, and Setting Doctor no longer recommends increasing every self-hosted V3 configuration to 60. + - Setting Doctor no longer flags the saved **Enhance chunk size** value. When rebuilding with V3 Rabin–Karp, we recommend setting it to 0. This is now an efficient choice across all supported remote types. - Existing explicit and agreed values are retained. Smaller Chunks favour reuse during editing, but increase document counts and can make first uploads and large rebuilds slower. Mostly immutable binary Vaults can still benefit from a larger explicit value. +### Hidden File Sync + +#### Fixed + +- We can synchronise selected hidden files while the normal-file allow-list is set. (#1272) + - Normal-file exclusion patterns still apply to hidden files, and the Selector descriptions now explain their scope. + +### CLI + +- Unreadable or malformed device-local state now stops initialisation instead of being treated as empty. Existing valid state files keep their format. + ## 1.0.39 20th October, 2026 @@ -161,27 +168,3 @@ Thank you for your contributions! - @zeedif (#1213, #1214) - @lingting0 (#1242) - -## 1.0.35 - -5th October, 2026 - -### Synchronisation - -#### Fixed - -- We can continue synchronising after rebuilding an encrypted CouchDB remote, including when Hidden File Sync and Customisation Sync are enabled. (#1234) - - This plug-in reads the rebuilt data without reusing HTTP responses encrypted with the previous Security Seed. - -### Customisation Sync - -#### Fixed - -- Customisation Sync now waits until this plug-in is ready before handling configuration file events, and applying settings while synchronisation is suspended no longer scans configuration files. (#1220) - - Resuming synchronisation scans once and restarts the periodic scan when enabled. - -### Acknowledgements - -Thank you for your contributions! - -- @kimjansheden (#1220) diff --git a/versions.json b/versions.json index 1ae45d74..19b183c0 100644 --- a/versions.json +++ b/versions.json @@ -50,5 +50,6 @@ "1.0.36": "1.7.2", "1.0.37": "1.7.2", "1.0.38": "1.7.2", - "1.0.39": "1.7.2" + "1.0.39": "1.7.2", + "1.0.40": "1.7.2" } From fd2eebdf38b0eb24196f07c2237dd74ff9ece494 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 24 Oct 2026 11:41:56 +0000 Subject: [PATCH 19/28] fix(startup): recheck sizes after reading storage --- .../startupLifecycle/incompleteDocuments.ts | 4 +++ .../incompleteDocuments.unit.spec.ts | 33 +++++++++++++++++++ updates.md | 4 +++ 3 files changed, 41 insertions(+) diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts index 0b4ee3de..42d4ee47 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts @@ -117,6 +117,10 @@ export async function checkIncompleteDocuments( continue; } sizeOnStorage = storageFileContent.byteLength; + // The file may have changed since stat, so recheck using the bytes read. + if (recordedSize === actualSize && sizeOnStorage === actualSize && !isConflicted) { + continue; + } const contentMatched = await isDocContentSame(doc.data, storageFileContent); errorFiles.push({ path, diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts index 2f30f12b..c67eddbc 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts @@ -182,6 +182,39 @@ describe("checkIncompleteDocuments", () => { expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); }); + it("does not offer repair when all sizes agree after reading storage", async () => { + const healthy = documentFixture("settled.md", { storageContent: "abc" }); + const fixture = createDependencies(documentsFrom([healthy]), [healthy]); + vi.mocked(fixture.dependencies.storageAccess.statHidden).mockResolvedValueOnce({ ...healthy.stub.stat, size: 4 }); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("settled.md"); + expect(fixture.askSelectStringDialogue).not.toHaveBeenCalled(); + expect(fixture.storeFileToDB).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).toHaveBeenCalledWith("checkIncompleteDocs", true); + }); + + it("uses the bytes read from storage when a size mismatch remains", async () => { + const mismatched = documentFixture("changing.md", { recordedSize: 3, storageContent: "abcde" }); + const fixture = createDependencies(documentsFrom([mismatched]), [mismatched]); + vi.mocked(fixture.dependencies.storageAccess.statHidden).mockResolvedValueOnce({ + ...mismatched.stub.stat, + size: 4, + }); + fixture.askSelectStringDialogue.mockImplementation(selectButton(0)); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("changing.md"); + expect(fixture.askSelectStringDialogue).toHaveBeenCalledOnce(); + expect(fixture.askSelectStringDialogue.mock.calls[0]?.[0]).toEqual( + expect.stringContaining("changing.md (M: 3, A: 3, S: 5)") + ); + expect(fixture.storeFileToDB).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); + }); + it("still reads conflicted storage for the existing repair dialogue", async () => { const conflicted = documentFixture("conflicted.md", { storageContent: "abc", conflicts: ["2-other"] }); const fixture = createDependencies(documentsFrom([conflicted]), [conflicted]); diff --git a/updates.md b/updates.md index ca2e3497..7e31a95e 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,10 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +- We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) + ## 1.0.39 20th October, 2026 From 68fa04af3c9aa8f6d971280ceaab138e917ffbd2 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 02:47:55 +0000 Subject: [PATCH 20/28] Expose explicit LiveSync batching and honour save failures Show the batching choice for the current synchronisation mode while preserving both mode preferences. Observe local save failures before file-open work and report failed asynchronous unload preparation. Record the accepted recovery and batching contract, update the primary settings documentation, and add two-device runtime coverage for batching, deletions, saved settings, and interrupted local processing. Requires Commonlib commit 72bdc66af5d7338d8f2f69ebe83c3866def93360. Update the exact registry dependency when its package version is published. Validation: full repository check; latest packed dependency type checks, production build, 1,373 unit tests (one skipped), and six focused runtime recovery cases. The earlier implementation passed the full 23-case runtime comparison and five additional interruption conditions. --- ...local_file_save_completion_and_batching.md | 356 +++++++ docs/settings.md | 17 +- src/main.ts | 9 +- .../essentialObsidian/ModuleObsidianEvents.ts | 13 +- .../ModuleObsidianEvents.unit.spec.ts | 42 + .../SettingDialogue/PaneSyncSettings.ts | 16 +- .../PaneSyncSettings.unit.spec.ts | 98 ++ .../scripts/issue1207-livesync-batch.ts | 961 ++++++++++++++++++ updates.md | 15 + 9 files changed, 1512 insertions(+), 15 deletions(-) create mode 100644 docs/adr/2026_10_local_file_save_completion_and_batching.md create mode 100644 src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/issue1207-livesync-batch.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md new file mode 100644 index 00000000..8abef973 --- /dev/null +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -0,0 +1,356 @@ +--- +date: 2026-10-24 +commonlib-version: "0.2.0" +self-hosted-livesync-version: "1.0.40" +status: accepted-unreleased +--- + +# Architectural Decision Record: Local File Save Completion and Batching + +## Status + +Accepted on 2026-10-25. The package versions above identify the investigated +baseline, rather than a released implementation of this decision. + +## Context + +[Issue #1207](https://github.com/vrtmrz/obsidian-livesync/issues/1207) includes +conflicts around automated note creation, editing, and renaming. Batching local +file updates can reduce the intermediate revisions which another device receives. +It also extends the period during which the Vault contains an edit which the +local database has not saved. Recovery and incoming-file protection must remain +correct throughout that period. + +Two-device Obsidian E2E tests using Self-hosted LiveSync 1.0.40 and Commonlib +0.2.0 found that a test-only removal of the LiveSync mode restriction reduced +six consecutive edits from six additional revisions to one, in both directions. +Incoming identical content produced no extra conflict, while genuinely different +edits or a deletion preserved the necessary branches. A synthetic +create–edit–rename sequence still delivered intermediate content. These tests +used Vault operations, without Templater or Modal Forms themselves. + +Restart tests exposed a separate recovery gap: + +1. The event manager removes an event from its waiting map when its timer is + released. It also removes a buffered event before dispatching it. Its saved + snapshot covers buffered and waiting events, but omits work in progress. +2. `waitForIdle()` waits for the waiting gates to open, rather than for the file + handlers to finish. `ServiceFileAccessBase` reports that pending events have + been committed after this gate-only wait. Some handler failures also settle + without propagating a failure to that caller. +3. If the process stops before the database write, the file can retain the new + content while the database retains the old content and the saved event + snapshot is empty. +4. At start-up, the Offline Scanner's `EVEN` branch tries to record missing + provenance, then reports completion. It ignores that helper's Boolean result. + Different content can consequently remain unsaved when both modification + times fall in the same two-second comparison bucket. + +The last condition was reproduced by pausing the target operation immediately +before database processing, then restarting the same isolated profile. A +different timestamp bucket recovered the edit; the same bucket did not. The +changed-condition success demonstrates the boundary, rather than a fix. This +does not establish the cause of the reporter's historical conflicts. + +The runtime evidence covers Linux aarch64, Obsidian 1.12.7, and CouchDB 3.5.0. +Windows, mobile lifecycles, and the reporter's full automation remain outside +that evidence. The inspected Commonlib source is commit +`aa54282937c20d3db7f43778d240feeb1586cced`. + +## Decision + +Make local file processing recoverable until its result is established, use +content and provenance to reconcile ambiguous timestamps, and enable batching +in LiveSync mode only through an explicit choice after those contracts pass +verification. Apply the recovery contract with batching enabled or disabled. + +The content-reconciliation change targets selected ordinary files in the +Offline Scanner's `NEWER_WINS` / `sync-newer` action. Keep the existing action +matrix, explicit `DB_APPLY` policy, selection and size limits, conflict guards, +and recovery suspensions. Hidden and internal files retain their owning +feature's recovery contract; an ordinary file scan cannot recover those paths +on its behalf. + +### 1. Distinguish local persistence from replication + +The following boundaries have different meanings: + +| Boundary | Meaning | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Batch wait released | The file operation may start. Its content can still be absent from the local database. | +| Local processing completed | The intended change is represented durably by the local database and its required content, or has been proved already represented or safely superseded. | +| Replication completed | A finite replication reports its own outcome. A local save alone does not prove remote receipt or reflection into another Vault. | + +Ordinary file saving by Obsidian remains independent of database saving. Pending +and processing indicators must include work in progress and unresolved failures, +so an empty timer map cannot imply that local processing is complete. + +### 2. Keep a recoverable record through processing + +The event manager owns each operation from queue admission through completion. +Its existing persistent snapshot must represent buffered, waiting, and +in-progress work in an order which preserves deletion and rename dependencies. +Moving work between those states must not create a persistent gap. + +An event can leave the recoverable set only after its effect is durable, current +state proves that the effect is already represented, or a durably recorded +successor safely subsumes it. Finishing an older operation must not remove a +newer operation for the same path. Cancellation of a batch timer alone is not +evidence that its content is safe to forget. + +Snapshot writes must preserve mutation order. An older asynchronous write must +not replace a newer snapshot, and a failed snapshot write must remain visible +to the lifecycle caller. Reuse the existing device-local snapshot store; a +second journal and a transaction spanning the file system and database are not +part of this proposal. + +The first implementation should retain the existing event snapshot format, +including event keys and rename intent, and add pending ownership within the +event manager. The persistence adapter remains responsible for storing the +ordered snapshots. Handler outcomes must distinguish a failed attempt from a +policy-approved skip before the manager can acknowledge either safely. + +A crash after the database write but before snapshot removal can replay an +operation. Replay must check current storage and revision provenance, retain +the existing protection against stale restored events, and avoid duplicate +revisions or deletion of a newer same-path file. It must be safe to interrupt +replay again. Existing saved snapshots remain readable. + +Boolean failure, an exception, and an unavailable comparison are not successful +completion. Keep unresolved work recoverable and report the failure through the +existing service and diagnostic boundaries. Explicit exclusions or a verified +superseding change may finish work without a new revision; a generic handler +failure may not. Recovery may continue with independent paths, while retaining +the failed path and its dependencies. This proposal adds no unbounded retry loop. + +The recoverable snapshot is an operation record, not a backup of every transient +editor buffer. File content already saved by the host remains available for +start-up reconciliation if the process stops before the first snapshot write. +Crash guarantees cannot include bytes or events which the host never persisted +or delivered, or repair arbitrary storage failure. + +### 3. Give flush a finite, truthful completion boundary + +An explicit commit of pending file events captures the work accepted before +that request, releases its batch delays, and waits for the actual processing +results, including buffered and already-running work. Later edits cannot extend +that boundary indefinitely. A newer coalesced operation may satisfy earlier work +only when its successful result covers that work. + +The internal ordering barrier before a delete or rename waits only for its +predecessors. It must not wait for itself or later events in the same dispatch +loop. Replacing every gate wait with a wait for global idleness would risk a +dependency cycle or starvation. + +Manual synchronisation and replication readiness use the completed local +boundary. Settings application must observe its result before reporting success +or resuming work which depends on it. Persisting a settings file is a separate +operation: `saveSettingData()` returning is not a file-processing completion +signal, and making every settings-saved notification synchronous is unnecessary. + +When a host permits orderly shutdown, stop admitting watcher work before +draining accepted operations or preserving their pending records. Keep their +storage available until that work settles. Suspension, forced termination, and +Obsidian's synchronous unload callback cannot guarantee that asynchronous work +finishes; correctness must also follow from the persistent record and start-up +reconciliation. Remote availability must not determine whether local pending +work survives shutdown. + +### 4. Reconcile content when timestamps are inconclusive + +Retain the existing timestamp tolerance for platforms with coarse modification +times. Timestamp equality, including exactly equal raw timestamps, does not +prove content equality. The Offline Scanner must establish one of the following +outcomes through the ordinary non-forced file handler and its provenance rules: + +| Evidence | Required outcome | +| ----------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Vault content already matches an eligible current revision | Record or retain valid provenance as appropriate; create no duplicate revision. | +| Vault content matches its recorded revision or saved content evidence, while the current database revision has advanced | Apply the database state through the existing protected reflection path. Do not resurrect the unchanged older content as an edit. | +| Vault content differs from its recorded revision | Preserve the edit on that recorded branch. Keep independent concurrent branches available for conflict resolution. | +| Vault content has no provable origin and matches no current live leaf | Preserve it according to the existing independent-branch rules. Do not attach it to an arbitrary database winner. | +| Required content or provenance cannot be read, or state changes during comparison | Retain unresolved work and report or re-evaluate it. Do not mark it reconciled merely because the timestamp comparison is `EVEN`. | + +Revalidate the relevant file and database state before acting on a comparison. +Reuse the existing content and provenance checks; avoid a separate conflict +algorithm in the scanner. Metadata continues to reference Chunks containing the +file content. + +`tryRecordUntrackedFileRevision()` answers whether it could safely record +provenance. Its `false` also covers an already-populated record, conflicting +history, unavailable content, and a concurrent change. It is not an 'edited +file' predicate. A fallback which treats every `false` as permission for a +forced save would violate the required outcomes. + +Propagate the file handler's actual outcome into the pair result, including for +storage-to-database writes. Preserve the scanner's `completed`, `skipped`, and +`failed` distinctions and its existing `continueOnFileFailure` policy. Ordinary +Obsidian start-up can report `completed-with-file-failures` and allow unaffected +files to proceed; an unresolved pair must not become `completed` to achieve +that result. Keep the stricter callers' existing completion requirements. + +This deliberately extends the current equal-time scanner contract. Existing +provenance tests expect no database write during that scan, including for an +edited file. Preserve their unchanged-file and later incoming-protection +scenarios; change the edited-file expectation to require preservation during +the scan itself. Test both identical timestamps and distinct raw timestamps +within one comparison bucket. + +### 5. Make batch timing and activation explicit + +Batching is a bounded delay before local database processing. The quiet-period +timer follows the latest eligible content update; the maximum intentional delay +starts with the first uncommitted update and is not reset by later edits or time +spent awaiting a processing slot. Once due, work must become eligible to run. +The maximum delay is not a guarantee of database completion, network delivery, +or another device's reflection time. Waiting timers should not consume the +capacity needed to process unrelated ready files. + +For ordinary local content updates, queue the changed path and operation intent +without reading the file body merely to wait. Acquire the current content when +the operation becomes due, after coalescing eligible events. Reuse that content +snapshot within the operation for comparison, provenance evidence, and Chunk +creation. This retains the current ordinary-save boundary; it does not require +reading and hashing the file for every keystroke or queued notification. + +Content reuse within one operation is distinct from keeping a body cache across +edits. A cached body would need a proven invalidation and freshness contract; +mtime and size alone cannot establish that it is current. A saved provenance +hash records evidence about earlier content, rather than replacing acquisition +of the current bytes. Restart recovery, an incoming update, or a comparison +which needs revalidation can still require additional reads. Preserve those +checks rather than making a global promise of one read per path. + +Batching therefore aims to reduce repeated content acquisition, comparison, +Chunk processing, and local database updates, as well as intermediate revisions +and subsequent replication and reflection work. Event filtering, bookkeeping, +and pending-operation persistence still occur while waiting. The revision +reduction measured by the existing E2E does not quantify file reads, CPU cost, +physical disk I/O, or network traffic; measure those costs separately. Host or +operating-system caching also means that a read API call is not a physical disk +read count. + +Manual flush and required deletion or rename ordering can end a batch early. +Ordinary incoming replication continues, using the existing protection for +unsaved local content. It must not require all unrelated local batches to flush +for every incoming document. + +Add an independent `batchSaveInLiveSync` Boolean, defaulting to `false` +for both existing settings and new Vaults. In LiveSync mode, this value selects +batching; other modes continue to use `batchSave`. Both choices use the existing +minimum and maximum delay settings. The interface presents the choice for the +current mode. An ordinary mode switch preserves each mode's choice. + +This adds one persisted setting to distinguish a new explicit choice from old +`liveSync: true, batchSave: true` configurations, where batching was ignored. +Neither upgrade nor import of that legacy pair may activate batching. An older +client continues to apply its existing LiveSync restriction. Include the new +property in Commonlib's setting policy and import/export tests; do not infer +consent from the presence of the old `batchSave` property. + +The setting and scheduling changes follow the recovery work. Recovery in the +currently supported modes must pass verification before the new mode is enabled. + +### 6. Keep batching separate from automation completion + +The intended benefit is fewer short-lived intermediate revisions. Batching +cannot discover when a third-party template has finished, particularly while a +dialogue is open or another device also runs automation. A maximum delay or a +rename boundary can legitimately publish intermediate content. + +Coalescing `CREATE` followed by `CHANGED` for the same still-existing file is a +useful follow-up, but it needs its own ordering and interruption tests. Preserve +the existing cross-path rename rule: store the destination before deleting the +proven source branch. Do not retarget or discard pending source operations +without proving the effect is already covered. + +A guarantee that a multi-step template appears remotely only after completion +would need an explicit producer completion contract and its failure behaviour. +That is a separate design. Enabling batching alone must not be described as +eliminating genuine concurrent-edit conflicts or as a confirmed fix for #1207. + +## Ownership and implementation order + +1. **Commonlib recovery:** event ownership, ordered snapshot persistence, + successful completion and failure propagation, finite flush boundaries, and + same-time content reconciliation. Add focused failing regressions before + changing production code. Reuse the existing event manager, file handler, + scanner, and provenance store. +2. **Host integration:** keep the Obsidian persistence adapter and lifecycle + wiring aligned with those contracts. Verify callers of + `commitPendingFileEvents`, including settings application and database + preparation, and preserve their failure outcomes. Validate the packed + Commonlib artefact and update the consumer's exact dependency. +3. **LiveSync mode batching:** add the explicit setting and its primary-language + presentation only after the recovery regressions pass. Establish the stated + timing semantics and repeat the two-device comparisons with the actual + package, including start-up with batching already enabled. +4. **Further coalescing:** assess same-path creation/editing and any proposed + rename optimisation separately, against the original automation problem. + +Keep these changes independently reviewable. Shared behaviour belongs in +Commonlib; the plug-in must not acquire a source mirror or patch installed +package files. Snapshot format changes, a second persistent store, or a new +automation transaction API require a further design decision. + +## Verification required + +Use controlled promises at the manager and persistence boundaries for focused +regressions, and real PouchDB tests for content and ancestry. The acceptance +matrix is: + +| Scenario | Required observation | +| -------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Stop after dequeue, timer release, or during a database write | Pending work remains recoverable; restart eventually represents the saved Vault edit. | +| Database write succeeds before snapshot cleanup; restart twice | Replay creates no duplicate revision and preserves later edits. | +| Old and new events share a path; snapshot writes overlap | Completing old work cannot remove new work or regress the durable snapshot. | +| Handler returns false, throws, or snapshot persistence fails | Flush reports failure; pending work is retained; unrelated recoverable work can continue. | +| Buffered or in-progress work exists with no waiting timer | Flush still waits for actual completion. | +| A flush overlaps new typing or a rename/delete barrier | The captured work completes without waiting for later work or deadlocking. | +| Equal timestamps: unchanged, edited, untracked, pruned provenance, unavailable Chunks, or concurrent recheck changes | Preserve correct ancestry and bytes, avoid duplicate revisions, and distinguish unresolved comparisons from successful reconciliation. | +| Settings save followed by stop; ordinary unload; interruption during replay | No operation is forgotten because a settings callback or shutdown released its timer. | +| Sustained edits and more paths than available processing slots | Observe the intentional-delay deadline and progress of unrelated ready files. | +| Old settings, mode changes, explicit opt-in, and settings import | Existing LiveSync mode users retain immediate saving until they choose batching. | +| Incoming identical content, an edit, a deletion, or delete/recreate while waiting | Preserve genuine branches without extra conflicts or loss of local work. | + +Keep the existing E2E comparison between batching disabled, an ignored legacy +setting, and explicitly enabled batching. Repeat both directions, manual +synchronisation, saved-snapshot restart, and controlled in-progress restart with +the fix. Record the original failure separately from repaired results; a pass +with a different timestamp bucket or a test-only bundle is not verification of +the production fix. Run platform-specific lifecycle checks before extending the +evidence to Windows or mobile. + +## Alternatives and consequences + +- **Remove only the LiveSync guard:** leaves recovery gaps and can activate a + previously ignored setting. The explicit choice follows correctness work. +- **Wait only at unload:** cannot cover abrupt termination and hosts which do + not await asynchronous unload. Continuous recoverability is still required. +- **Repair only the scanner:** improves surviving-file recovery, but does not + establish event ordering, deletion intent, or truthful flush completion. +- **Retain only in-progress events:** closes a persistence gap, but still needs + success/failure handling, replay checks, and recovery before the first snapshot. +- **Remove timestamp rounding or force the Vault copy into the current winner:** + fails for exactly equal timestamps and can destroy branch intent. Content and + provenance determine the safe action. + +Keeping work recoverable adds snapshot bookkeeping and can make local flush +wait longer because it now waits for real processing. Same-time reconciliation +can require additional content reads. Batching trades prompt remote visibility +for fewer database updates and may increase the window for genuine concurrent +edits. Measure these costs without weakening the content-preservation contract. + +## References + +- [Conflict resolution and revision provenance](../specs_conflict_resolution.md) +- [Batch database update settings](../settings.md#batch-database-update) +- [Common library package boundary](2026_07_common_library_package_boundary.md) +- [Fast Fetch persistence and completion](2026_08_fast_fetch_persistence_and_completion.md) +- [Two-device Batch save reproduction](../../test/e2e-obsidian/scripts/issue1207-livesync-batch.ts) +- [Commonlib event manager at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/managers/StorageEventManager.ts) +- [Commonlib Offline Scanner at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceFeatures/offlineScanner.ts) +- [Commonlib file handler at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceModules/ServiceFileHandlerBase.ts) +- [Commonlib scanner policy and stored event interpretation](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/docs/offline-scanner.md) +- [Commonlib event admission and reflection contract](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/docs/storage-events-and-reflection.md) +- [Existing equal-time provenance regression](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceFeatures/offlineScanner.provenance.unit.spec.ts) diff --git a/docs/settings.md b/docs/settings.md index c65b2949..b7592846 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -669,17 +669,28 @@ Setup URIs preserve both sleep preferences. Older Setup URIs which do not contai #### Batch database update Setting key: batchSave -Reducing the frequency with which on-disk changes are reflected into the DB + +Combine repeated local file updates before saving them to the local database in modes other than LiveSync mode. This reduces intermediate database revisions and delays their availability to other devices. + +#### Batch database updates in LiveSync mode + +Setting key: batchSaveInLiveSync + +Apply the same bounded delay in LiveSync mode. This separate choice is disabled by default for both existing settings and new Vaults. An older configuration containing `liveSync: true` and `batchSave: true` does not enable it. The settings interface shows the choice for the current mode, and switching modes preserves both choices. Older clients continue to ignore batching in LiveSync mode. + +Both choices share the delays below. Manual synchronisation, deletion, or renaming can end a batch early. Batching reduces intermediate updates; it does not identify when a template has finished or prevent genuine concurrent edits on different devices. #### Minimum delay for batch database updating Setting key: batchSaveMinimumDelay -Seconds. Saving to the local database will be delayed until this value after we stop typing or saving. + +Seconds of quiet after the latest eligible edit before saving to the local database, subject to the maximum delay below. #### Maximum delay for batch database updating Setting key: batchSaveMaximumDelay -Saving will be performed forcefully after this number of seconds. + +Maximum intentional delay in seconds from the first pending update to a file. Later edits do not restart this limit, and waiting for a processing slot does not add another batch delay. This limit makes the work eligible to run; it does not guarantee database completion or receipt by another device within that time. ### 4. Deletion Propagation (Advanced) diff --git a/src/main.ts b/src/main.ts index 8aebf8ad..d4e7d346 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,4 +1,5 @@ import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation"; +import { LOG_LEVEL_URGENT } from "octagonal-wheels/common/logger"; import { getLanguage, Notice, Plugin, type App, type PluginManifest } from "./deps"; import { setGetLanguage } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; setGetLanguage(getLanguage); @@ -233,6 +234,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin { void this._startUp(); } override onunload(): void { - return void this.core.services.control.onUnload(); + void this.core.services.control.onUnload().catch((error) => { + this.core.services.API.addLog( + "Failed to unload Self-hosted LiveSync: " + String(error), + LOG_LEVEL_URGENT, + "shutdown" + ); + }); } } diff --git a/src/modules/essentialObsidian/ModuleObsidianEvents.ts b/src/modules/essentialObsidian/ModuleObsidianEvents.ts index 5f689bc6..10001ca2 100644 --- a/src/modules/essentialObsidian/ModuleObsidianEvents.ts +++ b/src/modules/essentialObsidian/ModuleObsidianEvents.ts @@ -230,7 +230,13 @@ export class ModuleObsidianEvents extends AbstractObsidianModule { } this.isLastHidden = isHidden; - await this.services.fileProcessing.commitPendingFileEvents(); + const pendingFileEventsCommitted = await this.services.fileProcessing.commitPendingFileEvents(); + if (!pendingFileEventsCommitted) { + this._log( + "Pending file events remain because their commit did not complete before the window visibility changed.", + LOG_LEVEL_NOTICE + ); + } // Desktop opt-in (LiveSync/Periodic only): keep the background channel running while the // window is hidden, instead of suspending on hide. On hide we skip the suspend for both @@ -287,7 +293,10 @@ export class ModuleObsidianEvents extends AbstractObsidianModule { if (this.settings.suspendFileWatching) return; if (!this.settings.isConfigured) return; if (!this.services.appLifecycle.isReady()) return; - await this.services.fileProcessing.commitPendingFileEvents(); + const pendingFileEventsCommitted = await this.services.fileProcessing.commitPendingFileEvents(); + if (!pendingFileEventsCommitted) { + return; + } if (file == null) { return; } diff --git a/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts b/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts index 64b92b2b..5b61464b 100644 --- a/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts +++ b/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts @@ -3,6 +3,7 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { ModuleObsidianEvents } from "./ModuleObsidianEvents"; import { DEFAULT_SETTINGS, REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { reactiveSource } from "octagonal-wheels/dataobject/reactive"; +import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; type SetupOptions = { settings?: Partial; @@ -23,6 +24,8 @@ function setup(opts: SetupOptions) { onResumed: vi.fn(async () => true), }; const fileProcessing = { commitPendingFileEvents: vi.fn(async () => true) }; + const replication = { replicateUnattendedByEvent: vi.fn(async () => undefined) }; + const conflict = { queueCheckForIfOpen: vi.fn(async () => undefined) }; const boundedRemoteActivityCount = reactiveSource(0); const boundedLocalApplicationActivityCount = reactiveSource(0); @@ -39,6 +42,8 @@ function setup(opts: SetupOptions) { setting: { saveSettingData: vi.fn(async () => undefined) }, appLifecycle, fileProcessing, + replication, + conflict, replicator: { boundedRemoteActivityCount, boundedLocalApplicationActivityCount }, }, settings: { @@ -61,6 +66,9 @@ function setup(opts: SetupOptions) { module, appLifecycle, fileProcessing, + replication, + conflict, + api: core._services.API, boundedRemoteActivityCount, boundedLocalApplicationActivityCount, }; @@ -303,6 +311,23 @@ describe("watchWindowVisibilityAsync — keepReplicationActiveInBackground", () expect(appLifecycle.onResumed).toHaveBeenCalledTimes(1); }); + it("logs a failed pending-file commit and still suspends on hide", async () => { + const { module, appLifecycle, fileProcessing, api } = setup({ + settings: { keepReplicationActiveInBackground: false, liveSync: true }, + hidden: true, + }); + fileProcessing.commitPendingFileEvents.mockResolvedValue(false); + + await module.watchWindowVisibilityAsync(); + + expect(appLifecycle.onSuspending).toHaveBeenCalledTimes(1); + expect(api.addLog).toHaveBeenCalledWith( + expect.stringContaining("Pending file events remain"), + LOG_LEVEL_NOTICE, + expect.any(String) + ); + }); + it("does not apply on mobile even if the flag is set", async () => { const { module, appLifecycle } = setup({ settings: { keepReplicationActiveInBackground: true, liveSync: true }, @@ -313,3 +338,20 @@ describe("watchWindowVisibilityAsync — keepReplicationActiveInBackground", () expect(appLifecycle.onSuspending).toHaveBeenCalledTimes(1); }); }); + +describe("watchWorkspaceOpenAsync — pending file event commit", () => { + it("skips replication and conflict checks when pending file events fail to commit", async () => { + const { module, fileProcessing, replication, conflict } = setup({ + settings: { syncOnFileOpen: true }, + hidden: false, + }); + fileProcessing.commitPendingFileEvents.mockResolvedValue(false); + + await module.watchWorkspaceOpenAsync({ path: "note.md" } as Parameters< + typeof module.watchWorkspaceOpenAsync + >[0]); + + expect(replication.replicateUnattendedByEvent).not.toHaveBeenCalled(); + expect(conflict.queueCheckForIfOpen).not.toHaveBeenCalled(); + }); +}); diff --git a/src/modules/features/SettingDialogue/PaneSyncSettings.ts b/src/modules/features/SettingDialogue/PaneSyncSettings.ts index 70a4b566..76c5b1a9 100644 --- a/src/modules/features/SettingDialogue/PaneSyncSettings.ts +++ b/src/modules/features/SettingDialogue/PaneSyncSettings.ts @@ -171,20 +171,18 @@ export function paneSyncSettings( } }); - void addPanel( - paneEl, - $msg("obsidianLiveSyncSettingTab.titleUpdateThinning"), - undefined, - visibleOnly(() => !this.isConfiguredAs("syncMode", "LIVESYNC")) - ).then((paneEl) => { - new Setting(paneEl).autoWireToggle("batchSave"); + void addPanel(paneEl, $msg("obsidianLiveSyncSettingTab.titleUpdateThinning")).then((paneEl) => { + const inLiveSyncMode = () => this.isConfiguredAs("syncMode", "LIVESYNC"); + const batchEnabled = () => this.isConfiguredAs(inLiveSyncMode() ? "batchSaveInLiveSync" : "batchSave", true); + new Setting(paneEl).autoWireToggle("batchSave", { onUpdate: visibleOnly(() => !inLiveSyncMode()) }); + new Setting(paneEl).autoWireToggle("batchSaveInLiveSync", { onUpdate: visibleOnly(inLiveSyncMode) }); new Setting(paneEl).autoWireNumeric("batchSaveMinimumDelay", { acceptZero: true, - onUpdate: visibleOnly(() => this.isConfiguredAs("batchSave", true)), + onUpdate: visibleOnly(batchEnabled), }); new Setting(paneEl).autoWireNumeric("batchSaveMaximumDelay", { acceptZero: true, - onUpdate: visibleOnly(() => this.isConfiguredAs("batchSave", true)), + onUpdate: visibleOnly(batchEnabled), }); }); diff --git a/src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts b/src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts new file mode 100644 index 00000000..ca88eaee --- /dev/null +++ b/src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts @@ -0,0 +1,98 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab"; +import type { OnUpdateFunc, PageFunctions } from "./SettingPane"; + +const controls = vi.hoisted(() => new Map { visibility?: boolean } }>()); +vi.mock("@/common/translation", () => ({ $msg: (key: string) => key })); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/logger", () => ({ Logger: vi.fn() })); +vi.mock("./LiveSyncSetting.ts", () => ({ + LiveSyncSetting: class { + autoWireDropDown(key: string, options = {}) { + controls.set(key, options); + return this; + } + autoWireToggle(key: string, options = {}) { + controls.set(key, options); + return this; + } + autoWireNumeric(key: string, options = {}) { + controls.set(key, options); + return this; + } + }, +})); + +import { paneSyncSettings } from "./PaneSyncSettings"; + +function setup(liveSync: boolean, batchSave: boolean, batchSaveInLiveSync: boolean) { + const callbacks = new Map Promise>(); + const panels = new Map(); + const tab = { + editingSettings: { ...DEFAULT_SETTINGS, liveSync, batchSave, batchSaveInLiveSync }, + services: { API: { isMobile: () => false }, control: { applySettings: vi.fn(async () => {}) } }, + addOnSaved: (key: string, callback: (value: string) => Promise) => callbacks.set(key, callback), + isConfiguredAs(key: string, value: unknown) { + if (key === "syncMode") { + return ( + (this.editingSettings.liveSync + ? "LIVESYNC" + : this.editingSettings.periodicReplication + ? "PERIODIC" + : "ONEVENTS") === value + ); + } + return this.editingSettings[key as keyof typeof this.editingSettings] === value; + }, + saveSettings: vi.fn(async () => {}), + requestCatalogueRefresh: vi.fn(), + }; + const functions = { + addPanel: (_element: HTMLElement, title: string, _callback: unknown, onUpdate?: OnUpdateFunc) => ({ + then(callback: (element: HTMLElement) => void) { + if (!title.endsWith("titleSynchronizationMethod") && !title.endsWith("titleUpdateThinning")) return; + panels.set(title, onUpdate); + callback({} as HTMLElement); + }, + }), + } as unknown as PageFunctions; + paneSyncSettings.call(tab as unknown as ObsidianLiveSyncSettingTab, {} as HTMLElement, functions); + return { tab, callbacks, panels }; +} + +function visible(key: string): boolean { + const control = controls.get(key); + if (!control) throw new Error(`Missing setting control: ${key}`); + return control.onUpdate?.().visibility !== false; +} + +beforeEach(() => controls.clear()); + +describe("mode-specific database batching settings", () => { + it.each([ + { liveSync: true, batchSave: true, batchSaveInLiveSync: false, delays: false }, + { liveSync: true, batchSave: false, batchSaveInLiveSync: true, delays: true }, + { liveSync: false, batchSave: true, batchSaveInLiveSync: false, delays: true }, + { liveSync: false, batchSave: false, batchSaveInLiveSync: true, delays: false }, + ])("shows the current mode choice and its shared delays: %j", ({ delays, ...settings }) => { + const { panels } = setup(settings.liveSync, settings.batchSave, settings.batchSaveInLiveSync); + expect(panels.get("obsidianLiveSyncSettingTab.titleUpdateThinning")?.().visibility).not.toBe(false); + expect(visible("batchSave")).toBe(!settings.liveSync); + expect(visible("batchSaveInLiveSync")).toBe(settings.liveSync); + expect(visible("batchSaveMinimumDelay")).toBe(delays); + expect(visible("batchSaveMaximumDelay")).toBe(delays); + }); + + it.each([ + [true, false], + [false, true], + ])("retains both batching choices across mode changes: %j", async (batchSave, batchSaveInLiveSync) => { + const { tab, callbacks } = setup(false, batchSave, batchSaveInLiveSync); + const switchMode = callbacks.get("syncMode")!; + for (const mode of ["LIVESYNC", "PERIODIC", "ONEVENTS", "LIVESYNC"]) { + await switchMode(mode); + expect(tab.editingSettings).toMatchObject({ batchSave, batchSaveInLiveSync }); + } + expect(tab.services.control.applySettings).toHaveBeenCalledTimes(4); + }); +}); diff --git a/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts new file mode 100644 index 00000000..10f6f6a6 --- /dev/null +++ b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts @@ -0,0 +1,961 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:net"; +import { join } from "node:path"; +import type { Page } from "playwright"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + loadCouchDbConfig, + makeUniqueDatabaseName, +} from "../runner/couchdb.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + assertE2eCompatibilityUnpaused, + configureCouchDb, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; + +type Mode = "normal" | "ignored-batch-setting" | "enabled-batch"; +type Slot = { name: string; vault: TemporaryVault; session?: ObsidianLiveSyncSession }; +type Leaf = { rev: string; ancestry: string[]; logicalDeleted: boolean; pouchDeleted: boolean; body: string | null }; +type Snapshot = { + path: string; + rev: string | null; + content: string | null; + fileMtime: number | null; + databaseMtime: number | null; + exists: boolean; + leaves: Leaf[]; + waiting: Array<{ path: string; type: string; since: number }>; + queued: Array<{ path: string | null; type: string }>; + shouldBatchSave: boolean; +}; +type Pair = { a: Page; b: Page }; + +const diagnostics = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/issue1207-livesync-batch"; +const artifactRoot = process.env.ISSUE1207_ARTIFACT_ROOT ?? process.cwd(); +const evidence: Record = { cases: [], observations: [] }; +const cases = evidence.cases as Array>; +const observations = evidence.observations as unknown[]; +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const prefix = "issue1207-batch-"; + +function check(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function hashes() { + return Object.fromEntries( + ["main.js", "manifest.json", "styles.css"].map((name) => [ + name, + createHash("sha256") + .update(new Uint8Array(readFileSync(join(artifactRoot, name)))) + .digest("hex"), + ]) + ); +} + +async function availablePort(): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + check(address && typeof address !== "string", "Missing debug port"); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + return address.port; +} + +async function evaluate(page: Page, code: string): Promise { + return await page.evaluate(`(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + const manager=core.serviceModules.storageAccess.vaultManager; + ${code} + })()`); +} + +async function installProbe(page: Page): Promise { + await evaluate( + page, + ` + if(globalThis.__issue1207Batch) return; + const probe={events:[],changes:[]}; + globalThis.__issue1207Batch=probe; + for(const type of ['create','modify','rename','delete']) { + app.vault.on(type,(file,oldPath)=>{ + const path=file.path; + if(!path.startsWith(${JSON.stringify(prefix)})) return; + const item={type,path,oldPath:oldPath??null,at:Date.now(),content:null}; + probe.events.push(item); + if(type!=='delete' && file.stat) { + void app.vault.read(file).then(body=>item.content=body).catch(error=>item.error=String(error)); + } + }); + } + probe.feed=core.localDatabase.localDatabase.changes({live:true,since:'now',include_docs:true}) + .on('change',change=>{ + if(typeof change.doc?.path==='string' && change.doc.path.startsWith(${JSON.stringify(prefix)})) { + probe.changes.push({path:change.doc.path,rev:change.doc._rev, + logicalDeleted:!!change.doc.deleted,pouchDeleted:!!change.doc._deleted,at:Date.now()}); + } + }).on('error',error=>probe.changes.push({error:String(error),at:Date.now()})); + ` + ); +} + +async function options(page: Page, mode: Mode, minimum = 2, maximum = 6): Promise { + await evaluate( + page, + ` + const settings=core.services.setting.currentSettings(); + Object.assign(settings,{liveSync:true,periodicReplication:false, + batchSave:${mode === "ignored-batch-setting"},batchSaveInLiveSync:${mode === "enabled-batch"}, + batchSaveMinimumDelay:${minimum},batchSaveMaximumDelay:${maximum},deleteMetadataOfDeletedFiles:false}); + return {liveSync:settings.liveSync,batchSave:settings.batchSave, + batchSaveInLiveSync:settings.batchSaveInLiveSync,shouldBatchSave:manager.shouldBatchSave}; + ` + ); +} + +async function snapshot(page: Page, path: string): Promise { + return await evaluate( + page, + ` + const path=${JSON.stringify(path)}; + const id=await core.services.path.path2id(path); + const row=(await core.localDatabase.allDocsRaw({keys:[id],include_docs:true})).rows[0]; + let leaves=[]; + if(row?.value?.rev) { + const rawLeaves=await core.localDatabase.getRaw(id,{open_revs:'all',revs:true}); + for(const item of rawLeaves) { + if(!item.ok) continue; + const raw=item.ok; + let body=null; + if(!raw._deleted) { + const entry=await core.localDatabase.getDBEntryFromMeta(raw,false,false,true); + if(entry!==false) body=Array.isArray(entry.data)?entry.data.join(''):entry.data; + } + leaves.push({rev:raw._rev,logicalDeleted:!!raw.deleted,pouchDeleted:!!raw._deleted,body, + ancestry:raw._revisions?.ids.map((hash,i)=>(raw._revisions.start-i)+'-'+hash)??[]}); + } + } + const file=app.vault.getAbstractFileByPath(path); + return {path,rev:row?.value?.rev??null,exists:!!file?.stat, + fileMtime:file?.stat?.mtime??null,databaseMtime:row?.doc?.mtime??null, + content:file?.stat?await app.vault.read(file):null,leaves, + waiting:[...manager._waitingMap].map(([key,value])=>({path:key,type:value.type,since:value.since})), + queued:manager.bufferedQueuedItems.map(item=>({path:item.args?.file?.path??null,type:item.type})), + shouldBatchSave:!!manager.shouldBatchSave}; + ` + ); +} + +function liveLeaves(state: Snapshot) { + return state.leaves.filter((leaf) => !leaf.logicalDeleted && !leaf.pouchDeleted); +} + +async function waitFor( + page: Page, + path: string, + predicate: (state: Snapshot) => boolean, + timeout = 20000 +): Promise { + const deadline = Date.now() + timeout; + let state: Snapshot; + do { + state = await snapshot(page, path); + if (predicate(state)) return state; + await pause(100); + } while (Date.now() < deadline); + throw new Error(`State did not settle: ${JSON.stringify(state)}`); +} + +async function settled(pair: Pair, path: string, body: string): Promise<[Snapshot, Snapshot]> { + const ready = (state: Snapshot) => + state.content === body && + liveLeaves(state).length === 1 && + liveLeaves(state)[0].body === body && + state.leaves.length === 1 && + !state.waiting.some((item) => item.path === path); + const deadline = Date.now() + 20000; + let states: [Snapshot, Snapshot]; + do { + states = await Promise.all([snapshot(pair.a, path), snapshot(pair.b, path)]); + // The row, leaves, and storage are read asynchronously. Require each + // sample to agree internally, then compare both current devices. + if ( + states.every((state) => ready(state) && state.rev === liveLeaves(state)[0].rev) && + states[0].rev === states[1].rev + ) + return states; + await pause(100); + } while (Date.now() < deadline); + throw new Error(`Devices did not converge: ${JSON.stringify(states)}`); +} + +async function create(page: Page, path: string, body: string): Promise { + await evaluate(page, `await app.vault.create(${JSON.stringify(path)},${JSON.stringify(body)});`); +} + +async function modify(page: Page, path: string, body: string): Promise { + await evaluate( + page, + `await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),${JSON.stringify(body)});` + ); +} + +async function trace(page: Page, path: string) { + return await evaluate( + page, + `return { + events:globalThis.__issue1207Batch.events.filter(item=>item.path===${JSON.stringify(path)}||item.oldPath===${JSON.stringify(path)}), + changes:globalThis.__issue1207Batch.changes.filter(item=>item.path===${JSON.stringify(path)}||item.error)};` + ); +} + +async function main(): Promise { + await mkdir(diagnostics, { recursive: true }); + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + check(cli.binary, "Missing Obsidian CLI"); + const cliBinary = cli.binary; + const config = await loadCouchDbConfig(process.env.ISSUE1207_COUCHDB_ENV); + check(["127.0.0.1", "localhost"].includes(new URL(config.uri).hostname), "Use a disposable local CouchDB"); + await assertCouchDbReachable(config); + const dbName = makeUniqueDatabaseName(config.dbPrefix, "live-batch"); + const couch = { ...config, dbName }; + const overrides = { + doctorProcessedVersion: "1.0.0", + notifyThresholdOfRemoteStorageSize: -1, + liveSync: false, + periodicReplication: false, + batchSave: false, + batchSaveInLiveSync: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + checkConflictOnlyOnOpen: true, + showMergeDialogOnlyOnActive: true, + disableMarkdownAutoMerge: true, + resolveConflictsByNewerFile: false, + writeDocumentsIfConflicted: false, + writeLogToTheFile: false, + }; + const before = hashes(); + evidence.artefactsBefore = before; + evidence.manifest = JSON.parse(await readFile(join(artifactRoot, "manifest.json"), "utf8")); + evidence.commonlibVersion = JSON.parse( + await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8") + ).version; + evidence.sourceBaseRevision = "73a14d4e4b0b414a1acfe170d01500abb3dca2e9"; + evidence.unreleasedImplementation = true; + if (process.env.ISSUE1207_COMMONLIB_ARTEFACT) { + evidence.commonlibArtefactSha256 = createHash("sha256") + .update(new Uint8Array(readFileSync(process.env.ISSUE1207_COMMONLIB_ARTEFACT))) + .digest("hex"); + } + evidence.timing = { minimumSeconds: 2, maximumSeconds: 6, incomingAndManualSeconds: { minimum: 5, maximum: 60 } }; + const slots: Slot[] = []; + let ownedDatabase = false; + + function redact(value: unknown): string { + return JSON.stringify(value, null, 2) + .split(config.password) + .join("") + .split(Buffer.from(`${config.username}:${config.password}`).toString("base64")) + .join(""); + } + + async function start(slot: Slot, restart = false) { + console.log(JSON.stringify({ phase: "starting", device: slot.name, restart })); + slot.session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + artifactRoot, + vault: slot.vault, + startupGraceMs: 1000, + pluginData: restart ? undefined : createE2eCouchDbPluginData(couch, overrides), + localStorageEntries: createE2eObsidianDeviceLocalState(slot.vault.name), + env: { E2E_OBSIDIAN_REMOTE_DEBUGGING_PORT: String(await availablePort()) }, + lifecycle: { + beforeLaunch: async () => { + for (const registry of [ + join(slot.vault.homePath, ".config", "obsidian", "obsidian.json"), + join(slot.vault.homePath, "Library", "Application Support", "obsidian", "obsidian.json"), + join(slot.vault.xdgConfigPath, "obsidian", "obsidian.json"), + join(slot.vault.userDataPath, "obsidian.json"), + ]) { + const state = JSON.parse(await readFile(registry, "utf8")); + state.updateDisabled = true; + await writeFile(registry, JSON.stringify(state)); + } + }, + }, + }); + console.log( + JSON.stringify({ + phase: "device-started", + device: slot.name, + port: slot.session.remoteDebuggingPort, + vault: slot.vault.path, + }) + ); + await waitForLiveSyncCoreReady(cliBinary, slot.session.cliEnv); + await assertE2eCompatibilityUnpaused(cliBinary, slot.session.cliEnv, slot.session.remoteDebuggingPort); + if (!restart) { + await configureCouchDb(cliBinary, slot.session.cliEnv, couch, overrides); + await waitForLiveSyncCoreReady(cliBinary, slot.session.cliEnv); + await prepareRemote(cliBinary, slot.session.cliEnv); + } + } + + async function stop(slot: Slot) { + if (!slot.session) return; + await slot.session.app.stop(); + await writeFile( + join(diagnostics, `process-${slot.name}-${slot.session.remoteDebuggingPort}.json`), + redact(slot.session.app.output()) + ); + slot.session = undefined; + } + + async function withPair(callback: (pair: Pair) => Promise): Promise { + check(slots[0].session && slots[1].session, "Both devices must be running"); + return await withObsidianPage( + slots[0].session.remoteDebuggingPort, + async (a) => + await withObsidianPage(slots[1].session!.remoteDebuggingPort, async (b) => { + await installProbe(a); + await installProbe(b); + return await callback({ a, b }); + }) + ); + } + + async function run(name: string, body: () => Promise) { + if (process.env.ISSUE1207_CASE_FILTER && !new RegExp(process.env.ISSUE1207_CASE_FILTER).test(name)) return; + console.log(JSON.stringify({ phase: "case-start", name })); + try { + const result = await body(); + cases.push({ name, passed: true, result }); + console.log(JSON.stringify({ phase: "case-passed", name })); + } catch (error) { + const failure = error instanceof Error ? error.stack : String(error); + cases.push({ name, passed: false, error: failure }); + console.log(JSON.stringify({ phase: "case-failed", name, error: failure })); + await withPair(async (pair) => { + let flushFailure: unknown; + for (const [device, page] of Object.entries(pair)) { + observations.push({ + name, + device, + probe: await evaluate( + page, + `return { + events:globalThis.__issue1207Batch.events,changes:globalThis.__issue1207Batch.changes, + pending:(await manager.adapter.persistence.loadSnapshot()).map(item=>({ + type:item.type,path:item.args?.file?.path,oldPath:item.args?.oldPath}))};` + ), + }); + await page.screenshot({ path: join(diagnostics, `${name}-${device}-failure.png`) }); + try { + await evaluate(page, "await manager.waitForIdle();"); + } catch (error) { + observations.push({ name, device, flushFailure: String(error) }); + flushFailure ??= error; + } + } + if (flushFailure) throw flushFailure; + }); + await pause(500); + } + await writeFile(join(diagnostics, "evidence.json"), redact(evidence)); + } + + try { + await createCouchDbDatabase(config, dbName); + ownedDatabase = true; + for (const name of ["a", "b"]) { + const slot = { name, vault: await createTemporaryVault(`issue1207-live-batch-${name}-`) }; + slots.push(slot); + await start(slot); + } + evidence.runtime = await withPair(async (pair) => { + const versions = []; + for (const page of [pair.a, pair.b]) { + versions.push( + await evaluate( + page, + `let appVersion=null; + try {appVersion=JSON.parse(require('fs').readFileSync(require('path').join(process.resourcesPath,'app.asar','package.json'),'utf8')).version;} catch {} + return {appVersion,electronVersion:process.versions.electron, + pluginVersion:app.plugins.plugins['obsidian-livesync'].manifest.version};` + ) + ); + await options(page, "normal"); + const outcome = await evaluate( + page, + `return await core.services.replication.startContinuous({ + trigger:'daemon',interaction:{kind:'forbidden'}});` + ); + check( + (outcome as { status?: string })?.status === "completed", + `Continuous start failed: ${JSON.stringify(outcome)}` + ); + } + return versions; + }); + + evidence.connectivity = await withPair(async (pair) => { + const path = `${prefix}connectivity.md`; + await create(pair.a, path, "Connectivity probe\n"); + const first = await settled(pair, path, "Connectivity probe\n"); + await modify(pair.b, path, "Bidirectional connectivity probe\n"); + const second = await settled(pair, path, "Bidirectional connectivity probe\n"); + return { first, second }; + }); + + for (const mode of ["normal", "ignored-batch-setting", "enabled-batch"] as const) { + for (let repeat = 0; repeat < 2; repeat++) { + await run(`${mode}-coalesce-${repeat}`, () => + withPair(async (pair) => { + for (const page of [pair.a, pair.b]) await options(page, mode); + const source = repeat === 0 ? pair.a : pair.b; + const receiver = repeat === 0 ? pair.b : pair.a; + const path = `${prefix}${mode}-edits-${repeat}.md`; + await create(source, path, "Initial body\n"); + const initial = (await settled(pair, path, "Initial body\n"))[0]; + for (let edit = 0; edit < 6; edit++) { + await modify(source, path, `Edit ${edit}\n`); + await pause(150); + } + const during = await snapshot(source, path); + const final = await settled(pair, path, "Edit 5\n"); + const finalSource = repeat === 0 ? final[0] : final[1]; + const delta = Number(finalSource.rev?.split("-")[0]) - Number(initial.rev?.split("-")[0]); + if (mode === "enabled-batch") { + check( + during.waiting.some((item) => item.path === path), + "Batch did not retain the pending edit" + ); + check(delta === 1, `Repeated edits created ${delta} revisions`); + } else { + check(!during.shouldBatchSave, "LiveSync batching was enabled without an explicit choice"); + check(delta >= 3, `The control did not expose enough edits: ${delta}`); + } + return { + path, + initial, + during, + final, + revisionDelta: delta, + source: await trace(source, path), + receiver: await trace(receiver, path), + }; + }) + ); + } + + await run(`${mode}-template-lifecycle`, () => + withPair(async (pair) => { + for (const page of [pair.a, pair.b]) await options(page, mode); + const oldPath = `${prefix}${mode}-untitled.md`; + const path = `${prefix}${mode}-completed.md`; + await create(pair.a, oldPath, ""); + await pause(300); + await modify(pair.a, oldPath, "Template in progress\n"); + await pause(300); + await evaluate( + pair.a, + `await app.fileManager.renameFile(app.vault.getFileByPath(${JSON.stringify(oldPath)}),${JSON.stringify(path)});` + ); + await pause(300); + await modify(pair.a, path, "Completed template\n"); + const during = await snapshot(pair.a, path); + const final = await settled(pair, path, "Completed template\n"); + await waitFor(pair.b, oldPath, (state) => !state.exists && liveLeaves(state).length === 0); + return { + oldPath, + path, + during, + final, + oldSource: await snapshot(pair.a, oldPath), + oldReceiver: await snapshot(pair.b, oldPath), + source: await trace(pair.a, path), + receiver: await trace(pair.b, path), + oldSourceTrace: await trace(pair.a, oldPath), + oldReceiverTrace: await trace(pair.b, oldPath), + }; + }) + ); + + for (const physical of [false, true]) { + await run(`${mode}-recreate-${physical}`, () => + withPair(async (pair) => { + for (const page of [pair.a, pair.b]) { + await options(page, mode); + await evaluate( + page, + `core.services.setting.currentSettings().deleteMetadataOfDeletedFiles=${physical};` + ); + } + const path = `${prefix}${mode}-recreate-${physical}.md`; + await create(pair.a, path, "Transient body\n"); + await pause(400); + await evaluate( + pair.a, + `await app.vault.delete(app.vault.getFileByPath(${JSON.stringify(path)}),true);` + ); + await pause(400); + await create(pair.a, path, "Recreated body\n"); + const final = await settled(pair, path, "Recreated body\n"); + return { + physical, + final, + source: await trace(pair.a, path), + receiver: await trace(pair.b, path), + }; + }) + ); + } + } + + for (const incoming of ["identical", "different", "delete"] as const) { + await run(`enabled-pending-incoming-${incoming}`, () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal", 5, 60); + const path = `${prefix}pending-${incoming}.md`; + await create(pair.b, path, "Shared base\n"); + const initial = await settled(pair, path, "Shared base\n"); + const local = "Local edit\n"; + await modify(pair.a, path, local); + const pending = await waitFor(pair.a, path, (state) => + state.waiting.some((item) => item.path === path) + ); + check(pending.rev === initial[0].rev, "Local edit reached the DB before incoming delivery"); + const sentAt = Date.now(); + if (incoming === "delete") { + await evaluate( + pair.b, + `await app.vault.delete(app.vault.getFileByPath(${JSON.stringify(path)}),true);` + ); + } else await modify(pair.b, path, incoming === "identical" ? local : "Remote edit\n"); + const arrived = await waitFor( + pair.a, + path, + (state) => + incoming === "identical" + ? state.rev !== initial[0].rev && liveLeaves(state).some((leaf) => leaf.body === local) + : state.leaves.some((leaf) => leaf.body === local) && + state.leaves.some((leaf) => + incoming === "delete" ? leaf.logicalDeleted : leaf.body === "Remote edit\n" + ), + 4000 + ); + check( + arrived.waiting.some((item) => item.path === path), + "The incoming assertion did not occur during Batch waiting" + ); + const leavesBefore = arrived.leaves.map((leaf) => leaf.rev).sort(); + if (incoming !== "identical") { + check(arrived.content === local, "Incoming reflection overwrote the pending local edit"); + check(arrived.leaves.length === 2, "Expected exactly two genuine concurrent branches"); + check( + arrived.leaves.every((leaf) => leaf.ancestry[1] === initial[0].rev), + "Concurrent branches lost their shared base" + ); + } + await pause(5600); + const after = await snapshot(pair.a, path); + const peer = await snapshot(pair.b, path); + check(!after.waiting.some((item) => item.path === path), "Batch timer did not complete"); + check( + JSON.stringify(after.leaves.map((leaf) => leaf.rev).sort()) === JSON.stringify(leavesBefore), + "The delayed event created another revision" + ); + check( + JSON.stringify(peer.leaves.map((leaf) => leaf.rev).sort()) === JSON.stringify(leavesBefore), + "Both branches did not converge to the peer" + ); + if (incoming === "identical") + check(after.leaves.length === 1, "Identical content created a branch"); + return { initial, pending, arrived, after, peer, incomingLatencyMs: Date.now() - sentAt - 5600 }; + }) + ); + } + + await run("enabled-maximum-delay", () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 2, 6); + await options(pair.b, "normal"); + const path = `${prefix}maximum.md`; + await create(pair.b, path, "Initial maximum body\n"); + const initial = await settled(pair, path, "Initial maximum body\n"); + const started = Date.now(); + let finalBody = ""; + for (let edit = 0; edit < 26; edit++) { + finalBody = `Continuous edit ${edit}\n`; + await modify(pair.a, path, finalBody); + await pause(350); + } + const stopped = Date.now(); + const final = await settled(pair, path, finalBody); + const changes = await evaluate>( + pair.a, + `return globalThis.__issue1207Batch.changes.filter(item=>item.path===${JSON.stringify(path)}&&item.rev!==${JSON.stringify(initial[0].rev)});` + ); + check(changes.length >= 2, "Continuous edits did not produce bounded intermediate and final saves"); + const first = changes[0].at - started; + check(first >= 5000 && first <= 8000, `Unexpected first maximum-delay flush: ${first}ms`); + check(changes[0].at < stopped - 1000, "The maximum flush only occurred after typing stopped"); + return { started, stopped, firstSaveMs: first, changes, final }; + }) + ); + + await run("enabled-many-paths", () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 2, 6); + await options(pair.b, "normal"); + const started = Date.now(); + for (let index = 0; index < 12; index++) + await create(pair.a, `${prefix}many-${index}.md`, `File ${index}\n`); + const final = []; + for (let index = 0; index < 12; index++) + final.push(await settled(pair, `${prefix}many-${index}.md`, `File ${index}\n`)); + return { elapsedMs: Date.now() - started, final }; + }) + ); + + await run("enabled-manual-sync", () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal"); + const path = `${prefix}manual.md`; + await create(pair.b, path, "Before manual sync\n"); + await settled(pair, path, "Before manual sync\n"); + await modify(pair.a, path, "After manual sync\n"); + const pending = await waitFor(pair.a, path, (state) => + state.waiting.some((item) => item.path === path) + ); + const started = Date.now(); + const outcome = await evaluate(pair.a, "return await core.services.replication.replicate(true);"); + const atReturn = await snapshot(pair.a, path); + const final = await settled(pair, path, "After manual sync\n"); + const elapsedMs = Date.now() - started; + check(elapsedMs < 4000, `Manual sync waited for the Batch timer: ${elapsedMs}ms`); + return { pending, outcome, atReturn, final, elapsedMs }; + }) + ); + + const saveAfterEdit = process.env.ISSUE1207_RESTART_SAVE_AFTER_EDIT === "true"; + const restartBucket = process.env.ISSUE1207_RESTART_BUCKET; + check( + !restartBucket || ["same", "exact", "different"].includes(restartBucket), + "Unknown restart timestamp condition" + ); + const restartName = restartBucket + ? `enabled-inflight-restart-${restartBucket}` + : saveAfterEdit + ? "enabled-settings-save-restart" + : "enabled-restart"; + await run(restartName, async () => { + const path = `${prefix}restart.md`; + const beforeRestart = await withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal"); + if (!saveAfterEdit) + await evaluate( + pair.a, + "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" + ); + await create(pair.b, path, "Before restart\n"); + const initial = await settled(pair, path, "Before restart\n"); + if (restartBucket) { + check(initial[0].databaseMtime !== null, "Missing initial database mtime"); + const mtime = + restartBucket === "exact" + ? initial[0].databaseMtime + : Math.floor(initial[0].databaseMtime / 2000) * 2000 + + (restartBucket === "same" ? 1000 : 3000); + await evaluate( + pair.a, + `const original=manager.requestProcessQueue.bind(manager); + manager.requestProcessQueue=async item=>{ + if(item.args?.file?.path===${JSON.stringify(path)}) { + globalThis.__issue1207Batch.inflightPaused=true; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + return await original(item); + }; + await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),"Retained after restart\\n",{mtime:${mtime}});` + ); + } else await modify(pair.a, path, "Retained after restart\n"); + const pending = await waitFor(pair.a, path, (state) => + state.waiting.some((item) => item.path === path) + ); + if (saveAfterEdit) await evaluate(pair.a, "await core.services.setting.saveSettingData();"); + if (restartBucket) + await evaluate( + pair.a, + `const deadline=Date.now()+3000; + while(!globalThis.__issue1207Batch.inflightPaused&&Date.now()setTimeout(resolve,10)); + if(!globalThis.__issue1207Batch.inflightPaused) throw new Error('The in-flight test gate did not start');` + ); + await evaluate(pair.a, "await manager._takeSnapshot();"); + check(pending.rev === initial[0].rev, "Restart test did not retain a pending edit"); + if (restartBucket) { + const inFlight = await snapshot(pair.a, path); + const saved = await evaluate>( + pair.a, + `return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));` + ); + check( + inFlight.rev === initial[0].rev && !inFlight.waiting.some((item) => item.path === path), + "The in-flight fixture did not stop between gate release and database storage" + ); + check( + saved.some((item) => item.path === path), + "The in-flight event was lost from the recoverable snapshot" + ); + check(inFlight.fileMtime !== null && inFlight.databaseMtime !== null, "Missing in-flight mtimes"); + const sameBucket = + Math.floor(inFlight.fileMtime / 2000) === Math.floor(inFlight.databaseMtime / 2000); + check(sameBucket === (restartBucket !== "different"), "The timestamp condition was not applied"); + if (restartBucket === "exact") + check(inFlight.fileMtime === inFlight.databaseMtime, "Raw mtimes differ"); + observations.push({ + phase: "inflight-before-stop", + restartBucket, + pauseBeforeDatabaseMs: 30000, + inFlight, + saved, + sameBucket, + }); + } + if (!saveAfterEdit && !restartBucket) { + const strictState = await snapshot(pair.a, path); + const saved = await evaluate>( + pair.a, + `return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));` + ); + check( + strictState.rev === initial[0].rev && strictState.waiting.some((item) => item.path === path), + "Restart fixture flushed before shutdown" + ); + check( + saved.some((item) => item.path === path), + "Restart fixture did not persist the pending event" + ); + observations.push({ + phase: "restart-verified-pending-snapshot", + strictState, + saved, + initialized: await evaluate(pair.a, "return await core.kvDB.get('initialized');"), + }); + } + if (!saveAfterEdit) + observations.push({ + phase: "restart-before-stop", + initial, + pending, + persisted: await evaluate( + pair.a, + `const events=await manager.adapter.persistence.loadSnapshot(); + const settings=core.services.setting.currentSettings(); + return {initialized:await core.kvDB.get('initialized'),restorationRequested:manager.snapShotRestored!==null, + settings:{isConfigured:settings.isConfigured,liveSync:settings.liveSync,batchSave:settings.batchSave,batchSaveInLiveSync:settings.batchSaveInLiveSync, + suspendFileWatching:settings.suspendFileWatching}, + events:events?.map(item=>({type:item.type,path:item.args?.file?.path, + stat:item.args?.file?.stat,skipBatchWait:item.skipBatchWait}))};` + ), + }); + return { initial, pending }; + }); + await stop(slots[0]); + const persistedSettings = JSON.parse( + await readFile(join(slots[0].vault.path, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8") + ); + observations.push({ + phase: "restart-on-disk-settings", + settings: Object.fromEntries( + [ + "isConfigured", + "liveSync", + "batchSave", + "batchSaveInLiveSync", + "suspendFileWatching", + "settingVersion", + ].map((key) => [key, persistedSettings[key]]) + ), + }); + await start(slots[0], true); + const afterRestart = await withPair(async (pair) => { + observations.push({ + phase: "restart-after-boot", + state: await snapshot(pair.a, path), + persisted: await evaluate( + pair.a, + `const events=await manager.adapter.persistence.loadSnapshot(); + const settings=core.services.setting.currentSettings(); + return {initialized:await core.kvDB.get('initialized'),restorationRequested:manager.snapShotRestored!==null, + settings:{isConfigured:settings.isConfigured,liveSync:settings.liveSync,batchSave:settings.batchSave,batchSaveInLiveSync:settings.batchSaveInLiveSync, + suspendFileWatching:settings.suspendFileWatching}, + events:events?.map(item=>({type:item.type,path:item.args?.file?.path,stat:item.args?.file?.stat}))};` + ), + }); + const result = await settled(pair, path, "Retained after restart\n"); + check(result[0].shouldBatchSave, "The saved LiveSync batching choice was not restored"); + return { result, restoredGuard: result[0].shouldBatchSave }; + }); + return { beforeRestart, afterRestart }; + }); + + await run("enabled-after-write-restart", async () => { + const path = `${prefix}after-write.md`; + const saved = await withPair(async (pair) => { + await options(pair.a, "enabled-batch", 1, 3); + await options(pair.b, "normal"); + await evaluate( + pair.a, + "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" + ); + await create(pair.b, path, "Before the durable write\n"); + const initial = await settled(pair, path, "Before the durable write\n"); + await evaluate( + pair.a, + ` + const original=manager.requestProcessQueue.bind(manager); + manager.requestProcessQueue=async item=>{ + await original(item); + if(item.args?.file?.path===${JSON.stringify(path)}) { + globalThis.__issue1207Batch.afterWritePaused=true; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + }; + await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),"Durable before restart\\n"); + const deadline=Date.now()+6000; + while(!globalThis.__issue1207Batch.afterWritePaused&&Date.now()setTimeout(resolve,20)); + if(!globalThis.__issue1207Batch.afterWritePaused) throw new Error('The after-write gate did not start'); + await manager._takeSnapshot(); + ` + ); + const durable = await snapshot(pair.a, path); + const pending = await evaluate>( + pair.a, + "return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));" + ); + check(durable.rev !== initial[0].rev, "The database write did not complete before interruption"); + check( + pending.some((item) => item.path === path), + "The after-write operation was not recoverable" + ); + return { initial, durable, pending }; + }); + const restarts = []; + for (let attempt = 0; attempt < 2; attempt++) { + await stop(slots[0]); + await start(slots[0], true); + const final = await withPair((pair) => settled(pair, path, "Durable before restart\n")); + check(final[0].rev === saved.durable.rev, "Replay created a duplicate database revision"); + restarts.push(final); + } + const later = await withPair(async (pair) => { + await modify(pair.a, path, "A later edit survives\n"); + return await settled(pair, path, "A later edit survives\n"); + }); + return { saved, restarts, later }; + }); + + await run("enabled-before-snapshot-restart-exact", async () => { + const path = `${prefix}before-snapshot.md`; + const beforeStop = await withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal"); + await evaluate( + pair.a, + "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" + ); + await create(pair.b, path, "Before the first snapshot\n"); + const initial = await settled(pair, path, "Before the first snapshot\n"); + check(initial[0].databaseMtime !== null, "Missing initial database mtime"); + await evaluate( + pair.a, + ` + await manager.waitForIdle(); + const persistence=manager.adapter.persistence; + const original=persistence.saveSnapshot.bind(persistence); + persistence.saveSnapshot=async events=>{ + if(events.some(item=>item.type==='CHANGED'&&item.args?.file?.path===${JSON.stringify(path)})) { + globalThis.__issue1207Batch.beforeSnapshotPaused=true; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + return await original(events); + }; + await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}), + "Recovered without an event snapshot\\n",{mtime:${initial[0].databaseMtime}}); + const deadline=Date.now()+3000; + while(!globalThis.__issue1207Batch.beforeSnapshotPaused&&Date.now()setTimeout(resolve,10)); + if(!globalThis.__issue1207Batch.beforeSnapshotPaused) + throw new Error('The snapshot persistence gate did not start'); + ` + ); + const state = await snapshot(pair.a, path); + const persisted = await evaluate>( + pair.a, + "return (await manager.adapter.persistence.loadSnapshot()??[]).map(item=>({path:item.args?.file?.path}));" + ); + check(state.rev === initial[0].rev, "The unsaved edit reached the database before interruption"); + check(state.fileMtime === state.databaseMtime, "Raw mtimes differ before interruption"); + check(!persisted.some((item) => item.path === path), "The interrupted edit already has a saved event"); + return { initial, state, persisted }; + }); + await stop(slots[0]); + await start(slots[0], true); + const recovered = await withPair((pair) => settled(pair, path, "Recovered without an event snapshot\n")); + check(recovered[0].shouldBatchSave, "The saved LiveSync batching choice was not restored"); + return { beforeStop, recovered }; + }); + + evidence.artefactsAfter = hashes(); + check( + JSON.stringify(before) === JSON.stringify(evidence.artefactsAfter), + "Plug-in artefacts changed during testing" + ); + evidence.passed = cases.every((item) => item.passed); + await writeFile(join(diagnostics, "evidence.json"), redact(evidence)); + console.log( + JSON.stringify({ + phase: "completed", + passed: evidence.passed, + cases: cases.length, + failed: cases.filter((item) => !item.passed).map((item) => item.name), + }) + ); + if (!evidence.passed) process.exitCode = 1; + } finally { + for (const slot of slots) await stop(slot); + for (const slot of slots) await slot.vault.dispose(); + if (ownedDatabase) await deleteCouchDbDatabase(config, dbName); + evidence.cleanup = { devicesStopped: slots.every((slot) => !slot.session), databaseRemoved: ownedDatabase }; + await writeFile(join(diagnostics, "evidence.json"), redact(evidence)); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/updates.md b/updates.md index cefb2bce..e3e425e5 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,21 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### New Feature + +- We can enable **Batch database updates in LiveSync mode** to combine repeated local edits before sending them to other devices. + - The choice starts disabled, including when older settings contain the previously ignored batch option. Each mode retains its own choice and uses the same minimum and maximum delays. + - Batching delays remote visibility and reduces intermediate updates. It does not establish when a template has finished or prevent genuine concurrent edits. + +#### Fixed + +- Local changes remain available for recovery while their database save is running, including when the application stops after a batch delay ends. + - Manual synchronisation and settings application now wait for the accepted local saves to finish and report failures. Pending events retain the existing snapshot format. +- Start-up reconciliation checks content when modification times are equal, so an edit within the same timestamp interval is not mistaken for an unchanged file. + - Existing revision provenance preserves the correct edit branch. Unreadable or continually changing content remains unresolved rather than being reported as successfully reconciled. + ## 1.0.40 24th October, 2026 From 68dc40ff05f429c8b47448dbbfa9c27ee937179e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 05:40:10 +0000 Subject: [PATCH 21/28] Show CouchDB comparison progress in customisable status items --- README.md | 18 + package.json | 1 + src/modules/features/ModuleLog.ts | 129 ++- .../features/ReplicationStatusLabel.ts | 209 +++++ .../ReplicationStatusLabel.unit.spec.ts | 241 ++++++ src/modules/features/StatusBarDisplay.ts | 51 +- .../features/StatusBarDisplay.unit.spec.ts | 18 + src/modules/features/StatusBarRenderer.ts | 151 ++++ .../features/StatusBarRenderer.unit.spec.ts | 300 +++++++ styles.css | 15 + test/e2e-obsidian/README.md | 3 + .../scripts/replication-status.ts | 770 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + updates.md | 2 + 14 files changed, 1837 insertions(+), 72 deletions(-) create mode 100644 src/modules/features/ReplicationStatusLabel.ts create mode 100644 src/modules/features/ReplicationStatusLabel.unit.spec.ts create mode 100644 src/modules/features/StatusBarRenderer.ts create mode 100644 src/modules/features/StatusBarRenderer.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/replication-status.ts diff --git a/README.md b/README.md index f84b0267..38c78d2c 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Synchronisation status is shown in the status bar with the following icons. - 🌐N Approximate remote requests currently in progress - Status - ⏹️ Stopped + - ⏩ Comparing CouchDB change history, including entries already held locally - 💤 LiveSync enabled. Waiting for changes - ⚡️ Synchronisation in progress - ⚠ An error occurred @@ -104,6 +105,23 @@ Synchronisation status is shown in the status bar with the following icons. To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files. +When CouchDB change history is being compared, `↓ 0 (⏩ 1200)` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It disappears after three seconds without a change and reappears when the count changes. The ⏩ status icon follows comparison activity independently of that timer. + +### Customising status items with CSS snippets + +The status bar and editor overlay use individual spans with the common `livesync-stat-` class prefix. CSS snippets can hide or style the items we need. For example, this hides the comparison count and P2P summary: + +```css +.livesync-stat-ff, +.livesync-stat-p2p { + display: none; +} +``` + +The item classes include `livesync-stat-network`, `livesync-stat-sync`, `livesync-stat-sent`, `livesync-stat-push-remaining`, `livesync-stat-arrived`, `livesync-stat-pull-remaining`, `livesync-stat-ff`, and `livesync-stat-p2p`. File-processing items use `livesync-stat-file-processing`, `livesync-stat-file-pending`, and `livesync-stat-file-batched`; queue items use `livesync-stat-queue-` followed by `replication`, `database`, `storage`, `chunk-initial`, `chunk-retrying`, `plugin-scan`, `hidden-files`, or `conflicts`. The skipped-file indicator uses `livesync-stat-file-skip`. + +All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count until it is hidden, reducing movement as counts decrease. + ## Tips and Troubleshooting - If you want a faster and simpler initial replication when setting up subsequent devices, see the [Fast Setup Guide](docs/tips/fast-setup.md). - Configure [Hidden File Sync](docs/tips/hidden-file-sync.md) only after ordinary note synchronisation works. diff --git a/package.json b/package.json index cd383dde..90d9e730 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "test:e2e:obsidian:configuration-encryption-sync": "tsx test/e2e-obsidian/scripts/configuration-encryption-sync.ts", "test:e2e:obsidian:review-harness": "tsx test/e2e-obsidian/scripts/review-harness.ts", "test:e2e:obsidian:p2p-pane": "tsx test/e2e-obsidian/scripts/p2p-pane.ts", + "test:e2e:obsidian:replication-status": "tsx test/e2e-obsidian/scripts/replication-status.ts", "test:e2e:obsidian:vault-reflection": "tsx test/e2e-obsidian/scripts/vault-reflection.ts", "test:e2e:obsidian:couchdb-upload": "tsx test/e2e-obsidian/scripts/couchdb-upload.ts", "test:e2e:obsidian:couchdb-http-cache": "tsx test/e2e-obsidian/scripts/couchdb-http-cache.ts", diff --git a/src/modules/features/ModuleLog.ts b/src/modules/features/ModuleLog.ts index 99170ec1..0755506b 100644 --- a/src/modules/features/ModuleLog.ts +++ b/src/modules/features/ModuleLog.ts @@ -4,7 +4,6 @@ import { LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE, PREFIXMD_LOGFILE, - type DatabaseConnectingStatus, type LOG_LEVEL, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { scheduleTask } from "octagonal-wheels/concurrency/task"; @@ -31,16 +30,14 @@ import { LogPaneView, VIEW_TYPE_LOG } from "./Log/LogPaneView.ts"; import { serialized } from "octagonal-wheels/concurrency/lock"; import { $msg } from "@/common/translation"; import { P2PLogCollector } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/P2PLogCollector"; -import { - REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS, - formatRemoteActivityStatusLabel, - getTrackedRequestCount, -} from "./RemoteActivityStatus.ts"; +import { REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS, getTrackedRequestCount } from "./RemoteActivityStatus.ts"; import { createChunkFetchCounterLabel, createMinimumVisibleActivityCount, createPaddedCounterLabel, } from "./StatusBarDisplay.ts"; +import { createFastForwardCheckedChangesDisplay, projectReplicationStatusLabel } from "./ReplicationStatusLabel.ts"; +import { createStatusBarRenderer, type StatusBarPart, type StatusBarRenderer } from "./StatusBarRenderer.ts"; import type { LiveSyncCore } from "@/main.ts"; import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError"; import { isValidPath } from "@/common/utils.ts"; @@ -124,7 +121,14 @@ export class ModuleLog extends AbstractObsidianModule { logHistory?: HTMLDivElement; messageArea?: HTMLDivElement; - statusBarLabels!: ReactiveValue<{ message: string; status: string }>; + statusBarRenderer?: StatusBarRenderer; + statusLineRenderer?: StatusBarRenderer; + statusBarLabels!: ReactiveValue<{ + message: string; + status: string; + parts: readonly StatusBarPart[]; + statusBarParts: readonly StatusBarPart[]; + }>; statusLog = reactiveSource(""); activeFileStatus = reactiveSource(""); notifies: { [key: string]: { count: number } } = {}; @@ -164,79 +168,54 @@ export class ModuleLog extends AbstractObsidianModule { ); const requestingStatLabel = computed(() => { - return formatRemoteActivityStatusLabel({ + return { remoteOperationCount: Math.max(0, this.services.replicator.boundedRemoteActivityCount.value), trackedRequestCount: displayedTrackedRequestCount.value, - }); + }; }); - const replicationStatLabel = computed(() => { - const e = this.services.replicator.replicationStatics.value; - const sent = e.sent; - const arrived = e.arrived; - const maxPullSeq = e.maxPullSeq; - const maxPushSeq = e.maxPushSeq; - const lastSyncPullSeq = e.lastSyncPullSeq; - const lastSyncPushSeq = e.lastSyncPushSeq; - let pushLast = ""; - let pullLast = ""; - let w = ""; - const labels: Partial> = { - CONNECTED: "⚡", - JOURNAL_SEND: "📦↑", - JOURNAL_RECEIVE: "📦↓", - }; - switch (e.syncStatus) { - case "CLOSED": - case "COMPLETED": - case "NOT_CONNECTED": - w = "⏹"; - break; - case "STARTED": - w = "🌀"; - break; - case "PAUSED": - w = "💤"; - break; - case "CONNECTED": - case "JOURNAL_SEND": - case "JOURNAL_RECEIVE": - w = labels[e.syncStatus] || "⚡"; - pushLast = - lastSyncPushSeq == 0 - ? "" - : lastSyncPushSeq >= maxPushSeq - ? " (LIVE)" - : ` (${maxPushSeq - lastSyncPushSeq})`; - pullLast = - lastSyncPullSeq == 0 - ? "" - : lastSyncPullSeq >= maxPullSeq - ? " (LIVE)" - : ` (${maxPullSeq - lastSyncPullSeq})`; - break; - case "ERRORED": - w = "⚠"; - break; - default: - w = "?"; - } - return { w, sent, pushLast, arrived, pullLast }; - }); + const fastForwardCount = registerDisplay( + createFastForwardCheckedChangesDisplay(this.services.replicator.replicationStatics) + ); + const replicationStatLabel = computed(() => + projectReplicationStatusLabel( + this.services.replicator.replicationStatics.value, + requestingStatLabel(), + fastForwardCount.value + ) + ); const labelProc = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.processing, `⏳`)); const labelPend = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.totalQueued, `🛫`)); const labelInBatchDelay = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.batched, `📬`)); const waitingLabel = computed(() => { return `${labelProc.value}${labelPend.value}${labelInBatchDelay.value}`; }); + const counterPart = (key: string, mark: string, value: number | undefined): StatusBarPart[] => + value === undefined ? [] : [{ key, text: " " + mark, value }]; const statusLineLabel = computed(() => { - const { w, sent, pushLast, arrived, pullLast } = replicationStatLabel(); + const replication = replicationStatLabel(); const queued = queueCountLabel(); const waiting = waitingLabel(); - const networkActivity = requestingStatLabel(); const p2p = this.p2pLogCollector.p2pReplicationLine.value; + const firstLineParts = [ + ...replication.parts, + ...counterPart("file-processing", "⏳", labelProc.count.value), + ...counterPart("file-pending", "🛫", labelPend.count.value), + ...counterPart("file-batched", "📬", labelInBatchDelay.count.value), + ...counterPart("queue-replication", "📥", labelReplication.count.value), + ...counterPart("queue-database", "📄", labelDBCount.count.value), + ...counterPart("queue-storage", "💾", labelStorageCount.count.value), + ...counterPart("queue-chunk-initial", "🛄", labelChunkCount.initialCount.value), + ...counterPart("queue-chunk-retrying", "🔁", labelChunkCount.retryingCount.value), + ...counterPart("queue-plugin-scan", "🔌", labelPluginScanCount.count.value), + ...counterPart("queue-hidden-files", "⚙️", labelHiddenFilesCount.count.value), + ...counterPart("queue-conflicts", "🔩", labelConflictProcessCount.count.value), + ]; return { - message: `${networkActivity}Sync: ${w} ↑ ${sent}${pushLast} ↓ ${arrived}${pullLast}${waiting}${queued}${p2p == "" ? "" : "\n" + p2p}`, + message: replication.message + waiting + queued + (p2p === "" ? "" : "\n" + p2p), + parts: p2p === "" ? firstLineParts : [...firstLineParts, { key: "p2p", text: "\n" + p2p }], + statusBarParts: firstLineParts, + hasP2P: p2p !== "", }; }); @@ -244,13 +223,16 @@ export class ModuleLog extends AbstractObsidianModule { const scheduleMessage = this.services.appLifecycle.isReloadingScheduled() ? `WARNING! RESTARTING OBSIDIAN IS SCHEDULED!\n` : ""; - const { message } = statusLineLabel(); + const { message, parts, statusBarParts, hasP2P } = statusLineLabel(); const fileStatus = this.activeFileStatus.value; const status = scheduleMessage + this.statusLog.value; const fileStatusIcon = `${fileStatus && this.settings.hideFileWarningNotice ? " ⛔ SKIP" : ""}`; + const skipPart = fileStatusIcon === "" ? [] : [{ key: "file-skip", text: fileStatusIcon }]; return { message: `${message}${fileStatusIcon}`, status, + parts: [...parts, ...skipPart], + statusBarParts: hasP2P ? statusBarParts : [...statusBarParts, ...skipPart], }; }); this.statusBarLabels = statusBarLabels; @@ -391,7 +373,8 @@ export class ModuleLog extends AbstractObsidianModule { newLog = newLog.substring(moduleTagEnd + MARK_LOG_SEPARATOR.length + 1); } - this.statusBar?.setText(newMsg.split("\n")[0]); + const { parts, statusBarParts } = this.statusBarLabels.value; + this.statusBarRenderer?.render(statusBarParts); if (this.statusDiv) { this.statusDiv.setCssStyles({ display: this.settings?.showStatusOnEditor ? "" : "none" }); } @@ -409,7 +392,7 @@ export class ModuleLog extends AbstractObsidianModule { const recentLogs = recent.reverse().join("\n"); if (isDirty("recentLogs", recentLogs)) this.logHistory!.innerText = recentLogs; } - if (isDirty("newMsg", newMsg)) this.statusLine!.innerText = newMsg; + if (isDirty("newMsg", newMsg)) this.statusLineRenderer?.render(parts); if (isDirty("newLog", newLog)) this.logMessage!.innerText = newLog; } else { // const root = activeDocument.documentElement; @@ -505,6 +488,16 @@ ${stringifyYaml(info)} this.statusBar = this.services.API.addStatusBarItem(); this.statusBar?.addClass("syncstatusbar"); } + if (this.statusLine) { + const renderer = createStatusBarRenderer(this.statusLine); + this.statusLineRenderer = renderer; + this.plugin.register(() => renderer.dispose()); + } + if (this.statusBar) { + const renderer = createStatusBarRenderer(this.statusBar); + this.statusBarRenderer = renderer; + this.plugin.register(() => renderer.dispose()); + } this.adjustStatusDivPosition(); this._log("Log module loaded", LOG_LEVEL_INFO); this._log("Verbose log", LOG_LEVEL_VERBOSE); diff --git a/src/modules/features/ReplicationStatusLabel.ts b/src/modules/features/ReplicationStatusLabel.ts new file mode 100644 index 00000000..b36d4c95 --- /dev/null +++ b/src/modules/features/ReplicationStatusLabel.ts @@ -0,0 +1,209 @@ +import { reactiveSource, type ReactiveValue } from "octagonal-wheels/dataobject/reactive"; +import type { ReplicationStatics } from "@vrtmrz/livesync-commonlib/replication"; +import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; + +import type { DisposableReactiveValue } from "./StatusBarDisplay.ts"; +import type { StatusBarPart } from "./StatusBarRenderer.ts"; +import { + formatRemoteActivityStatusLabel, + REMOTE_OPERATION_ACTIVITY_ICON, + REMOTE_REQUEST_ACTIVITY_ICON, + type RemoteActivityStatus, +} from "./RemoteActivityStatus.ts"; + +export const FAST_FORWARD_COUNT_LINGER_MS = 3_000; + +function readPositiveCheckedChanges(source: ReactiveValue): number | undefined { + const count = source.value.couchDB?.pull?.checkedChanges; + return typeof count === "number" && Number.isFinite(count) && count > 0 ? count : undefined; +} + +/** + * Briefly displays changing CouchDB pull-feed comparison counts. + * + * Unrelated replication-stat changes do not extend the linger or redisplay an + * expired count. A positive initial snapshot is displayed to late subscribers. + */ +export function createFastForwardCheckedChangesDisplay( + source: ReactiveValue, + lingerMs = FAST_FORWARD_COUNT_LINGER_MS +): DisposableReactiveValue { + const displayed = reactiveSource(undefined); + const linger = Math.max(0, lingerMs); + let lastObserved = readPositiveCheckedChanges(source); + let clearTimer: number | undefined; + let disposed = false; + + const cancelClear = () => { + if (clearTimer === undefined) return; + compatGlobal.clearTimeout(clearTimer); + clearTimer = undefined; + }; + const scheduleClear = () => { + clearTimer = compatGlobal.setTimeout(() => { + clearTimer = undefined; + if (disposed) return; + if (readPositiveCheckedChanges(source) !== lastObserved) { + update(); + return; + } + displayed.value = undefined; + }, linger); + }; + const update = () => { + if (disposed) return; + const nextCount = readPositiveCheckedChanges(source); + if (nextCount === lastObserved) return; + lastObserved = nextCount; + cancelClear(); + if (nextCount === undefined) { + displayed.value = undefined; + return; + } + displayed.value = nextCount; + scheduleClear(); + }; + + if (lastObserved !== undefined) { + displayed.value = lastObserved; + scheduleClear(); + } + source.onChanged(update); + + return { + get value() { + return displayed.value; + }, + onChanged(handler) { + displayed.onChanged(handler); + }, + offChanged(handler) { + displayed.offChanged(handler); + }, + dispose() { + if (disposed) return; + disposed = true; + cancelClear(); + source.offChanged(update); + }, + }; +} + +export type ReplicationStatusProjection = { + icon: string; + message: string; + parts: StatusBarPart[]; +}; + +function remainingSequenceLabel( + key: string, + lastSyncSeq: number, + maxSeq: number +): { text: string; part?: StatusBarPart } { + if (lastSyncSeq === 0) return { text: "" }; + if (lastSyncSeq >= maxSeq) return { text: " (LIVE)", part: { key, text: " (LIVE)" } }; + const remaining = maxSeq - lastSyncSeq; + return { + text: " (" + remaining + ")", + part: { key, text: " (", value: remaining, suffix: ")" }, + }; +} + +/** Projects the existing replication labels into reusable numeric status items. */ +export function projectReplicationStatusLabel( + stats: ReplicationStatics, + networkStatus: RemoteActivityStatus, + fastForwardCount?: number +): ReplicationStatusProjection { + const { sent, arrived, maxPullSeq, maxPushSeq, lastSyncPullSeq, lastSyncPushSeq } = stats; + const networkActivity = formatRemoteActivityStatusLabel(networkStatus); + let pushRemaining: { text: string; part?: StatusBarPart } = { text: "" }; + let pullRemaining: { text: string; part?: StatusBarPart } = { text: "" }; + let icon: string; + switch (stats.syncStatus) { + case "CLOSED": + case "COMPLETED": + case "NOT_CONNECTED": + icon = "⏹"; + break; + case "STARTED": + icon = "🌀"; + break; + case "PAUSED": + icon = "💤"; + break; + case "CONNECTED": + case "JOURNAL_SEND": + case "JOURNAL_RECEIVE": + icon = stats.syncStatus === "JOURNAL_SEND" ? "📦↑" : stats.syncStatus === "JOURNAL_RECEIVE" ? "📦↓" : "⚡"; + pushRemaining = remainingSequenceLabel("push-remaining", lastSyncPushSeq, maxPushSeq); + pullRemaining = remainingSequenceLabel("pull-remaining", lastSyncPullSeq, maxPullSeq); + break; + case "ERRORED": + icon = "⚠"; + break; + default: + icon = "?"; + } + + const couchDB = stats.couchDB; + const isFastForwardChecking = + stats.syncStatus !== "ERRORED" && + couchDB !== undefined && + couchDB.phase !== "idle" && + (couchDB.pull?.checking === true || couchDB.push?.checking === true); + if (isFastForwardChecking) icon = "⏩"; + + const visibleFastForwardCount = + couchDB !== undefined && + typeof fastForwardCount === "number" && + Number.isFinite(fastForwardCount) && + fastForwardCount > 0 + ? fastForwardCount + : undefined; + const fastForwardSuffix = visibleFastForwardCount === undefined ? "" : " (⏩ " + visibleFastForwardCount + ")"; + + const parts: StatusBarPart[] = []; + if (networkStatus.trackedRequestCount > 0) { + parts.push({ + key: "network", + text: + (networkStatus.remoteOperationCount > 0 ? `${REMOTE_OPERATION_ACTIVITY_ICON} ` : "") + + REMOTE_REQUEST_ACTIVITY_ICON, + value: networkStatus.trackedRequestCount, + suffix: " ", + }); + } else if (networkActivity !== "") { + parts.push({ key: "network", text: networkActivity }); + } + parts.push({ key: "sync", text: "Sync: " + icon }); + parts.push({ key: "sent", text: " ↑ ", value: sent }); + if (pushRemaining.part !== undefined) parts.push(pushRemaining.part); + parts.push({ key: "arrived", text: " ↓ ", value: arrived }); + if (pullRemaining.part !== undefined) parts.push(pullRemaining.part); + if (visibleFastForwardCount !== undefined) { + parts.push({ + key: "ff", + text: " (⏩ ", + value: visibleFastForwardCount, + suffix: ")", + title: "Compared CouchDB feed entries", + }); + } + + return { + icon, + message: + networkActivity + + "Sync: " + + icon + + " ↑ " + + sent + + pushRemaining.text + + " ↓ " + + arrived + + pullRemaining.text + + fastForwardSuffix, + parts, + }; +} diff --git a/src/modules/features/ReplicationStatusLabel.unit.spec.ts b/src/modules/features/ReplicationStatusLabel.unit.spec.ts new file mode 100644 index 00000000..6f322e8e --- /dev/null +++ b/src/modules/features/ReplicationStatusLabel.unit.spec.ts @@ -0,0 +1,241 @@ +import { reactiveSource } from "octagonal-wheels/dataobject/reactive"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ReplicationStatics } from "@vrtmrz/livesync-commonlib/replication"; + +import { + FAST_FORWARD_COUNT_LINGER_MS, + createFastForwardCheckedChangesDisplay, + projectReplicationStatusLabel, +} from "./ReplicationStatusLabel.ts"; + +function couchDBProgress( + phase: "idle" | "one-shot" | "catch-up" | "continuous", + checkedChanges = 0, + checking = false +): NonNullable { + return { + phase, + pull: { checkedChanges, completedBatches: 0, checking }, + push: { checkedChanges: 0, completedBatches: 0, checking: false }, + }; +} + +const noNetworkActivity = { remoteOperationCount: 0, trackedRequestCount: 0 }; + +function stats(overrides: Partial = {}): ReplicationStatics { + return { + sent: 0, + arrived: 0, + maxPullSeq: 0, + maxPushSeq: 0, + lastSyncPullSeq: 0, + lastSyncPushSeq: 0, + syncStatus: "CONNECTED", + ...overrides, + }; +} + +describe("projectReplicationStatusLabel", () => { + it("exposes the remote-request count as a numeric status item", () => { + const projection = projectReplicationStatusLabel(stats(), { remoteOperationCount: 1, trackedRequestCount: 12 }); + expect(projection.parts.find((part) => part.key === "network")).toMatchObject({ + text: "📲 🌐", + value: 12, + suffix: " ", + }); + }); + + it("preserves the connected label, directions, and remaining-sequence text", () => { + const projection = projectReplicationStatusLabel( + stats({ + sent: 4, + arrived: 2, + maxPullSeq: 20, + maxPushSeq: 30, + lastSyncPullSeq: 17, + lastSyncPushSeq: 22, + }), + { remoteOperationCount: 0, trackedRequestCount: 1 } + ); + + expect(projection.message).toBe("🌐1 Sync: ⚡ ↑ 4 (8) ↓ 2 (3)"); + expect(projection.parts.map((part) => part.key)).toEqual([ + "network", + "sync", + "sent", + "push-remaining", + "arrived", + "pull-remaining", + ]); + expect(projection.parts[2]).toMatchObject({ key: "sent", text: " ↑ ", value: 4 }); + expect(projection.parts[3]).toMatchObject({ key: "push-remaining", text: " (", value: 8, suffix: ")" }); + expect(projection.parts[4]).toMatchObject({ key: "arrived", text: " ↓ ", value: 2 }); + expect(projection.parts[5]).toMatchObject({ key: "pull-remaining", text: " (", value: 3, suffix: ")" }); + }); + + it("keeps the existing status icons when CouchDB is idle or not present", () => { + const statuses = [ + ["CLOSED", "⏹"], + ["COMPLETED", "⏹"], + ["NOT_CONNECTED", "⏹"], + ["STARTED", "🌀"], + ["PAUSED", "💤"], + ["CONNECTED", "⚡"], + ["JOURNAL_SEND", "📦↑"], + ["JOURNAL_RECEIVE", "📦↓"], + ["ERRORED", "⚠"], + ] as const; + + for (const [syncStatus, icon] of statuses) { + const idleWithCount = projectReplicationStatusLabel( + stats({ syncStatus, couchDB: couchDBProgress("idle", 19, true) }), + noNetworkActivity, + 19 + ); + expect(idleWithCount.icon).toBe(icon); + expect(idleWithCount.parts.some((part) => part.key === "ff")).toBe(true); + + const withoutCouchDB = projectReplicationStatusLabel(stats({ syncStatus }), noNetworkActivity, 19); + expect(withoutCouchDB.icon).toBe(icon); + expect(withoutCouchDB.parts.some((part) => part.key === "ff")).toBe(false); + } + + const noFastForward = projectReplicationStatusLabel(stats(), noNetworkActivity); + expect(noFastForward.parts.some((part) => part.key === "ff")).toBe(false); + expect(noFastForward.message).toBe("Sync: ⚡ ↑ 0 ↓ 0"); + }); + + it("shows fast-forward checking with zero arrivals, transferred data, and the final idle count", () => { + const checking = projectReplicationStatusLabel( + stats({ + syncStatus: "PAUSED", + arrived: 0, + couchDB: couchDBProgress("catch-up", 0, true), + }), + noNetworkActivity + ); + expect(checking.icon).toBe("⏩"); + expect(checking.parts[2]).toMatchObject({ key: "arrived", value: 0 }); + + const transferred = projectReplicationStatusLabel( + stats({ + arrived: 7, + couchDB: couchDBProgress("continuous", 1200, false), + }), + noNetworkActivity, + 1200 + ); + expect(transferred.icon).toBe("⚡"); + expect(transferred.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏩ 1200)"); + expect(transferred.parts[transferred.parts.length - 1]).toMatchObject({ + key: "ff", + text: " (⏩ ", + value: 1200, + suffix: ")", + }); + + const idle = projectReplicationStatusLabel( + stats({ + syncStatus: "CONNECTED", + arrived: 7, + couchDB: couchDBProgress("idle", 1200, false), + }), + noNetworkActivity, + 1200 + ); + expect(idle.icon).toBe("⚡"); + expect(idle.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏩ 1200)"); + }); + + it("keeps the error icon ahead of an active CouchDB checking indicator", () => { + const projection = projectReplicationStatusLabel( + stats({ + syncStatus: "ERRORED", + couchDB: { + ...couchDBProgress("one-shot", 2, true), + push: { checkedChanges: 0, completedBatches: 0, checking: true }, + }, + }), + noNetworkActivity + ); + + expect(projection.icon).toBe("⚠"); + }); +}); + +describe("createFastForwardCheckedChangesDisplay", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("uses the initial snapshot, expires unchanged counts, and only reappears for new progress", () => { + const source = reactiveSource( + stats({ + couchDB: couchDBProgress("catch-up", 1200, true), + }) + ); + const display = createFastForwardCheckedChangesDisplay(source); + + expect(display.value).toBe(1200); + vi.advanceTimersByTime(1000); + source.value = stats({ + sent: 3, + couchDB: couchDBProgress("idle", 1200, false), + }); + expect(display.value).toBe(1200); + vi.advanceTimersByTime(FAST_FORWARD_COUNT_LINGER_MS - 1001); + expect(display.value).toBe(1200); + + source.value = stats({ + sent: 4, + couchDB: couchDBProgress("idle", 1200, false), + }); + vi.advanceTimersByTime(1); + expect(display.value).toBeUndefined(); + + source.value = stats({ + syncStatus: "PAUSED", + couchDB: couchDBProgress("continuous", 1200, true), + }); + expect(display.value).toBeUndefined(); + + source.value = stats({ + couchDB: couchDBProgress("continuous", 1201, true), + }); + expect(display.value).toBe(1201); + + source.value = stats({ + couchDB: couchDBProgress("one-shot", 0, true), + }); + expect(display.value).toBeUndefined(); + + source.value = stats(); + expect(display.value).toBeUndefined(); + display.dispose(); + }); + + it("clears its timer and subscription when disposed", () => { + const source = reactiveSource(stats({ couchDB: couchDBProgress("catch-up", 5, true) })); + const display = createFastForwardCheckedChangesDisplay(source); + + display.dispose(); + vi.advanceTimersByTime(FAST_FORWARD_COUNT_LINGER_MS); + source.value = stats({ couchDB: couchDBProgress("continuous", 6, true) }); + + expect(display.value).toBe(5); + }); + + it("does not display progress for providers without CouchDB replication state", () => { + const source = reactiveSource(stats()); + const display = createFastForwardCheckedChangesDisplay(source); + + expect(display.value).toBeUndefined(); + source.value = stats({ couchDB: { phase: "continuous" } }); + expect(display.value).toBeUndefined(); + display.dispose(); + }); +}); diff --git a/src/modules/features/StatusBarDisplay.ts b/src/modules/features/StatusBarDisplay.ts index 66fd10ed..9c24a972 100644 --- a/src/modules/features/StatusBarDisplay.ts +++ b/src/modules/features/StatusBarDisplay.ts @@ -10,6 +10,15 @@ export type DisposableReactiveValue = ReactiveValue & { dispose(): void; }; +export type PaddedCounterLabel = DisposableReactiveValue & { + readonly count: ReactiveValue; +}; + +export type ChunkFetchCounterLabel = DisposableReactiveValue & { + readonly initialCount: ReactiveValue; + readonly retryingCount: ReactiveValue; +}; + function asDisposableReactiveValue(value: ReactiveSource, dispose: () => void): DisposableReactiveValue { return { get value() { @@ -92,9 +101,10 @@ export function createPaddedCounterLabel( source: ReactiveValue, mark: string, inactiveLingerMs = STATUS_COUNTER_INACTIVE_LINGER_MS -): DisposableReactiveValue { +): PaddedCounterLabel { const linger = Math.max(0, inactiveLingerMs); const formatted = reactiveSource(""); + const displayedCount = reactiveSource(undefined); let maximumLength = 1; let clearTimer: number | undefined; let disposed = false; @@ -114,22 +124,39 @@ export function createPaddedCounterLabel( cancelClear(); const count = source.value; formatted.value = format(count); + displayedCount.value = count; if (count !== 0) return; clearTimer = compatGlobal.setTimeout(() => { clearTimer = undefined; if (disposed) return; formatted.value = ""; + displayedCount.value = undefined; maximumLength = 1; }, linger); }; source.onChanged(update); - return asDisposableReactiveValue(formatted, () => { + const display = asDisposableReactiveValue(formatted, () => { if (disposed) return; disposed = true; cancelClear(); source.offChanged(update); }); + return { + get value() { + return display.value; + }, + onChanged(handler) { + display.onChanged(handler); + }, + offChanged(handler) { + display.offChanged(handler); + }, + count: displayedCount, + dispose() { + display.dispose(); + }, + }; } /** @@ -138,7 +165,7 @@ export function createPaddedCounterLabel( */ export function createChunkFetchCounterLabel( source: ReactiveValue<{ initial: number; retrying: number }> -): DisposableReactiveValue { +): ChunkFetchCounterLabel { const initialCount = reactiveSource(0); const retryingCount = reactiveSource(0); const initialLabel = createPaddedCounterLabel(initialCount, "🛄"); @@ -168,7 +195,7 @@ export function createChunkFetchCounterLabel( source.onChanged(updateCounts); updateCounts(); - return asDisposableReactiveValue(formatted, () => { + const display = asDisposableReactiveValue(formatted, () => { if (disposed) return; disposed = true; source.offChanged(updateCounts); @@ -177,4 +204,20 @@ export function createChunkFetchCounterLabel( initialLabel.dispose(); retryingLabel.dispose(); }); + return { + get value() { + return display.value; + }, + onChanged(handler) { + display.onChanged(handler); + }, + offChanged(handler) { + display.offChanged(handler); + }, + initialCount: initialLabel.count, + retryingCount: retryingLabel.count, + dispose() { + display.dispose(); + }, + }; } diff --git a/src/modules/features/StatusBarDisplay.unit.spec.ts b/src/modules/features/StatusBarDisplay.unit.spec.ts index 777bf9cd..5ce8f69e 100644 --- a/src/modules/features/StatusBarDisplay.unit.spec.ts +++ b/src/modules/features/StatusBarDisplay.unit.spec.ts @@ -111,6 +111,24 @@ describe("createPaddedCounterLabel", () => { display.dispose(); }); + it("exposes the numeric value while retaining zero through the existing linger", () => { + const source = reactiveSource(0); + const display = createPaddedCounterLabel(source, "📥"); + + expect(display.count.value).toBeUndefined(); + source.value = 9; + expect(display.count.value).toBe(9); + source.value = 0; + expect(display.count.value).toBe(0); + + vi.advanceTimersByTime(STATUS_COUNTER_INACTIVE_LINGER_MS - 1); + expect(display.count.value).toBe(0); + vi.advanceTimersByTime(1); + expect(display.count.value).toBeUndefined(); + + display.dispose(); + }); + it("cancels the pending clear when counter activity resumes", () => { const source = reactiveSource(0); const display = createPaddedCounterLabel(source, "📄"); diff --git a/src/modules/features/StatusBarRenderer.ts b/src/modules/features/StatusBarRenderer.ts new file mode 100644 index 00000000..990cc9f5 --- /dev/null +++ b/src/modules/features/StatusBarRenderer.ts @@ -0,0 +1,151 @@ +export type StatusBarPart = { + key: string; + text: string; + value?: number; + suffix?: string; + title?: string; +}; + +export type StatusBarRenderer = { + render(parts: readonly StatusBarPart[]): void; + dispose(): void; +}; + +type RenderedItem = { + element: HTMLSpanElement; + prefix: Text; + value: HTMLSpanElement; + valueText: Text; + suffix: Text; + widestValueLength: number; +}; + +export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer { + const items = new Map(); + const managedElements = new Set(); + let disposed = false; + + const setText = (node: Text, text: string) => { + if (node.data !== text) node.data = text; + }; + + const setHidden = (element: HTMLElement, hidden: boolean) => { + if (element.hidden !== hidden) element.hidden = hidden; + }; + + const setWidth = (element: HTMLElement, width: string) => { + if (element.style.getPropertyValue("--livesync-stat-width") !== width) { + element.setCssProps({ "--livesync-stat-width": width }); + } + }; + + const createItem = (key: string): RenderedItem => { + const document = parent.ownerDocument; + const element = parent.createSpan(); + element.className = "livesync-stat-item livesync-stat-" + key; + + const prefix = document.createTextNode(""); + const value = element.createSpan(); + value.className = "livesync-stat-value"; + const valueText = document.createTextNode(""); + value.appendChild(valueText); + value.hidden = true; + const suffix = document.createTextNode(""); + + element.appendChild(prefix); + element.appendChild(value); + element.appendChild(suffix); + + const item = { element, prefix, value, valueText, suffix, widestValueLength: 0 }; + items.set(key, item); + managedElements.add(element); + return item; + }; + + const updateItem = (item: RenderedItem, part: StatusBarPart) => { + setText(item.prefix, part.text); + const title = part.title ?? ""; + if (item.element.title !== title) item.element.title = title; + + if (part.value === undefined) { + setText(item.valueText, ""); + setText(item.suffix, ""); + setHidden(item.value, true); + setWidth(item.value, ""); + item.widestValueLength = 0; + return; + } + + const formattedValue = String(part.value); + setText(item.valueText, formattedValue); + setText(item.suffix, part.suffix ?? ""); + setHidden(item.value, false); + + if (formattedValue.length > item.widestValueLength) { + item.widestValueLength = formattedValue.length; + const width = item.widestValueLength + "ch"; + setWidth(item.value, width); + } + }; + + const reorderIfNeeded = (visibleItems: readonly RenderedItem[]) => { + const currentOrder = Array.from(parent.childNodes).filter((node): node is HTMLSpanElement => + managedElements.has(node as HTMLSpanElement) + ); + const visibleElements = new Set(visibleItems.map((item) => item.element)); + const desiredOrder = [ + ...visibleItems.map((item) => item.element), + ...currentOrder.filter((element) => !visibleElements.has(element)), + ]; + if ( + currentOrder.length === desiredOrder.length && + currentOrder.every((node, index) => node === desiredOrder[index]) + ) { + return; + } + + let reference: ChildNode | null = currentOrder[0] ?? null; + for (const element of desiredOrder) { + if (element !== reference) parent.insertBefore(element, reference); + const childNodes = Array.from(parent.childNodes); + const elementIndex = childNodes.indexOf(element); + reference = + childNodes.slice(elementIndex + 1).find((node) => managedElements.has(node as HTMLSpanElement)) ?? null; + } + }; + + return { + render(parts) { + if (disposed) return; + + const visibleItems: RenderedItem[] = []; + const visibleKeys = new Set(); + for (const part of parts) { + let item = items.get(part.key); + if (!item) item = createItem(part.key); + updateItem(item, part); + setHidden(item.element, false); + visibleItems.push(item); + visibleKeys.add(part.key); + } + + for (const [key, item] of items) { + if (visibleKeys.has(key)) continue; + setHidden(item.element, true); + setWidth(item.value, ""); + item.widestValueLength = 0; + } + + reorderIfNeeded(visibleItems); + }, + dispose() { + if (disposed) return; + disposed = true; + for (const item of items.values()) { + if (item.element.parentNode === parent) parent.removeChild(item.element); + } + items.clear(); + managedElements.clear(); + }, + }; +} diff --git a/src/modules/features/StatusBarRenderer.unit.spec.ts b/src/modules/features/StatusBarRenderer.unit.spec.ts new file mode 100644 index 00000000..1f1be039 --- /dev/null +++ b/src/modules/features/StatusBarRenderer.unit.spec.ts @@ -0,0 +1,300 @@ +import { describe, expect, it } from "vitest"; + +import { createStatusBarRenderer, type StatusBarPart } from "./StatusBarRenderer.ts"; + +type TestNode = TestElement | TestTextNode; + +class TestDocument { + mutations = 0; + + createElement(tagName: string): TestElement { + return new TestElement(this, tagName); + } + + createTextNode(data: string): TestTextNode { + return new TestTextNode(this, data); + } +} + +class TestTextNode { + readonly nodeType = 3; + parentNode: TestElement | undefined; + private currentData: string; + + constructor( + private readonly document: TestDocument, + data: string + ) { + this.currentData = data; + } + + get data(): string { + return this.currentData; + } + + set data(value: string) { + this.document.mutations++; + this.currentData = value; + } + + get textContent(): string { + return this.currentData; + } +} + +class TestStyle { + private properties = new Map(); + + constructor(private readonly document: TestDocument) {} + + getPropertyValue(name: string): string { + return this.properties.get(name) ?? ""; + } + + setProperty(name: string, value: string) { + this.document.mutations++; + this.properties.set(name, value); + } +} + +class TestElement { + readonly nodeType = 1; + readonly ownerDocument: Document; + readonly childNodes: TestNode[] = []; + parentNode: TestElement | undefined; + readonly style: TestStyle; + private currentClassName = ""; + private currentHidden = false; + private currentTitle = ""; + + constructor( + private readonly document: TestDocument, + readonly tagName: string + ) { + this.ownerDocument = document as unknown as Document; + this.style = new TestStyle(document); + } + + get className(): string { + return this.currentClassName; + } + + set className(value: string) { + this.document.mutations++; + this.currentClassName = value; + } + + get hidden(): boolean { + return this.currentHidden; + } + + set hidden(value: boolean) { + this.document.mutations++; + this.currentHidden = value; + } + + get title(): string { + return this.currentTitle; + } + + set title(value: string) { + this.document.mutations++; + this.currentTitle = value; + } + + get children(): TestElement[] { + return this.childNodes.filter((node): node is TestElement => node instanceof TestElement); + } + + get textContent(): string { + return this.childNodes.map((node) => node.textContent ?? "").join(""); + } + + createSpan(): TestElement { + return this.appendChild(this.document.createElement("span")); + } + + setCssProps(properties: Record) { + for (const [name, value] of Object.entries(properties)) this.style.setProperty(name, value); + } + + appendChild(node: T): T { + if (node.parentNode) { + node.parentNode.removeChild(node); + } + this.childNodes.push(node); + node.parentNode = this; + this.document.mutations++; + return node; + } + + insertBefore(node: T, reference: TestNode | null): T { + if (node === reference) return node; + if (node.parentNode) { + node.parentNode.removeChild(node); + } + const index = reference ? this.childNodes.indexOf(reference) : -1; + if (index < 0) { + this.childNodes.push(node); + } else { + this.childNodes.splice(index, 0, node); + } + node.parentNode = this; + this.document.mutations++; + return node; + } + + removeChild(node: T): T { + const index = this.childNodes.indexOf(node); + if (index < 0) throw new Error("Node is not a child"); + this.childNodes.splice(index, 1); + node.parentNode = undefined; + this.document.mutations++; + return node; + } +} + +function createFixture() { + const document = new TestDocument(); + const root = new TestElement(document, "div"); + return { document, root, parent: root as unknown as HTMLElement }; +} + +function findItem(root: TestElement, key: string): TestElement { + const item = root.children.find((child) => child.className.split(" ").includes("livesync-stat-" + key)); + if (!item) throw new Error("Missing item: " + key); + return item; +} + +function findValue(item: TestElement): TestElement { + const value = item.children.find((child) => child.className.split(" ").includes("livesync-stat-value")); + if (!value) throw new Error("Missing numeric value"); + return value; +} + +function visibleItems(root: TestElement): TestElement[] { + return root.children.filter((child) => !child.hidden && child.className.split(" ").includes("livesync-stat-item")); +} + +describe("createStatusBarRenderer", () => { + it("renders complete text safely and gives each item an independent class hook", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + + renderer.render([ + { key: "sync", text: "" }, + { key: "queue", text: " (queued: ", value: 2, suffix: ")" }, + ]); + + const sync = findItem(root, "sync"); + const queue = findItem(root, "queue"); + expect(sync.className.split(" ")).toEqual(["livesync-stat-item", "livesync-stat-sync"]); + expect(sync.textContent).toBe(""); + expect(queue.className.split(" ")).toContain("livesync-stat-queue"); + expect(queue.textContent).toBe(" (queued: 2)"); + expect(findValue(queue).className).toBe("livesync-stat-value"); + + renderer.render([{ key: "queue", text: " (queued: ", value: 3, suffix: ")" }]); + expect(sync.hidden).toBe(true); + expect(queue.hidden).toBe(false); + expect(queue.textContent).toBe(" (queued: 3)"); + renderer.render([{ key: "queue", text: "Queue idle" }]); + expect(queue.textContent).toBe("Queue idle"); + expect(findValue(queue).hidden).toBe(true); + renderer.dispose(); + }); + + it("does not mutate the DOM when the same parts are rendered again", () => { + const { document, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + const parts: StatusBarPart[] = [ + { key: "pending", text: " pending ", value: 7, suffix: " files", title: "Pending" }, + ]; + + renderer.render(parts); + const mutations = document.mutations; + renderer.render(parts); + + expect(document.mutations).toBe(mutations); + renderer.dispose(); + }); + + it("updates values while retaining the widest visible numeric width", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + + renderer.render([{ key: "count", text: "(", value: 9, suffix: ")" }]); + const item = findItem(root, "count"); + const value = findValue(item); + expect(item.textContent).toBe("(9)"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("1ch"); + + renderer.render([{ key: "count", text: "(", value: 123, suffix: ")" }]); + expect(value.textContent).toBe("123"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("3ch"); + + renderer.render([{ key: "count", text: "(", value: 4, suffix: ")" }]); + expect(value.textContent).toBe("4"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("3ch"); + renderer.dispose(); + }); + + it("resets numeric width while omitted and reuses the same item when it returns", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + + renderer.render([{ key: "count", text: " ", value: 1234 }]); + const item = findItem(root, "count"); + const value = findValue(item); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); + + renderer.render([]); + expect(item.hidden).toBe(true); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe(""); + + renderer.render([{ key: "count", text: " ", value: 5 }]); + expect(findItem(root, "count")).toBe(item); + expect(item.hidden).toBe(false); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("1ch"); + renderer.dispose(); + }); + + it("reorders existing nodes without replacing them", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + renderer.render([ + { key: "first", text: "first " }, + { key: "second", text: "second " }, + ]); + const first = findItem(root, "first"); + const second = findItem(root, "second"); + + renderer.render([ + { key: "second", text: "second " }, + { key: "first", text: "first " }, + ]); + + expect(visibleItems(root)).toEqual([second, first]); + expect(findItem(root, "first")).toBe(first); + expect(findItem(root, "second")).toBe(second); + renderer.dispose(); + }); + + it("removes its nodes on dispose and ignores later renders", () => { + const { root, parent } = createFixture(); + const externalNode = new TestElement(new TestDocument(), "strong"); + root.appendChild(externalNode); + const renderer = createStatusBarRenderer(parent); + renderer.render([{ key: "sync", text: "sync" }]); + const item = findItem(root, "sync"); + + renderer.dispose(); + const childCountAfterDispose = root.childNodes.length; + renderer.dispose(); + renderer.render([{ key: "later", text: "later" }]); + + expect(item.parentNode).toBeUndefined(); + expect(root.childNodes).toEqual([externalNode]); + expect(root.childNodes).toHaveLength(childCountAfterDispose); + }); +}); diff --git a/styles.css b/styles.css index 462eab07..4a4b0d61 100644 --- a/styles.css +++ b/styles.css @@ -839,3 +839,18 @@ body.is-mobile .livesync-compatibility-review-notice { outline-offset: 1px; border-radius: 2px; } + +.livesync-stat-item { + font-variant-numeric: tabular-nums; +} + +.livesync-stat-value { + display: inline-block; + width: var(--livesync-stat-width, auto); + text-align: right; + font-variant-numeric: tabular-nums; +} + +.livesync-stat-value[hidden] { + display: none; +} diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 5f04c8c8..a13018fb 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -83,10 +83,13 @@ npm run test:e2e:obsidian:focused -- stale-file-restart npm run test:e2e:obsidian:focused -- folder-batch npm run test:e2e:obsidian:focused -- security-seed-reconnect npm run test:e2e:obsidian:focused -- couchdb-http-cache +npm run test:e2e:obsidian:focused -- replication-status ``` The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. +`replication-status` uses an isolated Vault without a remote service. It injects synthetic replication statistics into the real plug-in to check the ⏩ icon, the comparison count and its three-second expiry, independently styled numeric items, stable rendered digit widths, and unchanged DOM nodes for repeated values. It switches between split panes, opens a popout editor, checks log updates there, and returns to the main window. Where Electron exposes native window control, it also hides the main window and verifies that popout logs still update before restoring it. This checks display behaviour; Commonlib owns the tests which establish that PouchDB produces the underlying statistics. + `database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, the default-on integrity-only remote option, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout at 800×600 and 375×667, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. The same scenario checks the five actions in Reporting Tools, the separate Checks and Recovery panel, Metadata progress with its document total and gauge, and collapsible Markdown. It also observes Usage revision progress with a count and discovery note during a gated read. Local lookup checks cover multiple document IDs for one path, selection of an exact ID, and a follow-up report using the original salt. Closing the follow-up dialogue during a pending Metadata read must abort the report after that read settles, without querying Chunk availability. The compact Markdown must retain every JSON observation reference and pseudonym, and its copy and save payloads must match the full output. Results and export controls are checked at 800×600 on desktop and 375×667 in mobile emulation, including after copy and save feedback appears. diff --git a/test/e2e-obsidian/scripts/replication-status.ts b/test/e2e-obsidian/scripts/replication-status.ts new file mode 100644 index 00000000..1ef380a1 --- /dev/null +++ b/test/e2e-obsidian/scripts/replication-status.ts @@ -0,0 +1,770 @@ +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; +import type { Page } from "playwright"; + +const fixturePath = "replication-status-e2e.md"; +const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_UI_TIMEOUT_MS ?? 10000); +const fastForwardLingerMs = 3000; +const numericKeys = ["network", "sent", "push-remaining", "arrived", "pull-remaining", "ff"] as const; + +type TestReplicationStatics = { + sent: number; + arrived: number; + maxPullSeq: number; + maxPushSeq: number; + lastSyncPullSeq: number; + lastSyncPushSeq: number; + syncStatus: "CONNECTED"; + couchDB: { + phase: "continuous"; + pull: { checkedChanges: number; completedBatches: number; checking: boolean }; + push: { checkedChanges: number; completedBatches: number; checking: boolean }; + }; +}; + +type WritableSource = { value: T }; +type TestLeaf = { + containerEl?: HTMLElement; + view?: { containerEl?: HTMLElement }; + openFile?: (file: unknown) => Promise; + detach?: () => void; +}; +type TestWorkspace = { + activeLeaf?: TestLeaf | null; + getLeaf?: (newLeaf?: boolean | "split" | "tab" | "window", direction?: "horizontal" | "vertical") => TestLeaf; + openPopoutLeaf?: () => TestLeaf; + setActiveLeaf?: (leaf: TestLeaf, params?: { focus?: boolean }) => void; + iterateAllLeaves?: (callback: (leaf: TestLeaf) => void) => void; + getActiveFile?: () => { path?: string } | null; +}; +type TestLogModule = { + statusLog?: WritableSource; + statusDiv?: HTMLElement; + statusLine?: HTMLElement; + statusLineRenderer?: { render?: (parts: readonly unknown[]) => void }; + statusBarLabels?: WritableSource<{ parts: readonly unknown[] }>; + __addLog?: (message: unknown, level?: number, key?: string) => void; +}; +type TestCore = { + modules?: unknown[]; + services?: { + setting?: { currentSettings?: () => Record }; + replicator?: { + replicationStatics?: WritableSource; + }; + API?: { + requestCount?: WritableSource; + responseCount?: WritableSource; + }; + }; +}; +type TestApp = { + plugins?: { + plugins?: { "obsidian-livesync"?: { core?: TestCore } }; + }; + vault?: { + getAbstractFileByPath?: (path: string) => unknown | null; + create?: (path: string, content: string) => Promise; + }; + workspace?: TestWorkspace; +}; +type TestWindow = typeof globalThis & { + app?: TestApp; + require?: (moduleName: string) => unknown; + __livesyncStatusFixtureFile?: unknown; + __livesyncStatusFirstLeaf?: TestLeaf; + __livesyncStatusSplitLeaf?: TestLeaf; + __livesyncStatusPopoutLeaf?: TestLeaf; + __livesyncStatusNode?: HTMLElement; +}; +type MainWindowControl = { + hidden: boolean; + method?: string; + reason?: string; + restore(): Promise; +}; + +function failUnless(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function replicationStats(checkedChanges: number, sent: number, arrived: number): TestReplicationStatics { + return { + sent, + arrived, + maxPullSeq: sent + 1, + maxPushSeq: sent + 1, + lastSyncPullSeq: 1, + lastSyncPushSeq: 1, + syncStatus: "CONNECTED", + couchDB: { + phase: "continuous", + pull: { checkedChanges, completedBatches: 0, checking: true }, + push: { checkedChanges: 0, completedBatches: 0, checking: false }, + }, + }; +} + +async function installFixtureFile(page: Page): Promise { + const result = await page.evaluate(async (path) => { + const app = (globalThis as TestWindow).app; + const workspace = app?.workspace; + const vault = app?.vault; + if (!workspace?.getLeaf || !workspace.setActiveLeaf || !vault?.create || !vault.getAbstractFileByPath) { + return { ready: false, reason: "Obsidian workspace APIs are unavailable." }; + } + let file = vault.getAbstractFileByPath(path); + if (!file) file = await vault.create(path, "Replication status renderer fixture.\n"); + const leaf = workspace.getLeaf(false); + if (!leaf.openFile) return { ready: false, reason: "The active workspace leaf cannot open a file." }; + await leaf.openFile(file); + workspace.setActiveLeaf(leaf, { focus: true }); + (globalThis as TestWindow).__livesyncStatusFixtureFile = file; + (globalThis as TestWindow).__livesyncStatusFirstLeaf = leaf; + return { ready: true }; + }, fixturePath); + if (!result.ready) + throw new Error("reason" in result ? result.reason : "Could not initialise the fixture editor leaf."); +} + +async function injectReplicationStats( + page: Page, + stats: TestReplicationStatics, + requestCount: number, + responseCount: number +): Promise { + const result = await page.evaluate( + ({ stats, requestCount, responseCount }) => { + const core = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const replicator = core?.services?.replicator; + const api = core?.services?.API; + if (!replicator?.replicationStatics || !api?.requestCount || !api.responseCount) { + return { ready: false, reason: "LiveSync replication status sources are unavailable." }; + } + replicator.replicationStatics.value = stats; + api.requestCount.value = requestCount; + api.responseCount.value = responseCount; + return { ready: true }; + }, + { stats, requestCount, responseCount } + ); + if (!result.ready) + throw new Error("reason" in result ? result.reason : "Could not inject replication status fixtures."); +} + +async function appendLiveSyncLog(page: Page, message: string): Promise { + const result = await page.evaluate((message) => { + const core = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const modules = core?.modules ?? []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + const candidate = entry as TestLogModule; + return typeof candidate.__addLog === "function" && candidate.statusLog !== undefined; + }); + if (!moduleLog?.__addLog) return { ready: false, reason: "LiveSync logging is unavailable." }; + moduleLog.__addLog(message); + return { ready: true }; + }, message); + if (!result.ready) throw new Error("reason" in result ? result.reason : "Could not append a LiveSync log entry."); +} + +async function waitForPartText(page: Page, key: string, expected: string): Promise { + const selector = `.livesync-status-statusline .livesync-stat-${key} .livesync-stat-value`; + await page.waitForFunction( + ({ selector, expected }) => document.querySelector(selector)?.textContent?.trim() === expected, + { selector, expected }, + { timeout: uiTimeoutMs } + ); +} + +async function waitForLogMessage(page: Page, expected: string, timeout = uiTimeoutMs): Promise { + await page.waitForFunction( + (message) => document.querySelector(".livesync-status-logmessage")?.textContent?.includes(message) === true, + expected, + { timeout } + ); +} + +async function assertSingleOverlay(contextPages: Page[], expectedCount = 1): Promise { + const counts = await Promise.all( + contextPages.map( + async (page) => + await page + .locator(".livesync-status") + .count() + .catch(() => 0) + ) + ); + const actualCount = counts.reduce((total, count) => total + count, 0); + failUnless( + actualCount === expectedCount, + `Expected ${expectedCount} LiveSync status overlay across Obsidian windows, found ${actualCount} (${counts.join(", ")}).` + ); +} + +async function waitForOverlayOnActiveLeaf(page: Page): Promise { + await page.waitForFunction( + () => { + const host = globalThis as TestWindow; + const status = host.__livesyncStatusNode; + const activeLeaf = host.app?.workspace?.activeLeaf; + return ( + status !== undefined && + document.querySelectorAll(".livesync-status").length === 1 && + document.querySelector(".livesync-status") === status && + activeLeaf?.view?.containerEl?.contains(status) === true + ); + }, + undefined, + { timeout: uiTimeoutMs } + ); +} + +async function getStatusDiagnostics(page: Page): Promise> { + return await page.evaluate(() => { + const app = (globalThis as TestWindow).app; + const core = app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const moduleLog = core?.modules?.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + const candidate = entry as TestLogModule; + return candidate.statusDiv !== undefined || candidate.statusLineRenderer !== undefined; + }); + const currentSettings = core?.services?.setting?.currentSettings?.(); + const status = document.querySelector(".livesync-status"); + const statusParent = status?.parentElement; + const activeLeaf = app?.workspace?.activeLeaf; + const recentLeaf = app?.workspace?.getLeaf?.(false); + const statusStyle = status ? getComputedStyle(status) : undefined; + const statusParentStyle = statusParent ? getComputedStyle(statusParent) : undefined; + const activeLeafContainer = activeLeaf?.view?.containerEl; + const recentLeafContainer = recentLeaf?.view?.containerEl; + const activeLeafStyle = activeLeafContainer ? getComputedStyle(activeLeafContainer) : undefined; + const recentLeafStyle = recentLeafContainer ? getComputedStyle(recentLeafContainer) : undefined; + return { + settings: currentSettings + ? { + isConfigured: currentSettings.isConfigured, + showStatusOnEditor: currentSettings.showStatusOnEditor, + showStatusOnStatusbar: currentSettings.showStatusOnStatusbar, + } + : undefined, + statusCount: document.querySelectorAll(".livesync-status").length, + status: status + ? { + tag: status.tagName, + className: status.className, + display: statusStyle?.display, + visibility: statusStyle?.visibility, + } + : undefined, + statusParent: statusParent + ? { + tag: statusParent.tagName, + className: statusParent.className, + display: statusParentStyle?.display, + visibility: statusParentStyle?.visibility, + } + : undefined, + moduleHasStatusDiv: moduleLog?.statusDiv !== undefined, + moduleStatusLine: moduleLog?.statusLine + ? { + tag: moduleLog.statusLine.tagName, + className: moduleLog.statusLine.className, + } + : undefined, + activeFile: app?.workspace?.getActiveFile?.()?.path, + activeLeafContainer: activeLeafContainer + ? { + tag: activeLeafContainer.tagName, + className: activeLeafContainer.className, + display: activeLeafStyle?.display, + visibility: activeLeafStyle?.visibility, + } + : undefined, + recentLeafContainer: recentLeafContainer + ? { + tag: recentLeafContainer.tagName, + className: recentLeafContainer.className, + display: recentLeafStyle?.display, + visibility: recentLeafStyle?.visibility, + } + : undefined, + statusInActiveLeaf: activeLeafContainer?.contains(status) ?? false, + statusInRecentLeaf: recentLeafContainer?.contains(status) ?? false, + }; + }); +} + +async function waitForPopoutPage(page: Page, marker: string): Promise { + const deadline = Date.now() + uiTimeoutMs; + const selector = `[data-e2e-status-popout="${marker}"]`; + while (Date.now() < deadline) { + for (const candidate of page.context().pages()) { + if ( + (await candidate + .locator(selector) + .count() + .catch(() => 0)) > 0 + ) + return candidate; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`Obsidian did not expose the popout renderer containing ${selector}.`); +} + +async function createPopoutLeaf(page: Page, marker: string): Promise { + const result = await page.evaluate(async (marker) => { + const host = globalThis as TestWindow; + const app = host.app; + const workspace = app?.workspace; + const file = host.__livesyncStatusFixtureFile; + if (!workspace?.openPopoutLeaf || !workspace.setActiveLeaf || file === undefined) { + return { ready: false, reason: "Obsidian popout workspace APIs are unavailable." }; + } + const leaf = workspace.openPopoutLeaf(); + if (!leaf.openFile || !leaf.containerEl) return { ready: false, reason: "The popout leaf is incomplete." }; + leaf.containerEl.dataset.e2eStatusPopout = marker; + await leaf.openFile(file); + workspace.setActiveLeaf(leaf, { focus: true }); + host.__livesyncStatusPopoutLeaf = leaf; + return { ready: true }; + }, marker); + if (!result.ready) throw new Error("reason" in result ? result.reason : "Could not open an Obsidian popout leaf."); +} + +async function switchToLeaf(page: Page, leafName: "first" | "split"): Promise { + const result = await page.evaluate((leafName) => { + const host = globalThis as TestWindow; + const workspace = host.app?.workspace; + const leaf = leafName === "first" ? host.__livesyncStatusFirstLeaf : host.__livesyncStatusSplitLeaf; + if (!workspace?.setActiveLeaf || !leaf) return { ready: false }; + workspace.setActiveLeaf(leaf, { focus: true }); + return { ready: true }; + }, leafName); + failUnless(result.ready, `Could not activate the ${leafName} editor leaf.`); + await waitForOverlayOnActiveLeaf(page); +} + +async function createSplitLeaf(page: Page): Promise { + const result = await page.evaluate(async () => { + const host = globalThis as TestWindow; + const workspace = host.app?.workspace; + const file = host.__livesyncStatusFixtureFile; + if (!workspace?.getLeaf || !workspace.setActiveLeaf || file === undefined) { + return { ready: false, reason: "Obsidian split workspace APIs are unavailable." }; + } + const leaf = workspace.getLeaf("split", "vertical"); + if (!leaf.openFile) return { ready: false, reason: "The split leaf cannot open a file." }; + await leaf.openFile(file); + workspace.setActiveLeaf(leaf, { focus: true }); + host.__livesyncStatusSplitLeaf = leaf; + return { ready: true }; + }); + if (!result.ready) throw new Error("reason" in result ? result.reason : "Could not create a split editor leaf."); +} + +async function observeRepeatedRendering(page: Page): Promise<{ mutationCount: number; elapsedMs: number }> { + return await page.evaluate(async () => { + const core = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const modules = core?.modules ?? []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + const candidate = entry as TestLogModule; + return ( + typeof candidate.statusLineRenderer?.render === "function" && candidate.statusBarLabels !== undefined + ); + }); + const statusLine = document.querySelector(".livesync-status-statusline"); + if (!moduleLog?.statusLineRenderer?.render || !moduleLog.statusBarLabels || !statusLine) { + throw new Error("The LiveSync status renderer is unavailable for the idempotence observation."); + } + let mutationCount = 0; + const observer = new MutationObserver((records) => { + mutationCount += records.length; + }); + observer.observe(statusLine, { attributes: true, characterData: true, childList: true, subtree: true }); + const parts = moduleLog.statusBarLabels.value.parts; + const startedAt = performance.now(); + for (let iteration = 0; iteration < 1000; iteration++) { + moduleLog.statusLineRenderer.render(parts); + } + const elapsedMs = performance.now() - startedAt; + await new Promise((resolve) => setTimeout(resolve, 0)); + observer.disconnect(); + return { mutationCount, elapsedMs }; + }); +} + +async function measureNumericWidths(page: Page): Promise> { + const measurements = await page.evaluate((keys) => { + const result: Record = {}; + const statusLine = document.querySelector(".livesync-status-statusline"); + if (!statusLine) throw new Error("The LiveSync editor status line is unavailable."); + for (const key of keys) { + const value = statusLine.querySelector(`.livesync-stat-${key} .livesync-stat-value`); + if (!value || value.hidden) throw new Error(`The numeric status item ${key} is not visible.`); + if (getComputedStyle(value).fontVariantNumeric !== "tabular-nums") { + throw new Error(`The numeric status item ${key} does not use tabular figures.`); + } + result[key] = value.getBoundingClientRect().width; + } + return result; + }, numericKeys); + return measurements as Record<(typeof numericKeys)[number], number>; +} + +async function assertNumericWidths( + page: Page, + expectedValue: string +): Promise> { + for (const key of numericKeys) await waitForPartText(page, key, expectedValue); + const widths = await page.evaluate((keys) => { + const result: Record = {}; + const statusLine = document.querySelector(".livesync-status-statusline"); + if (!statusLine) throw new Error("The LiveSync editor status line is unavailable."); + for (const key of keys) { + const value = statusLine.querySelector(`.livesync-stat-${key} .livesync-stat-value`); + if (!value || value.hidden) throw new Error(`The numeric status item ${key} is not visible.`); + result[key] = { + width: value.style.getPropertyValue("--livesync-stat-width"), + value: value.textContent?.trim() ?? "", + }; + } + return result; + }, numericKeys); + for (const key of numericKeys) { + failUnless(widths[key]?.value === expectedValue, `Status item ${key} did not render ${expectedValue}.`); + failUnless(widths[key]?.width === "4ch", `Status item ${key} did not retain a 4ch width.`); + } + return await measureNumericWidths(page); +} + +async function measureStatusBarNetworkWidth( + page: Page, + expectedValue: string +): Promise<{ cssWidth: string; renderedWidth: number }> { + const selector = ".syncstatusbar .livesync-stat-network .livesync-stat-value"; + await page.waitForFunction( + ({ selector, expectedValue }) => { + const value = document.querySelector(selector); + return value !== null && !value.hidden && value.textContent?.trim() === expectedValue; + }, + { selector, expectedValue }, + { timeout: uiTimeoutMs } + ); + return await page.locator(selector).evaluate((element) => { + const value = element as HTMLElement; + return { + cssWidth: value.style.getPropertyValue("--livesync-stat-width"), + renderedWidth: value.getBoundingClientRect().width, + }; + }); +} + +async function hideMainWindow(page: Page): Promise { + const nativeWindow = await page.evaluate((action) => { + type ElectronWindow = { + isVisible?: () => boolean; + hide?: () => void; + show?: () => void; + focus?: () => void; + }; + type RemoteModule = { getCurrentWindow?: () => ElectronWindow }; + const host = globalThis as TestWindow; + let electronModule: unknown; + try { + electronModule = host.require?.("electron"); + } catch { + electronModule = undefined; + } + const electron = electronModule as { remote?: RemoteModule } | undefined; + let remoteModule: unknown = electron?.remote; + if (!remoteModule) { + try { + remoteModule = host.require?.("@electron/remote"); + } catch { + remoteModule = undefined; + } + } + const remote = remoteModule as RemoteModule | undefined; + const currentWindow = remote?.getCurrentWindow?.(); + if (!currentWindow?.isVisible || !currentWindow.hide || !currentWindow.show) { + return { + available: false, + visible: false, + changed: false, + reason: "Electron remote window controls are unavailable.", + }; + } + if (action === "hide") { + if (!currentWindow.isVisible()) { + return { + available: true, + visible: false, + changed: false, + reason: "The main Electron window was already hidden.", + }; + } + currentWindow.hide(); + const visible = currentWindow.isVisible(); + return { available: true, visible, changed: !visible }; + } + currentWindow.show(); + currentWindow.focus?.(); + return { available: true, visible: currentWindow.isVisible(), changed: false }; + }, "hide"); + if (nativeWindow.available && nativeWindow.changed && !nativeWindow.visible) { + return { + hidden: true, + method: "Electron remote", + async restore() { + const restored = await page.evaluate(() => { + type ElectronWindow = { + isVisible?: () => boolean; + hide?: () => void; + show?: () => void; + focus?: () => void; + }; + type RemoteModule = { getCurrentWindow?: () => ElectronWindow }; + const host = globalThis as TestWindow; + let electronModule: unknown; + try { + electronModule = host.require?.("electron"); + } catch { + electronModule = undefined; + } + const electron = electronModule as { remote?: RemoteModule } | undefined; + let remoteModule: unknown = electron?.remote; + if (!remoteModule) { + try { + remoteModule = host.require?.("@electron/remote"); + } catch { + remoteModule = undefined; + } + } + const remote = remoteModule as RemoteModule | undefined; + const currentWindow = remote?.getCurrentWindow?.(); + if (!currentWindow?.show) return false; + currentWindow.show(); + currentWindow.focus?.(); + return currentWindow.isVisible?.() === true; + }); + if (!restored) throw new Error("The hidden main Electron window could not be restored."); + }, + }; + } + return { + hidden: false, + reason: nativeWindow.reason ?? "Electron remote did not hide the main window.", + restore: async () => {}, + }; +} + +async function runStatusScenario(session: ObsidianLiveSyncSession): Promise { + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + let popoutMarker: string | undefined; + let popoutPage: Page | undefined; + let mainWindow: MainWindowControl | undefined; + try { + await installFixtureFile(page); + try { + await page.locator(".livesync-status").waitFor({ state: "attached", timeout: uiTimeoutMs }); + } catch (error) { + const diagnostics = await getStatusDiagnostics(page); + throw new Error( + `${error instanceof Error ? error.message : String(error)}\nLiveSync status setup: ${JSON.stringify(diagnostics)}` + ); + } + const setup = await getStatusDiagnostics(page); + const settings = setup.settings as { showStatusOnEditor?: unknown } | undefined; + failUnless( + setup.moduleHasStatusDiv === true && settings?.showStatusOnEditor === true, + `LiveSync editor status is not enabled in the isolated vault: ${JSON.stringify(setup)}` + ); + await injectReplicationStats(page, replicationStats(0, 0, 0), 0, 0); + try { + await page.locator(".livesync-status").waitFor({ state: "visible", timeout: uiTimeoutMs }); + } catch (error) { + const diagnostics = await getStatusDiagnostics(page); + throw new Error( + `${error instanceof Error ? error.message : String(error)}\nLiveSync status setup: ${JSON.stringify(diagnostics)}` + ); + } + await page.evaluate(() => { + const host = globalThis as TestWindow; + const status = document.querySelector(".livesync-status"); + if (!status) throw new Error("LiveSync editor status overlay was not created."); + host.__livesyncStatusNode = status; + }); + await waitForOverlayOnActiveLeaf(page); + + const initialStats = replicationStats(1000, 1000, 0); + const startedAt = Date.now(); + await injectReplicationStats(page, initialStats, 0, 0); + await waitForPartText(page, "arrived", "0"); + await waitForPartText(page, "ff", "1000"); + const syncIcon = await page.locator(".livesync-status-statusline .livesync-stat-sync").textContent(); + failUnless(syncIcon?.includes("⏩"), "The fast-forward checking icon was not independent of arrivals."); + + for (let update = 1; update <= 3; update++) { + await page.waitForTimeout(700); + await injectReplicationStats(page, replicationStats(1000, 1000 + update, 0), 0, 0); + } + const remainingLinger = Math.max(0, fastForwardLingerMs + 350 - (Date.now() - startedAt)); + if (remainingLinger > 0) await page.waitForTimeout(remainingLinger); + await page.waitForFunction( + () => + document.querySelector(".livesync-status-statusline .livesync-stat-ff")?.hidden === + true, + undefined, + { timeout: uiTimeoutMs } + ); + const iconAfterExpiry = await page.locator(".livesync-status-statusline .livesync-stat-sync").textContent(); + failUnless( + iconAfterExpiry?.includes("⏩"), + "The fast-forward checking icon disappeared when its count expired." + ); + + await injectReplicationStats(page, replicationStats(1001, 1001, 1001), 1001, 0); + const wideWidths = await assertNumericWidths(page, "1001"); + const wideStatusBarWidth = await measureStatusBarNetworkWidth(page, "1001"); + failUnless(wideStatusBarWidth.cssWidth === "4ch", "The status-bar network count did not reserve 4ch."); + await injectReplicationStats(page, replicationStats(7, 7, 7), 7, 0); + const narrowWidths = await assertNumericWidths(page, "7"); + const narrowStatusBarWidth = await measureStatusBarNetworkWidth(page, "7"); + for (const key of numericKeys) { + failUnless( + Math.abs(wideWidths[key] - narrowWidths[key]) < 0.01, + `Status item ${key} changed rendered width from ${wideWidths[key]}px to ${narrowWidths[key]}px.` + ); + } + failUnless( + narrowStatusBarWidth.cssWidth === "4ch" && + Math.abs(wideStatusBarWidth.renderedWidth - narrowStatusBarWidth.renderedWidth) < 0.01, + `The status-bar network count changed width from ${wideStatusBarWidth.renderedWidth}px to ${narrowStatusBarWidth.renderedWidth}px.` + ); + + const repeatedRender = await observeRepeatedRendering(page); + failUnless( + repeatedRender.mutationCount === 0, + `Repeated identical status rendering caused ${repeatedRender.mutationCount} DOM mutations.` + ); + console.log( + `Identical status render: 1000 iterations, 0 DOM mutations, ${repeatedRender.elapsedMs.toFixed(2)} ms.` + ); + + await createSplitLeaf(page); + await assertSingleOverlay(page.context().pages()); + await waitForOverlayOnActiveLeaf(page); + await switchToLeaf(page, "first"); + await assertSingleOverlay(page.context().pages()); + await switchToLeaf(page, "split"); + await assertSingleOverlay(page.context().pages()); + await switchToLeaf(page, "first"); + await assertSingleOverlay(page.context().pages()); + + popoutMarker = `livesync-status-${process.pid}-${Date.now()}`; + await createPopoutLeaf(page, popoutMarker); + popoutPage = await waitForPopoutPage(page, popoutMarker); + await popoutPage.locator(".livesync-status").waitFor({ state: "visible", timeout: uiTimeoutMs }); + await assertSingleOverlay(page.context().pages()); + await popoutPage.waitForFunction( + (marker) => { + const leaf = document.querySelector(`[data-e2e-status-popout="${marker}"]`); + const status = document.querySelector(".livesync-status"); + return leaf !== null && status !== null && leaf.contains(status); + }, + popoutMarker, + { timeout: uiTimeoutMs } + ); + + const popoutLog = `E2E popout log ${Date.now()}`; + await appendLiveSyncLog(page, popoutLog); + await waitForLogMessage(popoutPage, popoutLog); + + mainWindow = await hideMainWindow(page); + if (!mainWindow.hidden) { + console.warn( + `Skipped hidden-main-window assertion: ${mainWindow.reason ?? "minimisation unavailable"}` + ); + } else { + await popoutPage.bringToFront(); + const hiddenWindowLog = `E2E hidden-main log ${Date.now()}`; + await appendLiveSyncLog(page, hiddenWindowLog); + await waitForLogMessage(popoutPage, hiddenWindowLog, Math.min(uiTimeoutMs, 2500)); + console.log(`Popout log updated while the main window was hidden through ${mainWindow.method}.`); + } + await mainWindow.restore(); + mainWindow = undefined; + await page.bringToFront(); + await switchToLeaf(page, "first"); + await assertSingleOverlay(page.context().pages()); + await waitForOverlayOnActiveLeaf(page); + failUnless( + await page.evaluate( + () => (globalThis as TestWindow).__livesyncStatusNode === document.querySelector(".livesync-status") + ), + "The LiveSync status overlay node was replaced while moving between editor windows." + ); + const returnedLog = `E2E main-window log ${Date.now()}`; + await appendLiveSyncLog(page, returnedLog); + await waitForLogMessage(page, returnedLog); + console.log("Replication status, split-pane ownership, and popout status/log checks passed."); + } finally { + if (mainWindow) await mainWindow.restore(); + await page.bringToFront().catch(() => {}); + if (popoutMarker !== undefined) { + await page + .evaluate((marker) => { + const workspace = (globalThis as TestWindow).app?.workspace; + workspace?.iterateAllLeaves?.((leaf) => { + if (leaf.containerEl?.dataset.e2eStatusPopout === marker) leaf.detach?.(); + }); + }, popoutMarker) + .catch(() => {}); + } + if (popoutPage) await popoutPage.close().catch(() => {}); + } + }); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + const vault = await createTemporaryVault("obsidian-livesync-replication-status-"); + let session: ObsidianLiveSyncSession | undefined; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli.binary, + vault, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + pluginData: { + liveSync: false, + syncOnStart: false, + syncOnSave: false, + isConfigured: false, + remoteType: "", + showStatusOnEditor: true, + showOnlyIconsOnEditor: false, + showStatusOnStatusbar: true, + lessInformationInLog: false, + writeLogToTheFile: false, + }, + }); + failUnless(session.readiness.pluginId === "obsidian-livesync", "The LiveSync plug-in did not become ready."); + await runStatusScenario(session); + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 31a8f339..0d1bc3e2 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -21,6 +21,7 @@ const focusedScenarios = new Set([ "configuration-encryption-sync", "review-harness", "p2p-pane", + "replication-status", "vault-reflection", "couchdb-upload", "couchdb-http-cache", diff --git a/updates.md b/updates.md index 6a92c03f..e3c00366 100644 --- a/updates.md +++ b/updates.md @@ -14,6 +14,8 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Synchronisation +- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏩ status icon and a comparison count beside received items. An unchanged count disappears after three seconds. +- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. - We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) ## 1.0.40 From e7f7e62cfa87988caadf004f7790bbf16a2589ae Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 09:03:51 +0000 Subject: [PATCH 22/28] Refine status item placement and inactive styling --- README.md | 6 +- .../features/ReplicationStatusLabel.ts | 14 +- .../ReplicationStatusLabel.unit.spec.ts | 7 +- src/modules/features/StatusBarRenderer.ts | 33 +-- .../features/StatusBarRenderer.unit.spec.ts | 78 +++++- styles.css | 19 ++ test/e2e-obsidian/README.md | 2 +- .../scripts/replication-status.ts | 250 +++++++++++++++--- updates.md | 4 +- 9 files changed, 345 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 38c78d2c..8bdbacef 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Synchronisation status is shown in the status bar with the following icons. To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files. -When CouchDB change history is being compared, `↓ 0 (⏩ 1200)` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It disappears after three seconds without a change and reappears when the count changes. The ⏩ status icon follows comparison activity independently of that timer. +When CouchDB change history is being compared, `(⏩ 1200) ↑ 0 ↓ 0` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It appears before the upload count, disappears after three seconds without a change, and reappears when the count changes. The ⏩ status icon follows comparison activity independently of that timer. ### Customising status items with CSS snippets @@ -120,7 +120,9 @@ The status bar and editor overlay use individual spans with the common `livesync The item classes include `livesync-stat-network`, `livesync-stat-sync`, `livesync-stat-sent`, `livesync-stat-push-remaining`, `livesync-stat-arrived`, `livesync-stat-pull-remaining`, `livesync-stat-ff`, and `livesync-stat-p2p`. File-processing items use `livesync-stat-file-processing`, `livesync-stat-file-pending`, and `livesync-stat-file-batched`; queue items use `livesync-stat-queue-` followed by `replication`, `database`, `storage`, `chunk-initial`, `chunk-retrying`, `plugin-scan`, `hidden-files`, or `conflicts`. The skipped-file indicator uses `livesync-stat-file-skip`. -All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count until it is hidden, reducing movement as counts decrease. +All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count during an active period, reducing movement as counts decrease. The editor overlay uses wider gaps between items, while remaining counts stay beside their transfer totals. + +Inactive items receive `livesync-stat-inactive` instead of the HTML `hidden` attribute. The default CSS hides them immediately. They retain their last text, numeric width, and position amongst the other items so that CSS snippets can customise how they disappear. The numeric width starts afresh when an item becomes active again. ## Tips and Troubleshooting - If you want a faster and simpler initial replication when setting up subsequent devices, see the [Fast Setup Guide](docs/tips/fast-setup.md). diff --git a/src/modules/features/ReplicationStatusLabel.ts b/src/modules/features/ReplicationStatusLabel.ts index b36d4c95..8358127c 100644 --- a/src/modules/features/ReplicationStatusLabel.ts +++ b/src/modules/features/ReplicationStatusLabel.ts @@ -161,7 +161,7 @@ export function projectReplicationStatusLabel( fastForwardCount > 0 ? fastForwardCount : undefined; - const fastForwardSuffix = visibleFastForwardCount === undefined ? "" : " (⏩ " + visibleFastForwardCount + ")"; + const fastForwardLabel = visibleFastForwardCount === undefined ? "" : " (⏩ " + visibleFastForwardCount + ")"; const parts: StatusBarPart[] = []; if (networkStatus.trackedRequestCount > 0) { @@ -177,10 +177,6 @@ export function projectReplicationStatusLabel( parts.push({ key: "network", text: networkActivity }); } parts.push({ key: "sync", text: "Sync: " + icon }); - parts.push({ key: "sent", text: " ↑ ", value: sent }); - if (pushRemaining.part !== undefined) parts.push(pushRemaining.part); - parts.push({ key: "arrived", text: " ↓ ", value: arrived }); - if (pullRemaining.part !== undefined) parts.push(pullRemaining.part); if (visibleFastForwardCount !== undefined) { parts.push({ key: "ff", @@ -190,6 +186,10 @@ export function projectReplicationStatusLabel( title: "Compared CouchDB feed entries", }); } + parts.push({ key: "sent", text: " ↑ ", value: sent }); + if (pushRemaining.part !== undefined) parts.push(pushRemaining.part); + parts.push({ key: "arrived", text: " ↓ ", value: arrived }); + if (pullRemaining.part !== undefined) parts.push(pullRemaining.part); return { icon, @@ -197,13 +197,13 @@ export function projectReplicationStatusLabel( networkActivity + "Sync: " + icon + + fastForwardLabel + " ↑ " + sent + pushRemaining.text + " ↓ " + arrived + - pullRemaining.text + - fastForwardSuffix, + pullRemaining.text, parts, }; } diff --git a/src/modules/features/ReplicationStatusLabel.unit.spec.ts b/src/modules/features/ReplicationStatusLabel.unit.spec.ts index 6f322e8e..054d597e 100644 --- a/src/modules/features/ReplicationStatusLabel.unit.spec.ts +++ b/src/modules/features/ReplicationStatusLabel.unit.spec.ts @@ -126,8 +126,9 @@ describe("projectReplicationStatusLabel", () => { 1200 ); expect(transferred.icon).toBe("⚡"); - expect(transferred.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏩ 1200)"); - expect(transferred.parts[transferred.parts.length - 1]).toMatchObject({ + expect(transferred.message).toBe("Sync: ⚡ (⏩ 1200) ↑ 0 ↓ 7"); + expect(transferred.parts.map((part) => part.key)).toEqual(["sync", "ff", "sent", "arrived"]); + expect(transferred.parts[1]).toMatchObject({ key: "ff", text: " (⏩ ", value: 1200, @@ -144,7 +145,7 @@ describe("projectReplicationStatusLabel", () => { 1200 ); expect(idle.icon).toBe("⚡"); - expect(idle.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏩ 1200)"); + expect(idle.message).toBe("Sync: ⚡ (⏩ 1200) ↑ 0 ↓ 7"); }); it("keeps the error icon ahead of an active CouchDB checking indicator", () => { diff --git a/src/modules/features/StatusBarRenderer.ts b/src/modules/features/StatusBarRenderer.ts index 990cc9f5..ceeceafd 100644 --- a/src/modules/features/StatusBarRenderer.ts +++ b/src/modules/features/StatusBarRenderer.ts @@ -1,3 +1,5 @@ +const INACTIVE_CLASS = "livesync-stat-inactive"; + export type StatusBarPart = { key: string; text: string; @@ -22,7 +24,6 @@ type RenderedItem = { export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer { const items = new Map(); - const managedElements = new Set(); let disposed = false; const setText = (node: Text, text: string) => { @@ -33,6 +34,13 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer if (element.hidden !== hidden) element.hidden = hidden; }; + const setInactive = (element: HTMLElement, inactive: boolean) => { + if (element.classList.contains(INACTIVE_CLASS) === inactive) return; + element.classList.toggle(INACTIVE_CLASS, inactive); + if (inactive) element.setAttribute("aria-hidden", "true"); + else element.removeAttribute("aria-hidden"); + }; + const setWidth = (element: HTMLElement, width: string) => { if (element.style.getPropertyValue("--livesync-stat-width") !== width) { element.setCssProps({ "--livesync-stat-width": width }); @@ -58,7 +66,6 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer const item = { element, prefix, value, valueText, suffix, widestValueLength: 0 }; items.set(key, item); - managedElements.add(element); return item; }; @@ -89,20 +96,16 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer }; const reorderIfNeeded = (visibleItems: readonly RenderedItem[]) => { + const desiredOrder = visibleItems.map((item) => item.element); + const visibleElements = new Set(desiredOrder); const currentOrder = Array.from(parent.childNodes).filter((node): node is HTMLSpanElement => - managedElements.has(node as HTMLSpanElement) + visibleElements.has(node as HTMLSpanElement) ); - const visibleElements = new Set(visibleItems.map((item) => item.element)); - const desiredOrder = [ - ...visibleItems.map((item) => item.element), - ...currentOrder.filter((element) => !visibleElements.has(element)), - ]; if ( currentOrder.length === desiredOrder.length && currentOrder.every((node, index) => node === desiredOrder[index]) - ) { + ) return; - } let reference: ChildNode | null = currentOrder[0] ?? null; for (const element of desiredOrder) { @@ -110,7 +113,7 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer const childNodes = Array.from(parent.childNodes); const elementIndex = childNodes.indexOf(element); reference = - childNodes.slice(elementIndex + 1).find((node) => managedElements.has(node as HTMLSpanElement)) ?? null; + childNodes.slice(elementIndex + 1).find((node) => visibleElements.has(node as HTMLSpanElement)) ?? null; } }; @@ -123,17 +126,16 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer for (const part of parts) { let item = items.get(part.key); if (!item) item = createItem(part.key); + if (item.element.classList.contains(INACTIVE_CLASS)) item.widestValueLength = 0; updateItem(item, part); - setHidden(item.element, false); + setInactive(item.element, false); visibleItems.push(item); visibleKeys.add(part.key); } for (const [key, item] of items) { if (visibleKeys.has(key)) continue; - setHidden(item.element, true); - setWidth(item.value, ""); - item.widestValueLength = 0; + setInactive(item.element, true); } reorderIfNeeded(visibleItems); @@ -145,7 +147,6 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer if (item.element.parentNode === parent) parent.removeChild(item.element); } items.clear(); - managedElements.clear(); }, }; } diff --git a/src/modules/features/StatusBarRenderer.unit.spec.ts b/src/modules/features/StatusBarRenderer.unit.spec.ts index 1f1be039..c3232f99 100644 --- a/src/modules/features/StatusBarRenderer.unit.spec.ts +++ b/src/modules/features/StatusBarRenderer.unit.spec.ts @@ -66,6 +66,33 @@ class TestElement { private currentClassName = ""; private currentHidden = false; private currentTitle = ""; + private attributes = new Map(); + readonly classList = { + contains: (token: string): boolean => this.currentClassName.split(" ").includes(token), + toggle: (token: string, force: boolean): boolean => { + const tokens = this.currentClassName.split(" ").filter(Boolean); + const present = tokens.includes(token); + if (present !== force) { + this.className = force + ? [...tokens, token].join(" ") + : tokens.filter((item) => item !== token).join(" "); + } + return force; + }, + }; + + getAttribute(name: string): string | null { + return this.attributes.get(name) ?? null; + } + + setAttribute(name: string, value: string) { + this.document.mutations++; + this.attributes.set(name, value); + } + + removeAttribute(name: string) { + if (this.attributes.delete(name)) this.document.mutations++; + } constructor( private readonly document: TestDocument, @@ -173,7 +200,9 @@ function findValue(item: TestElement): TestElement { } function visibleItems(root: TestElement): TestElement[] { - return root.children.filter((child) => !child.hidden && child.className.split(" ").includes("livesync-stat-item")); + return root.children.filter( + (child) => !child.classList.contains("livesync-stat-inactive") && child.classList.contains("livesync-stat-item") + ); } describe("createStatusBarRenderer", () => { @@ -195,8 +224,10 @@ describe("createStatusBarRenderer", () => { expect(findValue(queue).className).toBe("livesync-stat-value"); renderer.render([{ key: "queue", text: " (queued: ", value: 3, suffix: ")" }]); - expect(sync.hidden).toBe(true); - expect(queue.hidden).toBe(false); + expect(sync.classList.contains("livesync-stat-inactive")).toBe(true); + expect(sync.hidden).toBe(false); + expect(sync.getAttribute("aria-hidden")).toBe("true"); + expect(queue.classList.contains("livesync-stat-inactive")).toBe(false); expect(queue.textContent).toBe(" (queued: 3)"); renderer.render([{ key: "queue", text: "Queue idle" }]); expect(queue.textContent).toBe("Queue idle"); @@ -239,7 +270,7 @@ describe("createStatusBarRenderer", () => { renderer.dispose(); }); - it("resets numeric width while omitted and reuses the same item when it returns", () => { + it("retains omitted content and width, then resets width when the same item returns", () => { const { root, parent } = createFixture(); const renderer = createStatusBarRenderer(parent); @@ -249,16 +280,49 @@ describe("createStatusBarRenderer", () => { expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); renderer.render([]); - expect(item.hidden).toBe(true); - expect(value.style.getPropertyValue("--livesync-stat-width")).toBe(""); + expect(item.classList.contains("livesync-stat-inactive")).toBe(true); + expect(item.hidden).toBe(false); + expect(item.getAttribute("aria-hidden")).toBe("true"); + expect(item.textContent).toBe(" 1234"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); renderer.render([{ key: "count", text: " ", value: 5 }]); expect(findItem(root, "count")).toBe(item); - expect(item.hidden).toBe(false); + expect(item.classList.contains("livesync-stat-inactive")).toBe(false); + expect(item.getAttribute("aria-hidden")).toBeNull(); expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("1ch"); renderer.dispose(); }); + it("keeps inactive items in place when other items appear and does not repeat inactive mutations", () => { + const { document, root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + const first = { key: "first", text: "first " }; + const count = { key: "count", text: "count ", value: 1234 }; + const last = { key: "last", text: "last" }; + renderer.render([first, count, last]); + const initialOrder = [...root.children]; + const omitted = findItem(root, "count"); + + renderer.render([first, last]); + expect(root.children.map((item) => initialOrder.indexOf(item))).toEqual([0, 1, 2]); + expect(visibleItems(root)).toEqual([initialOrder[0], initialOrder[2]]); + const mutations = document.mutations; + renderer.render([first, last]); + expect(document.mutations).toBe(mutations); + + const added = { key: "added", text: "added " }; + renderer.render([first, added, last]); + expect(root.children.map((item) => item.textContent)).toEqual(["first ", "count 1234", "added ", "last"]); + expect(omitted.textContent).toBe("count 1234"); + expect(findValue(omitted).style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); + + renderer.render([first, count, added, last]); + expect(findItem(root, "count")).toBe(omitted); + expect(visibleItems(root)).toEqual(root.children); + renderer.dispose(); + }); + it("reorders existing nodes without replacing them", () => { const { root, parent } = createFixture(); const renderer = createStatusBarRenderer(parent); diff --git a/styles.css b/styles.css index 4a4b0d61..e957358b 100644 --- a/styles.css +++ b/styles.css @@ -844,6 +844,25 @@ body.is-mobile .livesync-compatibility-review-notice { font-variant-numeric: tabular-nums; } +.livesync-stat-item.livesync-stat-inactive { + display: none; +} + +.syncstatusbar .livesync-stat-item { + white-space: pre; +} + +.livesync-status-statusline + > .livesync-stat-item:not(.livesync-stat-inactive) + ~ .livesync-stat-item:not( + .livesync-stat-push-remaining, + .livesync-stat-pull-remaining, + .livesync-stat-ff, + .livesync-stat-p2p + ) { + margin-inline-start: 1em; +} + .livesync-stat-value { display: inline-block; width: var(--livesync-stat-width, auto); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index a13018fb..72155f9b 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -88,7 +88,7 @@ npm run test:e2e:obsidian:focused -- replication-status The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. -`replication-status` uses an isolated Vault without a remote service. It injects synthetic replication statistics into the real plug-in to check the ⏩ icon, the comparison count and its three-second expiry, independently styled numeric items, stable rendered digit widths, and unchanged DOM nodes for repeated values. It switches between split panes, opens a popout editor, checks log updates there, and returns to the main window. Where Electron exposes native window control, it also hides the main window and verifies that popout logs still update before restoring it. This checks display behaviour; Commonlib owns the tests which establish that PouchDB produces the underlying statistics. +`replication-status` uses an isolated Vault without a remote service. It injects synthetic replication statistics into the real plug-in to check the ⏩ icon, the comparison count and its three-second expiry, independently styled numeric items, stable rendered digit widths, and unchanged DOM nodes for repeated values. It verifies the inactive class, retained inactive content and width, and the ability to override the default hiding behaviour through CSS. A separate session enters mobile emulation before loading the plug-in and measures the editor item gaps at a 390px viewport width. The desktop checks also ensure that the status bar keeps its compact spacing. The mobile spacing fixture uses the existing overlay placement method before measurement; it does not verify Android window events. It switches between split panes, opens a popout editor, checks log updates there, and returns to the main window. Where Electron exposes native window control, it also hides the main window and verifies that popout logs still update before restoring it. This checks display behaviour; Commonlib owns the tests which establish that PouchDB produces the underlying statistics. `database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, the default-on integrity-only remote option, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout at 800×600 and 375×667, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. diff --git a/test/e2e-obsidian/scripts/replication-status.ts b/test/e2e-obsidian/scripts/replication-status.ts index 1ef380a1..8445d5c2 100644 --- a/test/e2e-obsidian/scripts/replication-status.ts +++ b/test/e2e-obsidian/scripts/replication-status.ts @@ -3,6 +3,7 @@ import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../r import { withObsidianPage } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; import type { Page } from "playwright"; +import { setObsidianMobileTestModeBeforePluginStart } from "../runner/mobileUi.ts"; const fixturePath = "replication-status-e2e.md"; const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_UI_TIMEOUT_MS ?? 10000); @@ -40,10 +41,12 @@ type TestWorkspace = { getActiveFile?: () => { path?: string } | null; }; type TestLogModule = { + adjustStatusDivPosition?: () => void; statusLog?: WritableSource; statusDiv?: HTMLElement; statusLine?: HTMLElement; statusLineRenderer?: { render?: (parts: readonly unknown[]) => void }; + statusBarRenderer?: { render?: (parts: readonly unknown[]) => void }; statusBarLabels?: WritableSource<{ parts: readonly unknown[] }>; __addLog?: (message: unknown, level?: number, key?: string) => void; }; @@ -171,7 +174,7 @@ async function appendLiveSyncLog(page: Page, message: string): Promise { } async function waitForPartText(page: Page, key: string, expected: string): Promise { - const selector = `.livesync-status-statusline .livesync-stat-${key} .livesync-stat-value`; + const selector = `.livesync-status-statusline .livesync-stat-${key}:not(.livesync-stat-inactive) .livesync-stat-value`; await page.waitForFunction( ({ selector, expected }) => document.querySelector(selector)?.textContent?.trim() === expected, { selector, expected }, @@ -442,11 +445,152 @@ async function assertNumericWidths( return await measureNumericWidths(page); } +async function assertEditorItemSpacing(page: Page, mobile: boolean): Promise { + const spacing = await page.evaluate(() => { + const modules = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core?.modules ?? []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + return typeof (entry as TestLogModule).statusLineRenderer?.render === "function"; + }); + if (!moduleLog?.statusLineRenderer?.render || !moduleLog.statusBarLabels) { + throw new Error("The LiveSync editor status renderer is unavailable."); + } + const original = moduleLog.statusBarLabels.value.parts; + const parts = [ + { key: "queue-replication", text: " 📥", value: 1 }, + { key: "queue-database", text: " 📄", value: 12 }, + { key: "queue-storage", text: " 💾", value: 3 }, + ]; + const results: { selector: string; gapsEm: number[] }[] = []; + try { + moduleLog.statusLineRenderer.render(parts); + moduleLog.statusBarRenderer?.render?.(parts); + for (const selector of [".livesync-status-statusline", ".syncstatusbar"]) { + const root = document.querySelector(selector); + if (!root || root.getBoundingClientRect().width === 0) continue; + const firstItem = root.querySelector(`.livesync-stat-${parts[0].key}`); + if (!firstItem || Number.parseFloat(getComputedStyle(firstItem).marginInlineStart) !== 0) { + throw new Error("Inactive predecessors left a gap before the first active status item."); + } + const fontSize = Number.parseFloat(getComputedStyle(root).fontSize); + const gapsEm: number[] = []; + for (let i = 1; i < parts.length; i++) { + const previous = root.querySelector( + `.livesync-stat-${parts[i - 1].key} .livesync-stat-value` + ); + const next = root.querySelector(`.livesync-stat-${parts[i].key}`)?.firstChild; + if (!previous || next?.nodeType !== Node.TEXT_NODE) + throw new Error("A spacing fixture item is missing."); + const range = document.createRange(); + range.setStart(next, next.textContent?.search(/\S/u) ?? 0); + range.setEnd(next, next.textContent?.length ?? 0); + const nextBox = range.getBoundingClientRect(); + const previousBox = previous.getBoundingClientRect(); + if (Math.abs(nextBox.top - previousBox.top) > 2) + throw new Error("The spacing fixture unexpectedly wrapped."); + gapsEm.push((nextBox.left - previousBox.right) / fontSize); + } + results.push({ selector, gapsEm }); + } + } finally { + moduleLog.statusLineRenderer.render(original); + moduleLog.statusBarRenderer?.render?.(original); + } + return results; + }); + const editor = spacing.find((entry) => entry.selector === ".livesync-status-statusline"); + failUnless(editor?.gapsEm.length === 2, "The editor status gaps could not be measured."); + for (const gap of editor.gapsEm) { + failUnless(gap >= 1 && gap <= 1.5, `Editor status items need a 1–1.5em gap; found ${gap.toFixed(3)}em.`); + } + if (!mobile) { + const statusBar = spacing.find((entry) => entry.selector === ".syncstatusbar"); + failUnless(statusBar?.gapsEm.length === 2, "The status-bar gaps could not be measured."); + failUnless( + statusBar.gapsEm.every((gap) => gap < 1), + "The status-bar items unexpectedly acquired the wider editor gaps." + ); + } + console.log( + `${mobile ? "Mobile" : "Desktop"} editor item gaps: ${editor.gapsEm.map((gap) => gap.toFixed(3)).join(", ")}em.` + ); +} + +async function assertStatusBarItemSpacing(page: Page): Promise { + const collapsedItems = await page.evaluate(() => { + const items = document.querySelectorAll(".syncstatusbar > .livesync-stat-item"); + const collapsed: string[] = []; + for (const item of items) { + if (item.classList.contains("livesync-stat-inactive")) continue; + const prefix = item.firstChild; + if (prefix?.nodeType !== Node.TEXT_NODE || !prefix.textContent?.startsWith(" ")) continue; + const range = document.createRange(); + range.setStart(prefix, 0); + range.setEnd(prefix, 1); + if (range.getBoundingClientRect().width <= 0) collapsed.push(item.className); + } + return collapsed; + }); + failUnless( + collapsedItems.length === 0, + `Status-bar item separators lost their visible spaces: ${collapsedItems.join(", ")}.` + ); +} + +async function assertInactiveItemStyling(page: Page): Promise { + const roots = [".livesync-status-statusline", ".syncstatusbar"]; + for (const root of roots) { + const state = await page.locator(`${root} .livesync-stat-ff`).evaluate((element) => { + const item = element as HTMLElement; + return { + inactive: item.classList.contains("livesync-stat-inactive"), + hidden: item.hidden, + ariaHidden: item.getAttribute("aria-hidden"), + display: getComputedStyle(item).display, + text: item.textContent, + width: item + .querySelector(".livesync-stat-value") + ?.style.getPropertyValue("--livesync-stat-width"), + precedesUploads: item.nextElementSibling?.classList.contains("livesync-stat-sent"), + }; + }); + failUnless( + state.inactive && !state.hidden && state.ariaHidden === "true", + `Inactive state was not exposed through the class in ${root}.` + ); + failUnless(state.display === "none", `Inactive items were not hidden by the default CSS in ${root}.`); + failUnless( + state.text?.includes("1000") && state.width === "4ch" && state.precedesUploads, + `An inactive item lost its content, width, or position in ${root}.` + ); + } + + const snippet = await page.addStyleTag({ + content: ".livesync-stat-item.livesync-stat-inactive { display: inline-block; }", + }); + try { + for (const root of roots) { + failUnless( + await page.locator(`${root} .livesync-stat-ff`).isVisible(), + `A CSS override could not reveal the inactive item in ${root}.` + ); + } + } finally { + await snippet.evaluate((element) => element.parentNode?.removeChild(element)); + } + for (const root of roots) { + failUnless( + await page.locator(`${root} .livesync-stat-ff`).isHidden(), + `The default inactive style was not restored in ${root}.` + ); + } +} + async function measureStatusBarNetworkWidth( page: Page, expectedValue: string ): Promise<{ cssWidth: string; renderedWidth: number }> { - const selector = ".syncstatusbar .livesync-stat-network .livesync-stat-value"; + const selector = ".syncstatusbar .livesync-stat-network:not(.livesync-stat-inactive) .livesync-stat-value"; await page.waitForFunction( ({ selector, expectedValue }) => { const value = document.querySelector(selector); @@ -606,6 +750,13 @@ async function runStatusScenario(session: ObsidianLiveSyncSession): Promise 0) await page.waitForTimeout(remainingLinger); await page.waitForFunction( () => - document.querySelector(".livesync-status-statusline .livesync-stat-ff")?.hidden === - true, + document + .querySelector(".livesync-status-statusline .livesync-stat-ff") + ?.classList.contains("livesync-stat-inactive") === true, undefined, { timeout: uiTimeoutMs } ); + await assertInactiveItemStyling(page); const iconAfterExpiry = await page.locator(".livesync-status-statusline .livesync-stat-sync").textContent(); failUnless( iconAfterExpiry?.includes("⏩"), @@ -735,32 +888,69 @@ async function main(): Promise { const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); - const vault = await createTemporaryVault("obsidian-livesync-replication-status-"); - let session: ObsidianLiveSyncSession | undefined; - try { - session = await startObsidianLiveSyncSession({ - binary, - cliBinary: cli.binary, - vault, - startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), - pluginData: { - liveSync: false, - syncOnStart: false, - syncOnSave: false, - isConfigured: false, - remoteType: "", - showStatusOnEditor: true, - showOnlyIconsOnEditor: false, - showStatusOnStatusbar: true, - lessInformationInLog: false, - writeLogToTheFile: false, - }, - }); - failUnless(session.readiness.pluginId === "obsidian-livesync", "The LiveSync plug-in did not become ready."); - await runStatusScenario(session); - } finally { - if (session) await session.app.stop(); - await vault.dispose(); + for (const mobile of [false, true]) { + const vault = await createTemporaryVault("obsidian-livesync-replication-status-"); + let session: ObsidianLiveSyncSession | undefined; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli.binary, + vault, + lifecycle: mobile + ? { + beforePluginStart: async ({ remoteDebuggingPort }) => { + await setObsidianMobileTestModeBeforePluginStart(remoteDebuggingPort, true, uiTimeoutMs); + }, + } + : undefined, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + pluginData: { + liveSync: false, + syncOnStart: false, + syncOnSave: false, + isConfigured: false, + remoteType: "", + showStatusOnEditor: true, + showOnlyIconsOnEditor: false, + showStatusOnStatusbar: true, + lessInformationInLog: false, + writeLogToTheFile: false, + }, + }); + failUnless( + session.readiness.pluginId === "obsidian-livesync", + "The LiveSync plug-in did not become ready." + ); + if (!mobile) await runStatusScenario(session); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await installFixtureFile(page); + if (mobile) { + await page.evaluate(() => { + const modules = + (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core?.modules ?? + []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + return typeof (entry as TestLogModule).adjustStatusDivPosition === "function"; + }); + moduleLog?.adjustStatusDivPosition?.(); + }); + } + try { + await page.locator(".livesync-status").waitFor({ state: "attached", timeout: uiTimeoutMs }); + await injectReplicationStats(page, replicationStats(0, 0, 0), 0, 0); + await page.locator(".livesync-status").waitFor({ state: "visible", timeout: uiTimeoutMs }); + } catch (error) { + throw new Error( + `${String(error)}; spacing setup: ${JSON.stringify(await getStatusDiagnostics(page))}` + ); + } + await assertEditorItemSpacing(page, mobile); + }); + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + } } } diff --git a/updates.md b/updates.md index e3c00366..0929f2e0 100644 --- a/updates.md +++ b/updates.md @@ -14,8 +14,8 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Synchronisation -- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏩ status icon and a comparison count beside received items. An unchanged count disappears after three seconds. -- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. +- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏩ status icon and a comparison count before the upload count. An unchanged count disappears after three seconds. +- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. Wider gaps between editor status items make the counts easier to read. The `livesync-stat-inactive` class lets CSS snippets control how inactive items disappear. - We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) ## 1.0.40 From 62b2edba9dde87d6d154457fa49faeeaa9e4161d Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 13:26:02 +0000 Subject: [PATCH 23/28] fix: await optional file database saves --- ...local_file_save_completion_and_batching.md | 31 +++ package.json | 1 + .../CmdConfigSync.completion.unit.spec.ts | 220 +++++++++++++++ src/features/ConfigSync/CmdConfigSync.ts | 100 ++++++- src/features/ConfigSync/PendingConfigSaves.ts | 49 ++++ .../PendingConfigSaves.unit.spec.ts | 84 ++++++ .../CmdHiddenFileSync.completion.unit.spec.ts | 260 ++++++++++++++++++ .../HiddenFileSync/CmdHiddenFileSync.ts | 155 ++++++++++- test/e2e-obsidian/README.md | 2 + .../scripts/optional-file-save-completion.ts | 230 ++++++++++++++++ updates.md | 1 + 11 files changed, 1113 insertions(+), 20 deletions(-) create mode 100644 src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts create mode 100644 src/features/ConfigSync/PendingConfigSaves.ts create mode 100644 src/features/ConfigSync/PendingConfigSaves.unit.spec.ts create mode 100644 src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/optional-file-save-completion.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md index 8abef973..27e6e4f6 100644 --- a/docs/adr/2026_10_local_file_save_completion_and_batching.md +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -85,6 +85,37 @@ Ordinary file saving by Obsidian remains independent of database saving. Pending and processing indicators must include work in progress and unresolved failures, so an empty timer map cannot imply that local processing is complete. +### Optional file ownership and completion + +Hidden File Sync and Customisation Sync register the new +`processOptionalFileEventWithResult` contract together. `firstHandled` tries +providers in order until one returns `completed` or `veto`; only +`not-applicable` advances to the next provider. Exceptions propagate unchanged. +`HandlerOutcome` gives completion data and refusal reasons +independent types. These saves use `void` completion data and string reasons. + +A provider returns `completed` after its database operation succeeds or the +current state is already represented. Failed writes and deletions throw, and +the manager retains the event. Hidden File Sync acknowledges its processed +fingerprint after successful processing. Customisation Sync waits for the real +save after its existing 100ms quiet period. All notifications coalesced into a +V1 group receive that save's result; V2 files retain separate save groups. + +A finite flush resolves each captured event's `whenFlushRequested` signal. This +releases its customisation delay and freezes that group; later groups do not +extend its completion. During start-up replay or suspension, providers use +physical database readiness rather than waiting for application readiness or +resume. Hidden File Sync loads its existing per-database caches before replay +and reuses them during the following initialisation hook. + +The existing boolean optional API remains a legacy notification contract for +other callers and older hosts. Once a host registers typed providers, the +manager dispatches through that chain exclusively. Without a typed provider, +a resolved legacy notification completes that dispatch only; its boolean does +not establish a database save. Snapshot-write failures remain visible under +both contracts. No persistent journal, new snapshot schema, or automatic retry +loop is introduced. + ### 2. Keep a recoverable record through processing The event manager owns each operation from queue admission through completion. diff --git a/package.json b/package.json index cd383dde..5b35668b 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", "test:e2e:obsidian:hidden-file-selectors": "tsx test/e2e-obsidian/scripts/hidden-file-selectors.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", + "test:e2e:obsidian:optional-file-save-completion": "tsx test/e2e-obsidian/scripts/optional-file-save-completion.ts", "test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts", "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", "test:e2e:obsidian:internal-metadata-migration": "tsx test/e2e-obsidian/scripts/internal-metadata-migration.ts", diff --git a/src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts new file mode 100644 index 00000000..147ee857 --- /dev/null +++ b/src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts @@ -0,0 +1,220 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + diff_match_patch: class DiffMatchPatch {}, + normalizePath: vi.fn((path: string) => path), + parseYaml: vi.fn(), + Platform: {}, +})); +vi.mock("./PluginDialogModal.ts", () => ({ + PluginDialogModal: class PluginDialogModal {}, +})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ + JsonResolveModal: class JsonResolveModal {}, +})); +vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({ + ConflictResolveModal: class ConflictResolveModal {}, +})); +// The readiness helpers come from the real LiveSyncCommands, so only its log view dependency is replaced. +vi.mock("@/modules/features/ModuleLog.ts", () => ({ + MARK_DONE: "", +})); +vi.mock("@/common/types.ts", () => ({ + ICXHeader: "ix:", + PERIODIC_PLUGIN_SWEEP: 60, +})); +vi.mock("@/common/utils.ts", () => ({ + cancelTask: vi.fn(), + EVEN: Symbol("even"), + isCustomisationSyncMetadata: vi.fn(), + isPluginMetadata: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ + PeriodicProcessor: class PeriodicProcessor {}, +})); +vi.mock("@/common/events.ts", () => ({ + EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync", + eventHub: { + onEvent: vi.fn(), + }, +})); +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ + getObsidianCommunityPluginManager: vi.fn(), +})); + +import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { getDocData } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { ConfigSync } from "./CmdConfigSync"; +import { createOptionalFileSyncDatabase } from "../../../test/fixtures/optionalFileSyncDatabase"; + +const path = ".obsidian/snippets/test.css" as FilePath; +const fixtures: Awaited>[] = []; + +async function createConfigSync(useV2 = false) { + const fixture = await createOptionalFileSyncDatabase(); + fixtures.push(fixture); + fixture.settings.usePluginSyncV2 = useV2; + const files = new Map([ + [path, { content: "body { color: red; }", mtime: 2000 }], + ]); + const storageAccess = { + statHidden: vi.fn(async (file: string) => { + const entry = files.get(file); + return entry + ? { type: "file" as const, ctime: 1000, mtime: entry.mtime, size: entry.content.length } + : null; + }), + readHiddenFileBinary: vi.fn(async (file: string) => { + const entry = files.get(file); + if (!entry) throw new Error(`Missing fixture: ${file}`); + return new TextEncoder().encode(entry.content).buffer; + }), + }; + const services = { + ...fixture.services, + appLifecycle: { isReady: () => false, isSuspended: () => true }, + database: { ...fixture.services.database, isDatabaseReady: vi.fn(() => true) }, + path: { + ...fixture.services.path, + isMarkedAsSameChanges: vi.fn(), + markChangesAreSame: vi.fn(), + }, + }; + const config = Object.create(ConfigSync.prototype) as ConfigSync; + Object.assign(config, { + core: { settings: fixture.settings, localDatabase: fixture.localDatabase, storageAccess, services }, + _log: vi.fn(), + updatePluginList: vi.fn(async () => undefined), + updatePluginListV2: vi.fn(async () => undefined), + recentProcessedInternalFiles: [], + }); + const databasePath = useV2 ? config.filenameWithUnifiedKey(path) : config.filenameToUnifiedKey(path); + return { fixture, config, files, services, storageAccess, databasePath }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + for (const fixture of fixtures.splice(0)) await fixture.dispose(); +}); + +describe("Customisation Sync optional save completion", () => { + const flush = { whenFlushRequested: Promise.resolve() }; + + it("waits for the database write while the application is suspended or still starting", async () => { + const { fixture, config } = await createConfigSync(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const put = fixture.localDatabase.putDBEntry.bind(fixture.localDatabase); + const spy = vi.spyOn(fixture.localDatabase, "putDBEntry").mockImplementation(async (...args) => { + await gate; + return await put(...args); + }); + let completed = false; + const result = config._processOptionalFileEventWithResult(path, flush).then((outcome) => { + completed = true; + return outcome; + }); + await vi.waitFor(() => expect(spy).toHaveBeenCalledOnce()); + expect(completed).toBe(false); + release(); + await expect(result).resolves.toEqual({ kind: "completed" }); + }); + + it.each([false, true])("confirms identical contents and stores a same-time edit (V2: %s)", async (useV2) => { + const { fixture, config, databasePath, files } = await createConfigSync(useV2); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + const first = await fixture.database.get(databasePath); + await config._processOptionalFileEventWithResult(path, flush); + expect((await fixture.database.get(databasePath))._rev).toBe(first._rev); + files.get(path)!.content = "body { color: tan; }"; + await config._processOptionalFileEventWithResult(path, flush); + expect((await fixture.database.get(databasePath))._rev).not.toBe(first._rev); + }); + + it.each([ + { useV2: false, throws: false }, + { useV2: true, throws: false }, + { useV2: false, throws: true }, + { useV2: true, throws: true }, + ])("rejects a failed write and permits the unchanged notification to retry: %j", async ({ useV2, throws }) => { + const { fixture, config, databasePath } = await createConfigSync(useV2); + const failure = new Error("temporary write failure"); + const put = vi.spyOn(fixture.localDatabase, "putDBEntry"); + if (throws) put.mockRejectedValueOnce(failure); + else put.mockResolvedValueOnce(false); + const failed = config._processOptionalFileEventWithResult(path, flush); + if (throws) await expect(failed).rejects.toBe(failure); + else await expect(failed).rejects.toThrow("Could not save customisation"); + await expect(fixture.localDatabase.getDBEntry(databasePath)).resolves.toBe(false); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + expect(put).toHaveBeenCalledTimes(2); + expect(await fixture.localDatabase.getDBEntry(databasePath)).not.toBe(false); + }); + + it.each([false, true])("confirms deletion and treats a repeated deletion as complete (V2: %s)", async (useV2) => { + const { fixture, config, databasePath, files } = await createConfigSync(useV2); + await config._processOptionalFileEventWithResult(path, flush); + files.delete(path); + const put = vi.spyOn(fixture.localDatabase, "putRaw"); + put.mockResolvedValueOnce({ ok: false, id: "failed", rev: "" }); + await expect(config._processOptionalFileEventWithResult(path, flush)).rejects.toThrow( + "Could not delete customisation" + ); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + const deleted = await fixture.database.get(databasePath); + expect("deleted" in deleted && deleted.deleted).toBe(true); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + expect((await fixture.database.get(databasePath))._rev).toBe(deleted._rev); + }); + + it("distinguishes a different owner and unmet prerequisites", async () => { + const { config, services, storageAccess } = await createConfigSync(); + await expect(config._processOptionalFileEventWithResult("notes.md" as FilePath, flush)).resolves.toEqual({ + kind: "not-applicable", + }); + services.database.isDatabaseReady.mockReturnValue(false); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ + kind: "veto", + value: "database-not-ready", + }); + services.database.isDatabaseReady.mockReturnValue(true); + vi.spyOn(services.setting, "getDeviceAndVaultName").mockReturnValue(""); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ + kind: "veto", + value: "device-name-not-configured", + }); + expect(storageAccess.readHiddenFileBinary).not.toHaveBeenCalled(); + }); + + it("preserves every V2 file when neighbouring plug-in files arrive together", async () => { + const { fixture, config, files } = await createConfigSync(true); + const main = ".obsidian/plugins/example/main.js" as FilePath; + const style = ".obsidian/plugins/example/styles.css" as FilePath; + files.set(main, { content: "module.exports = {};", mtime: 3000 }); + files.set(style, { content: "body {}", mtime: 3000 }); + await Promise.all([ + config._processOptionalFileEventWithResult(main, flush), + config._processOptionalFileEventWithResult(style, flush), + ]); + expect(await fixture.localDatabase.getDBEntry(config.filenameWithUnifiedKey(main))).not.toBe(false); + expect(await fixture.localDatabase.getDBEntry(config.filenameWithUnifiedKey(style))).not.toBe(false); + }); + + it("does not acknowledge a database deletion which was not accepted", async () => { + const { fixture, config, databasePath } = await createConfigSync(); + await config.storeCustomizationFiles(path); + const original = await fixture.localDatabase.getDBEntry(databasePath); + expect(original).not.toBe(false); + vi.spyOn(fixture.localDatabase, "putRaw").mockResolvedValueOnce({ ok: false, id: "failed", rev: "" }); + await expect(config.deleteConfigOnDatabase(databasePath)).resolves.toBe(false); + const unchanged = await fixture.localDatabase.getDBEntry(databasePath); + expect(unchanged && getDocData(unchanged.data)).toBe(original && getDocData(original.data)); + }); +}); diff --git a/src/features/ConfigSync/CmdConfigSync.ts b/src/features/ConfigSync/CmdConfigSync.ts index 0e1412ee..cbdad936 100644 --- a/src/features/ConfigSync/CmdConfigSync.ts +++ b/src/features/ConfigSync/CmdConfigSync.ts @@ -69,6 +69,10 @@ import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError" import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts"; +import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; +import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import { PendingConfigSaves } from "./PendingConfigSaves"; + const d = "\u200b"; const d2 = "\n"; const UPDATED_CONFIGURATION_NOTICE_KEY = "config-sync:updated-configuration"; @@ -1206,7 +1210,7 @@ export class ConfigSync extends LiveSyncCommands { } recentProcessedInternalFiles = [] as string[]; - async makeEntryFromFile(path: FilePath): Promise { + async makeEntryFromFile(path: FilePath, requireCompletion = false): Promise { const stat = await this.core.storageAccess.statHidden(path); let version: string | undefined; let displayName: string | undefined; @@ -1240,6 +1244,7 @@ export class ConfigSync extends LiveSyncCommands { } catch (ex) { this._log(`The file ${path} could not be encoded`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } const mtime = stat.mtime; @@ -1254,6 +1259,15 @@ export class ConfigSync extends LiveSyncCommands { } async storeCustomisationFileV2(path: FilePath, term: string, force = false) { + return await this.storeCustomisationFileV2Core(path, term, force, false); + } + + private async storeCustomisationFileV2Core( + path: FilePath, + term: string, + force: boolean, + requireCompletion: boolean + ) { const vf = this.filenameWithUnifiedKey(path, term); return await serialized(`plugin-${vf}`, async () => { const prefixedFileName = vf; @@ -1261,7 +1275,8 @@ export class ConfigSync extends LiveSyncCommands { const id = await this.path2id(prefixedFileName); const stat = await this.core.storageAccess.statHidden(path); if (!stat) { - return false; + if (!requireCompletion) return false; + return await this.deleteConfigOnDatabaseCore(vf, force, true); } const mtime = stat.mtime; const content = await this.core.storageAccess.readHiddenFileBinary(path); @@ -1285,7 +1300,10 @@ export class ConfigSync extends LiveSyncCommands { eden: {}, }; } else { - if (this.services.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN) { + if ( + !requireCompletion && + this.services.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN + ) { this._log( `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, LOG_LEVEL_DEBUG @@ -1300,7 +1318,7 @@ export class ConfigSync extends LiveSyncCommands { const dataStart = dataSrc.indexOf(DUMMY_END); const oldContent = dataSrc.substring(dataStart + DUMMY_END.length); const oldContentArray = base64ToArrayBuffer(oldContent); - if (await isDocContentSame(oldContentArray, content)) { + if (!old.deleted && (await isDocContentSame(oldContentArray, content))) { this._log( `STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, LOG_LEVEL_VERBOSE @@ -1320,24 +1338,36 @@ export class ConfigSync extends LiveSyncCommands { }; } const ret = await this.localDatabase.putDBEntry(saveData); + if (requireCompletion && (!ret || !ret.ok)) + throw new LiveSyncError(`Could not save customisation: ${path}`); this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); fireAndForget(() => this.updatePluginListV2(false, this.filenameWithUnifiedKey(path))); return ret; } catch (ex) { this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } }); } async storeCustomizationFiles(path: FilePath, termOverRide?: string) { + return await this.storeCustomizationFilesCore(path, termOverRide, false); + } + + private async storeCustomizationFilesCore( + path: FilePath, + termOverRide: string | undefined, + requireCompletion: boolean + ) { const term = termOverRide || this.services.setting.getDeviceAndVaultName(); if (term == "") { this._log($msg("We have to configure the device name"), LOG_LEVEL_NOTICE); + if (requireCompletion) throw new LiveSyncError("The device name is not configured"); return; } if (this.useV2) { - return await this.storeCustomisationFileV2(path, term); + return await this.storeCustomisationFileV2Core(path, term, false, requireCompletion); } const vf = this.filenameToUnifiedKey(path, term); // console.warn(`Storing ${path} to ${bareVF} :--> ${keyedVF}`); @@ -1378,7 +1408,7 @@ export class ConfigSync extends LiveSyncCommands { fileTargets = ["manifest.json", "theme.css"].map((e) => `${parentPath}/${e}` as FilePath); } for (const target of fileTargets) { - const data = await this.makeEntryFromFile(target); + const data = await this.makeEntryFromFile(target, requireCompletion); if (data == false) { this._log(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE); continue; @@ -1398,9 +1428,9 @@ export class ConfigSync extends LiveSyncCommands { // this._log(`Configuration saving: ${prefixedFileName}`); if (dt.files.length == 0) { this._log(`Nothing left: deleting.. ${path}`); - await this.deleteConfigOnDatabase(prefixedFileName); + const deleted = await this.deleteConfigOnDatabaseCore(prefixedFileName, false, requireCompletion); await this.updatePluginList(false, prefixedFileName); - return; + return requireCompletion ? deleted : undefined; } const content = createTextBlob(serialize(dt)); @@ -1422,7 +1452,7 @@ export class ConfigSync extends LiveSyncCommands { eden: {}, }; } else { - if (old.mtime == mtime) { + if (!requireCompletion && old.mtime == mtime) { // this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE); return true; } @@ -1443,7 +1473,7 @@ export class ConfigSync extends LiveSyncCommands { } }); const isSame = (await Promise.all(diffs)).every((e) => e == true); - if (isSame) { + if (isSame && !old.deleted) { this._log( `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, LOG_LEVEL_VERBOSE @@ -1464,16 +1494,51 @@ export class ConfigSync extends LiveSyncCommands { }; } const ret = await this.localDatabase.putDBEntry(saveData); + if (requireCompletion && (!ret || !ret.ok)) + throw new LiveSyncError(`Could not save customisation: ${path}`); await this.updatePluginList(false, saveData.path); this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); return ret; } catch (ex) { this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } }); } + private pendingOptionalSaves?: PendingConfigSaves; + + async _processOptionalFileEventWithResult( + path: FilePath, + options?: OptionalFileEventOptions + ): Promise> { + if (!this.isThisModuleEnabled() || !this.isTargetPath(path)) return { kind: "not-applicable" }; + const configDir = normalizePath(this.configDir); + const handledElsewhere = Object.values(this.settings.pluginSyncExtendedSetting) + .filter((entry) => entry.mode !== MODE_SELECTIVE && entry.mode !== MODE_SHINY) + .flatMap((entry) => entry.files) + .some((file) => `${configDir}/${file}`.toLowerCase().startsWith(path.toLowerCase())); + if (handledElsewhere) return { kind: "not-applicable" }; + if (!this._isDatabaseReady()) return { kind: "veto", value: "database-not-ready" }; + const term = this.services.setting.getDeviceAndVaultName(); + if (!term) return { kind: "veto", value: "device-name-not-configured" }; + const stat = await this.core.storageAccess.statHidden(path); + if (stat && stat.type !== "file") return { kind: "not-applicable" }; + + // V1 saves a complete file group; V2 persists each file separately. + const key = this.useV2 ? this.filenameWithUnifiedKey(path, term) : this.filenameToUnifiedKey(path, term); + this.pendingOptionalSaves ??= new PendingConfigSaves(); + await this.pendingOptionalSaves.enqueue( + key, + async () => { + await this.storeCustomizationFilesCore(path, term, true); + }, + options?.whenFlushRequested + ); + return { kind: "completed" }; + } + async _anyProcessOptionalFileEvent(path: FilePath): Promise { return await this.watchVaultRawEventsAsync(path); } @@ -1615,6 +1680,14 @@ export class ConfigSync extends LiveSyncCommands { } async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false) { + return await this.deleteConfigOnDatabaseCore(prefixedFileName, forceWrite, false); + } + + private async deleteConfigOnDatabaseCore( + prefixedFileName: FilePathWithPrefix, + forceWrite: boolean, + requireCompletion: boolean + ) { // const id = await this.path2id(prefixedFileName); const mtime = new Date().getTime(); return await serialized("file-x-" + prefixedFileName, async () => { @@ -1640,13 +1713,15 @@ export class ConfigSync extends LiveSyncCommands { type: "newnote", }; } - await this.localDatabase.putRaw(saveData); + const result = await this.localDatabase.putRaw(saveData); + if (!result.ok) throw new LiveSyncError(`Could not delete customisation: ${prefixedFileName}`); await this.updatePluginList(false, prefixedFileName); this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Done`); return true; } catch (ex) { this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } }); @@ -1756,6 +1831,9 @@ export class ConfigSync extends LiveSyncCommands { } override onBindFunction(core: LiveSyncCore, services: InjectableServiceHub): void { services.fileProcessing.processOptionalFileEvent.addHandler(this._anyProcessOptionalFileEvent.bind(this)); + services.fileProcessing.processOptionalFileEventWithResult?.addHandler( + this._processOptionalFileEventWithResult.bind(this) + ); services.conflict.getOptionalConflictCheckMethod.addHandler(this._anyGetOptionalConflictCheckMethod.bind(this)); services.replication.processVirtualDocument.addHandler(this._anyModuleParsedReplicationResultItem.bind(this)); services.setting.onRealiseSetting.addHandler(this._everyRealizeSettingSyncMode.bind(this)); diff --git a/src/features/ConfigSync/PendingConfigSaves.ts b/src/features/ConfigSync/PendingConfigSaves.ts new file mode 100644 index 00000000..6243fe93 --- /dev/null +++ b/src/features/ConfigSync/PendingConfigSaves.ts @@ -0,0 +1,49 @@ +import { compatGlobal, type CompatTimeoutHandle } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; + +type PendingSave = { + save: () => Promise; + completion: Promise; + resolve: () => void; + reject: (error: unknown) => void; + timer?: CompatTimeoutHandle; + started: boolean; +}; + +/** Owns the quiet-period delay and completion of coalesced customisation saves. */ +export class PendingConfigSaves { + private readonly pending = new Map(); + + enqueue(key: string, save: () => Promise, whenFlushRequested?: Promise): Promise { + let group = this.pending.get(key); + if (!group) { + let resolve!: () => void; + let reject!: (error: unknown) => void; + const completion = new Promise((onResolve, onReject) => { + resolve = onResolve; + reject = onReject; + }); + group = { save, completion, resolve, reject, started: false }; + this.pending.set(key, group); + } else { + group.save = save; + if (group.timer !== undefined) compatGlobal.clearTimeout(group.timer); + } + const captured = group; + group.timer = compatGlobal.setTimeout(() => this.run(key, captured), 100); + // Capture this group: a late flush of an earlier event must not release + // a newer group which happens to use the same customisation key. + void whenFlushRequested?.then( + () => this.run(key, captured), + () => {} + ); + return group.completion; + } + + private run(key: string, group: PendingSave): void { + if (group.started) return; + group.started = true; + if (group.timer !== undefined) compatGlobal.clearTimeout(group.timer); + this.pending.delete(key); + void Promise.resolve().then(group.save).then(group.resolve, group.reject); + } +} diff --git a/src/features/ConfigSync/PendingConfigSaves.unit.spec.ts b/src/features/ConfigSync/PendingConfigSaves.unit.spec.ts new file mode 100644 index 00000000..2b38e159 --- /dev/null +++ b/src/features/ConfigSync/PendingConfigSaves.unit.spec.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { PendingConfigSaves } from "./PendingConfigSaves"; + +function deferred() { + let resolve!: () => void; + let reject!: (error: unknown) => void; + const promise = new Promise((onResolve, onReject) => { + resolve = onResolve; + reject = onReject; + }); + return { promise, resolve, reject }; +} + +afterEach(() => vi.useRealTimers()); + +describe("pending customisation saves", () => { + it("retains the quiet period and completes every coalesced notification after the actual save", async () => { + vi.useFakeTimers(); + const pending = new PendingConfigSaves(); + const save = deferred(); + const firstSave = vi.fn(async () => {}); + const lastSave = vi.fn(() => save.promise); + const first = pending.enqueue("plugin", firstSave); + await vi.advanceTimersByTimeAsync(80); + const second = pending.enqueue("plugin", lastSave); + let completed = false; + const all = Promise.all([first, second]).then(() => { + completed = true; + }); + await vi.advanceTimersByTimeAsync(99); + expect(lastSave).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(firstSave).not.toHaveBeenCalled(); + expect(lastSave).toHaveBeenCalledOnce(); + expect(completed).toBe(false); + save.resolve(); + await all; + expect(completed).toBe(true); + }); + + it("flushes the captured group without waiting for or releasing later notifications", async () => { + vi.useFakeTimers(); + const pending = new PendingConfigSaves(); + const flush = deferred(); + const lateFlush = deferred(); + const firstSave = deferred(); + const work = vi.fn(() => firstSave.promise); + const first = pending.enqueue("plugin", work, flush.promise); + const coalesced = pending.enqueue("plugin", work, lateFlush.promise); + flush.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(work).toHaveBeenCalledOnce(); + const nextWork = vi.fn(async () => {}); + const later = pending.enqueue("plugin", nextWork); + lateFlush.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(nextWork).not.toHaveBeenCalled(); + firstSave.resolve(); + await Promise.all([first, coalesced]); + expect(nextWork).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(100); + await later; + expect(nextWork).toHaveBeenCalledOnce(); + }); + + it("rejects every coalesced notification with the original failure and permits a later retry", async () => { + vi.useFakeTimers(); + const pending = new PendingConfigSaves(); + const failure = new Error("database write failed"); + const save = async () => { + throw failure; + }; + const first = pending.enqueue("plugin", save); + const second = pending.enqueue("plugin", save); + const outcomes = Promise.allSettled([first, second]); + await vi.advanceTimersByTimeAsync(100); + expect(await outcomes).toEqual([ + { status: "rejected", reason: failure }, + { status: "rejected", reason: failure }, + ]); + const retry = pending.enqueue("plugin", async () => {}, Promise.resolve()); + await retry; + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts new file mode 100644 index 00000000..5ef0e4de --- /dev/null +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts @@ -0,0 +1,260 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; + +vi.mock("@/deps.ts", () => ({})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("./configureHiddenFileSyncMode.ts", () => ({ configureHiddenFileSyncMode: vi.fn() })); + +import type { FilePath, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { readContent } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { HiddenFileSync } from "./CmdHiddenFileSync"; +import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase"; + +const path = ".obsidian/plugins/example/main.js" as FilePath; +const content = "module.exports = class ExamplePlugin {};"; +const fixtureOwners: Awaited>[] = []; + +async function getText(entry: Parameters[0]) { + const value = await readContent(entry); + return typeof value === "string" ? value : new TextDecoder().decode(value); +} + +async function createHiddenFileSync() { + const fixture = await createOptionalFileSyncDatabase(); + fixtureOwners.push(fixture); + fixture.settings.usePluginSync = false; + fixture.settings.syncInternalFilesTargetPatterns = "" as typeof fixture.settings.syncInternalFilesTargetPatterns; + fixture.settings.syncInternalFilesIgnorePatterns = "" as typeof fixture.settings.syncInternalFilesIgnorePatterns; + let fileContent = content; + let stat: UXStat | null = { + ctime: 1_000, + mtime: 2_000, + size: fileContent.length, + type: "file", + }; + const values = new Map(); + const kvDB = { + get: vi.fn(async (key: string) => values.get(key) as T | undefined), + set: vi.fn(async (key: string, value: T) => { + values.set(key, value); + }), + }; + const storageAccess = { + statHidden: vi.fn(async () => stat), + readHiddenFileAuto: vi.fn(async () => fileContent), + writeHiddenFileAuto: vi.fn(async () => undefined), + removeHidden: vi.fn(async () => true), + isExistsIncludeHidden: vi.fn(async () => true), + ensureDir: vi.fn(async () => undefined), + triggerHiddenFile: vi.fn(async () => undefined), + setFile(nextContent: string | null, mtime = 2_000) { + fileContent = nextContent ?? ""; + stat = nextContent === null ? null : { ctime: 1_000, mtime, size: fileContent.length, type: "file" }; + }, + }; + const databaseService = { isDatabaseReady: vi.fn(() => true) }; + const appLifecycle = { isReady: vi.fn(() => false), isSuspended: vi.fn(() => true) }; + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { + settings: fixture.settings, + localDatabase: fixture.localDatabase, + kvDB, + storageAccess, + services: { + ...fixture.services, + database: databaseService, + appLifecycle, + path: { + ...fixture.services.path, + markChangesAreSame: vi.fn(), + unmarkChanges: vi.fn(), + }, + vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) }, + }, + }, + cacheFileRegExps: new Map(), + cacheCustomisationSyncIgnoredFiles: new Map(), + _fileInfoLastProcessed: new Map(), + _databaseInfoLastProcessed: new Map(), + _fileInfoLastKnown: new Map(), + semaphore: Semaphore(10), + _log: vi.fn(), + }); + return { fixture, hiddenFileSync, storageAccess, kvDB, databaseService, appLifecycle }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + for (const fixture of fixtureOwners.splice(0)) await fixture.dispose(); +}); + +describe("Hidden File Sync optional save outcomes", () => { + it("retries a failed database write when the storage fingerprint is unchanged", async () => { + const { fixture, hiddenFileSync } = await createHiddenFileSync(); + await expect(hiddenFileSync.isTargetFile(path)).resolves.toBe(true); + vi.spyOn(fixture.localDatabase, "putDBEntry").mockRejectedValueOnce(new Error("temporary database failure")); + + await expect(hiddenFileSync.trackStorageFileModification(path)).resolves.toBe(false); + await expect(hiddenFileSync.trackStorageFileModification(path)).resolves.toBe(true); + + expect(fixture.localDatabase.putDBEntry).toHaveBeenCalledTimes(2); + const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(saved).not.toBe(false); + }); + + it("rejects a failed strict write and retries the same fingerprint without delaying readiness", async () => { + const { fixture, hiddenFileSync, appLifecycle } = await createHiddenFileSync(); + const putDBEntry = vi + .spyOn(fixture.localDatabase, "putDBEntry") + .mockRejectedValueOnce(new Error("temporary database failure")); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow( + "temporary database failure" + ); + expect(hiddenFileSync.getLastProcessedFileKey(path)).toBeUndefined(); + + await expect( + hiddenFileSync._processOptionalFileEventWithResult(path, { + whenFlushRequested: new Promise(() => undefined), + }) + ).resolves.toMatchObject({ + kind: "completed", + }); + expect(putDBEntry).toHaveBeenCalledTimes(2); + const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(saved).not.toBe(false); + if (saved) expect(await getText(saved)).toBe(content); + expect(appLifecycle.isReady()).toBe(false); + expect(appLifecycle.isSuspended()).toBe(true); + }); + + it("rejects a false database-write result and can store a later edit after a same-content event", async () => { + const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync(); + const putDBEntry = vi + .spyOn(fixture.localDatabase, "putDBEntry") + .mockImplementationOnce(async () => false as never); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow( + `Failed to store hidden file in the database: ${path}` + ); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + const first = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(first).not.toBe(false); + if (first) { + const firstRevision = first._rev; + hiddenFileSync._fileInfoLastProcessed.delete(path); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + const unchanged = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(unchanged).not.toBe(false); + if (unchanged) expect(unchanged._rev).toBe(firstRevision); + } + + storageAccess.setFile("module.exports = class UpdatedExamplePlugin {};", 3_000); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + const updated = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(updated).not.toBe(false); + if (updated) expect(await getText(updated)).toBe("module.exports = class UpdatedExamplePlugin {};"); + expect(putDBEntry).toHaveBeenCalledTimes(3); + }); + + it("vetoes when the physical database is unavailable and ignores disabled or filtered events", async () => { + const { fixture, hiddenFileSync, databaseService } = await createHiddenFileSync(); + + databaseService.isDatabaseReady.mockReturnValue(false); + await expect( + hiddenFileSync._processOptionalFileEventWithResult(path, { + whenFlushRequested: new Promise(() => undefined), + }) + ).resolves.toMatchObject({ kind: "veto", value: expect.any(String) }); + databaseService.isDatabaseReady.mockReturnValue(true); + + fixture.settings.syncInternalFiles = false; + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "not-applicable", + }); + fixture.settings.syncInternalFiles = true; + fixture.settings.syncInternalFilesTargetPatterns = + "^\\.obsidian/snippets/" as typeof fixture.settings.syncInternalFilesTargetPatterns; + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "not-applicable", + }); + }); + + it("treats an absent replayed deletion as complete and leaves cache initialisation intact", async () => { + const { hiddenFileSync, storageAccess, kvDB } = await createHiddenFileSync(); + storageAccess.setFile(null); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + expect(hiddenFileSync.getLastProcessedFileKey(path)).toBe("0-0"); + const lifecycle = hiddenFileSync as unknown as { + _everyOnDatabaseInitialized(showNotice: boolean): Promise; + }; + hiddenFileSync.core.settings.syncInternalFiles = false; + await lifecycle._everyOnDatabaseInitialized(false); + expect(kvDB.get).toHaveBeenCalledTimes(3); + expect(hiddenFileSync.getLastProcessedFileKey(path)).toBe("0-0"); + }); + + it("rejects actual deletion failures and completes duplicate deletions without another write", async () => { + const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync(); + await fixture.localDatabase.putDBEntry(optionalFileEntry(`i:${path}`, content)); + storageAccess.setFile(null); + const putRaw = vi + .spyOn(fixture.localDatabase, "putRaw") + .mockRejectedValueOnce(new Error("temporary deletion failure")); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow( + "temporary deletion failure" + ); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + hiddenFileSync.resetLastProcessedFile([path]); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + expect(putRaw).toHaveBeenCalledTimes(2); + }); + + it("recreates an empty hidden file when its database entry is logically deleted", async () => { + const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync(); + await fixture.localDatabase.putDBEntry(optionalFileEntry(`i:${path}`, "")); + await expect(hiddenFileSync.deleteInternalFileOnDatabase(path)).resolves.toBe(true); + const deletedBase = await fixture.localDatabase.getDBEntry( + `i:${path}` as FilePath, + undefined, + false, + true, + true + ); + expect(deletedBase).not.toBe(false); + if (deletedBase) expect(deletedBase.deleted).toBe(true); + if (!deletedBase) throw new Error("The logically deleted test entry could not be loaded"); + vi.spyOn(fixture.localDatabase, "getDBEntry").mockResolvedValueOnce(deletedBase); + + storageAccess.setFile("", 4_000); + const putDBEntry = vi.spyOn(fixture.localDatabase, "putDBEntry"); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + + expect(putDBEntry).toHaveBeenCalledTimes(1); + const recreated = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(recreated).not.toBe(false); + if (recreated) { + expect(recreated.deleted).not.toBe(true); + expect(await getText(recreated)).toBe(""); + } + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.ts index 5b945c69..5abc7027 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.ts @@ -54,6 +54,8 @@ import { EVENT_SETTING_SAVED, eventHub } from "@/common/events.ts"; import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; import type { LiveSyncCore } from "@/main.ts"; import { tryGetFilePath } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc"; +import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; +import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import { configureHiddenFileSyncMode, type ConfigureHiddenFileSyncResult } from "./configureHiddenFileSyncMode.ts"; import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts"; @@ -161,9 +163,7 @@ export class HiddenFileSync extends LiveSyncCommands { // return true; // } private async _everyOnDatabaseInitialized(showNotice: boolean) { - this._fileInfoLastProcessed = await autosaveCache(this.kvDB, "hidden-file-lastProcessed"); - this._databaseInfoLastProcessed = await autosaveCache(this.kvDB, "hidden-file-lastProcessed-database"); - this._fileInfoLastKnown = await autosaveCache(this.kvDB, "hidden-file-lastKnown"); + await this.ensureProcessedCaches(); if (this.isThisModuleEnabled()) { if (this._fileInfoLastProcessed.size == 0 && this._fileInfoLastProcessed.size == 0) { this._log(`No cache found. Performing startup scan.`, LOG_LEVEL_VERBOSE); @@ -249,6 +249,26 @@ export class HiddenFileSync extends LiveSyncCommands { return false; } + async _processOptionalFileEventWithResult( + path: FilePath, + _options?: OptionalFileEventOptions + ): Promise> { + if (!this.isThisModuleEnabled()) return { kind: "not-applicable" }; + if (!(await this.isTargetFile(path))) return { kind: "not-applicable" }; + if (!this._isDatabaseReady()) { + return { kind: "veto", value: "The local database is not ready" }; + } + + await this.ensureProcessedCaches(); + if (!this.isThisModuleEnabled() || !(await this.isTargetFile(path))) { + return { kind: "not-applicable" }; + } + if (!(await this.trackStorageFileModificationStrict(path))) { + return { kind: "not-applicable" }; + } + return { kind: "completed" }; + } + _anyGetOptionalConflictCheckMethod(path: FilePathWithPrefix): Promise { if (isInternalMetadata(path)) { this.queueConflictCheck(path); @@ -317,6 +337,51 @@ export class HiddenFileSync extends LiveSyncCommands { _fileInfoLastKnown!: MapLike; _databaseInfoLastProcessed!: MapLike; + private processedCacheStore: unknown; + private processedCacheStoreReady = false; + private processedCacheLoadStore: unknown; + private processedCacheLoad?: Promise; + + private async ensureProcessedCaches() { + const store = this.kvDB; + if (!store) throw new Error("The Hidden File Sync cache store is unavailable"); + if (this.processedCacheStoreReady && this.processedCacheStore === store) return; + if (this.processedCacheLoad && this.processedCacheLoadStore === store) { + await this.processedCacheLoad; + if (!this.processedCacheStoreReady || this.processedCacheStore !== this.kvDB) { + throw new Error("The Hidden File Sync cache store changed while loading its processing state"); + } + return; + } + + let load!: Promise; + load = Promise.all([ + autosaveCache(store, "hidden-file-lastProcessed"), + autosaveCache(store, "hidden-file-lastProcessed-database"), + autosaveCache(store, "hidden-file-lastKnown"), + ]) + .then(([fileInfoLastProcessed, databaseInfoLastProcessed, fileInfoLastKnown]) => { + if (this.processedCacheLoad !== load || this.kvDB !== store) return; + this._fileInfoLastProcessed = fileInfoLastProcessed; + this._databaseInfoLastProcessed = databaseInfoLastProcessed; + this._fileInfoLastKnown = fileInfoLastKnown; + this.processedCacheStore = store; + this.processedCacheStoreReady = true; + }) + .finally(() => { + if (this.processedCacheLoad === load) { + this.processedCacheLoad = undefined; + this.processedCacheLoadStore = undefined; + } + }); + this.processedCacheLoadStore = store; + this.processedCacheLoad = load; + await load; + if (!this.processedCacheStoreReady || this.processedCacheStore !== this.kvDB) { + throw new Error("The Hidden File Sync cache store changed while loading its processing state"); + } + } + statToKey(stat: UXStat | null) { return `${stat?.mtime ?? 0}-${stat?.size ?? 0}`; } @@ -600,7 +665,6 @@ Offline Changed files: ${processFiles.length}`; this._log(`Hidden file:${path} is changed.`, LOG_LEVEL_VERBOSE); stat = cache.stat; } - this.updateLastProcessedFile(path, stat!); const lastIsNotFound = !lastKey || lastKey.endsWith("-0-0"); const nowIsNotFound = cache.deleted; const type = lastIsNotFound && nowIsNotFound ? "invalid" : nowIsNotFound ? "delete" : "modified"; @@ -651,6 +715,27 @@ Offline Changed files: ${processFiles.length}`; return true; } + private async trackStorageFileModificationStrict(path: FilePath): Promise { + return await this.serializedForEvent(path, async () => { + if (!this.isThisModuleEnabled() || !(await this.isTargetFile(path))) return false; + const stat = await this.core.storageAccess.statHidden(path); + if (stat != null && stat.type != "file") return true; + const key = await this.fileToStatKey(path, stat); + if (this.getLastProcessedFileKey(path) === key) { + this._log(`${path} Already processed.`, LOG_LEVEL_DEBUG); + return true; + } + + const fileInfo = await this.loadFileWithInfo(path); + if (fileInfo.deleted) { + await this.deleteInternalFileOnDatabaseStrict(path); + return true; + } + await this.storeInternalFileToDatabaseStrict(fileInfo); + return true; + }); + } + // --> Conflict processing // Keep one in-flight conflict check per path so repeated sync events do not close the active merge dialogue. @@ -1500,6 +1585,18 @@ Offline Changed files: ${files.length}`; // --> Storage To Database Functions async __loadBaseSaveData(file: FilePath, includeContent = true): Promise { + return await this.loadBaseSaveData(file, includeContent, false); + } + + private async loadBaseSaveDataStrict(file: FilePath, includeContent = true): Promise { + return await this.loadBaseSaveData(file, includeContent, true); + } + + private async loadBaseSaveData( + file: FilePath, + includeContent: boolean, + strict: boolean + ): Promise { const prefixedFileName = addPrefix(file, ICHeader); const id = await this.path2id(prefixedFileName, ICHeader); try { @@ -1525,6 +1622,7 @@ Offline Changed files: ${files.length}`; return old; } } catch (ex) { + if (strict) throw ex; this._log(`Getting base save data failed`); this._log(ex, LOG_LEVEL_VERBOSE); return false; @@ -1552,6 +1650,18 @@ Offline Changed files: ${files.length}`; } async storeInternalFileToDatabase(file: InternalFileInfo | UXFileInfo, forceWrite = false) { + return await this.storeInternalFileToDatabaseCore(file, forceWrite, false); + } + + private async storeInternalFileToDatabaseStrict(file: InternalFileInfo | UXFileInfo, forceWrite = false) { + return await this.storeInternalFileToDatabaseCore(file, forceWrite, true); + } + + private async storeInternalFileToDatabaseCore( + file: InternalFileInfo | UXFileInfo, + forceWrite: boolean, + strict: boolean + ) { const storeFilePath = stripAllPrefixes(file.path); const storageFilePath = file.path; if (await this.services.vault.isIgnoredByIgnoreFile(storageFilePath)) { @@ -1565,9 +1675,11 @@ Offline Changed files: ${files.length}`; if (fileInfo.deleted) { throw new Error(`Hidden file:${storeFilePath} is deleted. This should not be occurred.`); } - const baseData = await this.__loadBaseSaveData(storeFilePath, true); + const baseData = strict + ? await this.loadBaseSaveDataStrict(storeFilePath, true) + : await this.__loadBaseSaveData(storeFilePath, true); if (baseData === false) throw new Error("Failed to load base data"); - if (baseData._rev && !forceWrite) { + if (baseData._rev && !baseData.deleted && !baseData._deleted && !forceWrite) { // Not newly created, we should check the content is actually modified. const isSame = await isDocContentSame(readAsBlob(baseData), fileInfo.body); if (isSame) { @@ -1592,11 +1704,15 @@ Offline Changed files: ${files.length}`; this.updateLastProcessed(storeFilePath, saveData, fileInfo.stat); } const success = ret && ret.ok; + if (!success && strict) { + throw new Error(`Failed to store hidden file in the database: ${storeFilePath}`); + } this._log(`STORAGE --> DB:${storageFilePath}: (hidden) ${success ? "Done" : "Failed"}`); return success; } catch (ex) { this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (strict) throw ex; return false; } }); @@ -1676,6 +1792,14 @@ Offline Changed files: ${files.length}`; } async deleteInternalFileOnDatabase(filenameSrc: FilePath, forceWrite = false) { + return await this.deleteInternalFileOnDatabaseCore(filenameSrc, forceWrite, false); + } + + private async deleteInternalFileOnDatabaseStrict(filenameSrc: FilePath, forceWrite = false) { + return await this.deleteInternalFileOnDatabaseCore(filenameSrc, forceWrite, true); + } + + private async deleteInternalFileOnDatabaseCore(filenameSrc: FilePath, _forceWrite: boolean, strict: boolean) { const storeFilePath = filenameSrc; const storageFilePath = filenameSrc; const displayFileName = filenameSrc; @@ -1686,18 +1810,26 @@ Offline Changed files: ${files.length}`; } return await serialized("file-" + prefixedFileName, async () => { try { - const baseData = await this.__loadBaseSaveData(storeFilePath, false); + const baseData = strict + ? await this.loadBaseSaveDataStrict(storeFilePath, false) + : await this.__loadBaseSaveData(storeFilePath, false); if (baseData === false) throw new Error("Failed to load base data during deleting"); if (baseData._conflicts !== undefined) { for (const conflictRev of baseData._conflicts) { - await this.localDatabase.removeRevision(baseData._id, conflictRev); + const removed = await this.localDatabase.removeRevision(baseData._id, conflictRev); + if (strict && removed === false) { + throw new Error(`Failed to remove conflicted hidden-file revision ${conflictRev}`); + } this._log( `STORAGE -x> DB: ${displayFileName}: (hidden) conflict removed ${baseData._rev} => ${conflictRev}`, LOG_LEVEL_VERBOSE ); } } - if (baseData.deleted) { + const alreadyDeleted = strict + ? !baseData._rev || baseData.deleted || baseData._deleted + : baseData.deleted; + if (alreadyDeleted) { this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) already deleted`, LOG_LEVEL_VERBOSE); this.updateLastProcessedDeletion(storeFilePath, baseData); return true; @@ -1718,11 +1850,13 @@ Offline Changed files: ${files.length}`; return true; } else { this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`); + if (strict) throw new Error(`Failed to delete hidden file from the database: ${storeFilePath}`); return false; } } catch (ex) { this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (strict) throw ex; return false; } }); @@ -2084,6 +2218,9 @@ Offline Changed files: ${files.length}`; // services.databaseEvents.handleOnDatabaseInitialisation(this._everyOnInitializeDatabase.bind(this)); services.appLifecycle.onSettingLoaded.addHandler(this._everyOnloadAfterLoadSettings.bind(this)); services.fileProcessing.processOptionalFileEvent.addHandler(this._anyProcessOptionalFileEvent.bind(this)); + services.fileProcessing.processOptionalFileEventWithResult?.addHandler( + this._processOptionalFileEventWithResult.bind(this) + ); services.conflict.getOptionalConflictCheckMethod.addHandler(this._anyGetOptionalConflictCheckMethod.bind(this)); services.replication.processOptionalSynchroniseResult.addHandler(this._anyProcessOptionalSyncFiles.bind(this)); services.setting.onRealiseSetting.addHandler(this._everyRealizeSettingSyncMode.bind(this)); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 5f04c8c8..59d1f437 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -256,6 +256,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. +`test:e2e:obsidian:optional-file-save-completion` exercises Hidden File Sync and both Customisation Sync formats in one isolated real Obsidian Vault. It holds a local database write to verify that its event stays in the snapshot and that flush waits for the write, then checks unchanged notifications, subsequent edits, and repeated deletions. Injected save failures remain pending and must replay successfully before the next start reaches readiness. This local scenario disables replication and requires no remote service. It is outside `test:e2e:obsidian:local-suite`. + `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`. `test:e2e:obsidian:remote-feature-change` starts real Obsidian with continuous CouchDB replication, then changes the remote version document from generation 12 to generation 13 with an unknown feature. It waits for the control document to reach the local database and the active Replicator to retire, checks that another replication is refused, and verifies that an already accepted Vault note remains intact. After restarting the same Vault, the current remote declaration still blocks finite replication and the actual continuous connection attempt. No KV feature history is involved. It is a focused test outside `test:e2e:obsidian:local-suite`; recovery with a future compatible client remains a separate validation boundary. diff --git a/test/e2e-obsidian/scripts/optional-file-save-completion.ts b/test/e2e-obsidian/scripts/optional-file-save-completion.ts new file mode 100644 index 00000000..f6590eca --- /dev/null +++ b/test/e2e-obsidian/scripts/optional-file-save-completion.ts @@ -0,0 +1,230 @@ +/** Exercises optional file admission, actual save completion, and replay in a real Obsidian Vault. */ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const vault = await createTemporaryVault("obsidian-optional-save-"); + let session: ObsidianLiveSyncSession | undefined; + const observations: unknown[] = []; + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const settings = createE2eCouchDbPluginData( + // These scenarios use only the local database; every replication trigger is disabled. + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "optional-save-local" }, + { + syncInternalFiles: true, + usePluginSync: true, + usePluginSyncV2: true, + deviceAndVaultName: "optional-save-device", + watchInternalFileChanges: true, + syncInternalFilesTargetPatterns: "^\\.obsidian(?:$|/(?:optional-save|snippets)(?:/|$))", + syncInternalFilesIgnorePatterns: "", + autoSweepPlugins: false, + autoSweepPluginsPeriodic: false, + syncInternalFilesInterval: 0, + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + showVerboseLog: true, + } + ); + + async function start(restart = false): Promise { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cliBinary, + vault, + pluginStartup: restart ? "natural" : undefined, + pluginData: restart ? undefined : settings, + localStorageEntries: restart ? undefined : createE2eObsidianDeviceLocalState(vault.name), + lifecycle: { + beforeLaunch: async () => { + const registryPath = join(vault.userDataPath, "obsidian.json"); + const registry = JSON.parse(await readFile(registryPath, "utf8")); + registry.updateDisabled = true; + await writeFile(registryPath, JSON.stringify(registry)); + }, + }, + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + } + + async function evaluate(body: string): Promise { + if (!session) throw new Error("The Obsidian session is not running"); + return await evalObsidianJson( + cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + const manager=core.serviceModules.storageAccess.vaultManager; + const config=core.getAddOn('ConfigSync'); + const sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms)); + const require=(condition,message)=>{if(!condition)throw new Error(message);}; + const waitFor=async (predicate,message)=>{ + const until=Date.now()+10000; + while(!(await predicate())) { + if(Date.now()>until)throw new Error(message); + await sleep(20); + } + }; + const pending=async path=>(await manager.adapter.persistence.loadSnapshot()??[]) + .some(event=>event.args?.file?.path===path); + const entry=async path=>await core.localDatabase.getDBEntry(path,undefined,false,true); + const contents=doc=>Array.isArray(doc.data)?doc.data.join(''):doc.data; + const represents=(doc,content)=>doc&& !doc.deleted && + (contents(doc).includes(content)||contents(doc).includes(btoa(content))); + const notify=async path=>{ + await manager._watchVaultRawEvents(path); + await sleep(30); + }; + ${body} + })()`, + session.cliEnv + ); + } + + try { + await start(); + await evaluate(` + require(core.services.fileProcessing.processOptionalFileEventWithResult?.hasHandlers, + 'Typed optional providers were not registered'); + core.services.setting.setDeviceAndVaultName('optional-save-device'); + await core.services.setting.saveSettingData(); + for(const path of ['.obsidian/optional-save','.obsidian/snippets']) { + if(!(await app.vault.adapter.exists(path)))await app.vault.adapter.mkdir(path); + } + return JSON.stringify({ready:true}); + `); + + for (const mode of ["hidden", "config-v1", "config-v2"]) { + observations.push( + await evaluate(` + const mode=${JSON.stringify(mode)}; + core.settings.usePluginSyncV2=mode!=='config-v1'; + await core.services.setting.saveSettingData(); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Initial flush failed'); + const path=mode==='hidden'?'.obsidian/optional-save/content.json': + '.obsidian/snippets/optional-save-'+mode+'.css'; + const databasePath=mode==='hidden'?'i:'+path:mode==='config-v1'? + config.filenameToUnifiedKey(path):config.filenameWithUnifiedKey(path); + const content='optional save '+mode+' initial'; + const original=core.localDatabase.putDBEntry.bind(core.localDatabase); + let release; + const gate=new Promise(resolve=>{release=resolve;}); + let started=false; + core.localDatabase.putDBEntry=async data=>{ + if(data.path===databasePath){started=true;await gate;} + return await original(data); + }; + try { + await app.vault.adapter.write(path,content); + await notify(path); + await waitFor(()=>started,'The optional database save did not start: '+mode); + require(await pending(path),'The in-flight optional save was missing from the snapshot: '+mode); + let flushed=false; + const flush=core.services.fileProcessing.commitPendingFileEvents().then(result=>{flushed=true;return result;}); + await sleep(30); + require(!flushed,'Flush returned before the actual database save: '+mode); + release(); + require(await flush,'Flush failed after the write was released: '+mode); + } finally { + release(); + core.localDatabase.putDBEntry=original; + } + const first=await entry(databasePath); + require(represents(first,content),'The completed save did not contain the expected content: '+mode); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Unchanged notification failed: '+mode); + require((await entry(databasePath))._rev===first._rev,'Unchanged notification added a revision: '+mode); + const updated=content+' with a later edit'; + await app.vault.adapter.write(path,updated); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Later edit was blocked: '+mode); + require(represents(await entry(databasePath),updated),'Later edit was not saved: '+mode); + await app.vault.adapter.remove(path); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Deletion failed: '+mode); + const deleted=await core.localDatabase.getDBEntryMeta(databasePath,undefined,true); + require(deleted&&deleted.deleted,'Deletion was not represented: '+mode); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Repeated deletion failed: '+mode); + require((await core.localDatabase.getDBEntryMeta(databasePath,undefined,true))._rev===deleted._rev, + 'Repeated deletion added a revision: '+mode); + return JSON.stringify({mode,saveWaited:true,unchangedPreservedRevision:true,laterEditSaved:true,deletionReplayed:true}); + `) + ); + } + + const replay = [ + { path: ".obsidian/optional-save/replay.json", content: "hidden save retained across restart" }, + { + path: ".obsidian/snippets/optional-save-replay.css", + content: "customisation save retained across restart", + }, + ]; + observations.push( + await evaluate(` + const fixtures=${JSON.stringify(replay)}; + core.settings.usePluginSyncV2=true; + await core.services.setting.saveSettingData(); + const failures=new Set(fixtures.map(({path})=>path.endsWith('.json')?'i:'+path:config.filenameWithUnifiedKey(path))); + const original=core.localDatabase.putDBEntry.bind(core.localDatabase); + let failed=0; + core.localDatabase.putDBEntry=async data=>{ + if(failures.delete(data.path)){failed++;return false;} + return await original(data); + }; + try { + for(const {path,content} of fixtures) { + await app.vault.adapter.write(path,content); + await notify(path); + } + await waitFor(()=>failed===fixtures.length,'Injected optional failures were not reached'); + require(!(await core.services.fileProcessing.commitPendingFileEvents()),'Failed saves were acknowledged'); + for(const {path} of fixtures)require(await pending(path),'Failed save was lost: '+path); + } finally { + core.localDatabase.putDBEntry=original; + } + return JSON.stringify({failedSavesRetained:failed}); + `) + ); + await session!.app.stop(); + session = undefined; + await start(true); + observations.push( + await evaluate(` + const fixtures=${JSON.stringify(replay)}; + for(const {path,content} of fixtures) { + const databasePath=path.endsWith('.json')?'i:'+path:config.filenameWithUnifiedKey(path); + require(represents(await entry(databasePath),content),'Start-up did not recover '+path); + require(!(await pending(path)),'Recovered save is still pending: '+path); + } + return JSON.stringify({replayedBeforeReadiness:fixtures.length,ready:core.services.appLifecycle.isReady()}); + `) + ); + console.log(JSON.stringify({ ok: true, bundleHash, observations }, null, 2)); + } finally { + await session?.app.stop(); + await vault.dispose(); + } +} + +await main(); diff --git a/updates.md b/updates.md index 9467ef89..94859d63 100644 --- a/updates.md +++ b/updates.md @@ -24,6 +24,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - Local changes remain available for recovery while their database save is running, including when the application stops after a batch delay ends. - Manual synchronisation and settings application now wait for the accepted local saves to finish and report failures. Pending events retain the existing snapshot format. + - Hidden File Sync and Customisation Sync now report completion after saving, including delayed customisation updates. Failed saves remain pending, while unchanged notifications allow later edits to proceed. - Start-up reconciliation checks content when modification times are equal, so an edit within the same timestamp interval is not mistaken for an unchanged file. - Existing revision provenance preserves the correct edit branch. Unreadable or continually changing content remains unresolved rather than being reported as successfully reconciled. - We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) From b449dfd2bb8e2863f8ed396b931f57acfd9f7c0b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 26 Oct 2026 12:18:52 +0000 Subject: [PATCH 24/28] Prepare 1.0.41 notes, dependencies, and recovery coverage Align release metadata and audited development dependencies, and consolidate release notes with contributor acknowledgements. Cover upgrades, start-up integrity, text reflection, and interrupted saves in real Obsidian. Document the accepted provenance persistence interval: identical replayed content converges, safe merge retains the later remote content and ancestry, and overlapping edits remain available for review. --- ...local_file_save_completion_and_batching.md | 39 +- docs/specs_conflict_resolution.md | 43 ++ manifest.json | 2 +- package-lock.json | 409 ++++++++---------- package.json | 13 +- src/apps/cli/package.json | 6 +- src/apps/webapp/package.json | 4 +- src/apps/webpeer/package.json | 4 +- test/e2e-obsidian/README.md | 14 +- .../scripts/issue1207-livesync-batch.ts | 284 +++++++++++- test/e2e-obsidian/scripts/run-focused.ts | 3 + .../e2e-obsidian/scripts/startup-integrity.ts | 396 +++++++++++++++++ .../scripts/third-party-chunks.ts | 269 ++++++++++++ .../scripts/upgrade-livesync-batching.ts | 312 +++++++++++++ updates.md | 74 ++-- versions.json | 3 +- 16 files changed, 1583 insertions(+), 292 deletions(-) create mode 100644 test/e2e-obsidian/scripts/startup-integrity.ts create mode 100644 test/e2e-obsidian/scripts/third-party-chunks.ts create mode 100644 test/e2e-obsidian/scripts/upgrade-livesync-batching.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md index 27e6e4f6..33ee9b8c 100644 --- a/docs/adr/2026_10_local_file_save_completion_and_batching.md +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -143,9 +143,30 @@ policy-approved skip before the manager can acknowledge either safely. A crash after the database write but before snapshot removal can replay an operation. Replay must check current storage and revision provenance, retain -the existing protection against stale restored events, and avoid duplicate -revisions or deletion of a newer same-path file. It must be safe to interrupt -replay again. Existing saved snapshots remain readable. +the existing protection against stale restored events, and never delete a +newer same-path file. Once both the database write and provenance write have +completed, replay must not add another revision. If the database write +succeeded before provenance was recorded, replay may first create an identical +sibling even when no remote descendant has arrived. In the tested case +without a remote edit, replay creates that sibling. Duplicate handling then +tombstones one of the identical branches and converges to one live leaf with +the exact saved content. +Another restart must add no revision. Replay must remain safe to interrupt; +existing saved snapshots remain readable. + +The database write and the device-local provenance write are separate, +sequential operations. A stop between them can leave the previous provenance +record in place. This pre-provenance gap is the known exception to the +no-extra-revision result after both writes complete: replay can first create a +duplicate sibling; one of the identical branches is then tombstoned while +one live leaf retains the saved bytes when no remote descendant has arrived. If continuous +replication advances the local database during the gap, replay can instead +expose the local content as a sibling of that descendant. The [conflict +resolution specification](../specs_conflict_resolution.md#known-potentially-problematic-behaviour) +records the required mergeable and manual-resolution outcomes. The per-file +lock does not fence replication writes. The interruption window is expected +to be very rare, but its field frequency is unmeasured; this decision adds no +transaction or second journal to close the interval. Boolean failure, an exception, and an unavailable comparison are not successful completion. Keep unresolved work recoverable and report the failure through the @@ -202,6 +223,14 @@ outcomes through the ordinary non-forced file handler and its provenance rules: | Vault content has no provable origin and matches no current live leaf | Preserve it according to the existing independent-branch rules. Do not attach it to an arbitrary database winner. | | Required content or provenance cannot be read, or state changes during comparison | Retain unresolved work and report or re-evaluate it. Do not mark it reconciled merely because the timestamp comparison is `EVEN`. | +The table describes stable-state reconciliation, not atomic database and +provenance persistence. Replay in the pre-provenance gap may create an +identical sibling; with no remote descendant, one of the identical branches +is tombstoned, leaving one live leaf with the exact saved content. If replication has +already added a descendant, preserve its exact content through an existing +safe merge, or retain both branches for manual resolution. See [Known +potentially problematic behaviour](../specs_conflict_resolution.md#known-potentially-problematic-behaviour). + Revalidate the relevant file and database state before acting on a comparison. Reuse the existing content and provenance checks; avoid a separate conflict algorithm in the scanner. Metadata continues to reference Chunks containing the @@ -333,7 +362,9 @@ matrix is: | Scenario | Required observation | | -------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | | Stop after dequeue, timer release, or during a database write | Pending work remains recoverable; restart eventually represents the saved Vault edit. | -| Database write succeeds before snapshot cleanup; restart twice | Replay creates no duplicate revision and preserves later edits. | +| Database and provenance writes are complete before replay; restart twice | Replay adds no extra revision. | +| Stop after the database write but before provenance; no remote descendant; replay twice | The tested replay creates an identical sibling and tombstones one of the two branches; one live leaf retains exact saved content, and the second restart adds no revision. | +| A remote descendant arrives during the pre-provenance gap | The mergeable case preserves its exact content and ancestry; the overlapping case retains both bodies for manual resolution. | | Old and new events share a path; snapshot writes overlap | Completing old work cannot remove new work or regress the durable snapshot. | | Handler returns false, throws, or snapshot persistence fails | Flush reports failure; pending work is retained; unrelated recoverable work can continue. | | Buffered or in-progress work exists with no waiting timer | Flush still waits for actual completion. | diff --git a/docs/specs_conflict_resolution.md b/docs/specs_conflict_resolution.md index aabf5807..6f29fedb 100644 --- a/docs/specs_conflict_resolution.md +++ b/docs/specs_conflict_resolution.md @@ -166,6 +166,49 @@ For ordinary saves and incoming reflection, a missing or unreadable recorded bas Provenance keys follow the selected [Unicode path identity](design_docs/unicode_path_identity.md#state-which-follows-the-selected-identity). In NFC mode, equivalent path spellings address the same record; the content evidence continues to describe file bytes, independently of that path spelling. +## Known potentially problematic behaviour + +Saving a local file writes its revision to the local database, then writes +device-local provenance in a separate operation. The writes are not atomic. +If the process stops between them, provenance can still identify the revision +which preceded the local edit. Continuous replication which was already +running can add a remote descendant to the local database during this +interval. The per-file lock covers file handling and Vault reflection; it +does not block raw replication writes to the database. + +For example, local content `2-B` may have been saved as a child of +displayed revision `1`, while provenance still identifies `1`. +On restart, replay of the pending save can treat `2-B` as an edit of +`1` and write sibling `2-D`. The tested case demonstrates that this +pre-provenance gap alone can produce the duplicate. Without a remote +descendant, replay creates this sibling; duplicate handling tombstones one +of the identical branches and converges to one live leaf whose exact content +matches the saved local file. A second restart must add no revision. + +If replication has already added descendant `3-C` during the gap, replay +can expose an additional branch. When the revisions have a complete shared +base and their text changes can be combined unambiguously, sensible merge can retain the +exact content of the incoming descendant. In the tested fixture, merge +produced revision `4` as a child of `3-C`; its file bytes were exactly +those of `3-C`, `3-C` remained in its ancestry, and sibling +`2-D` was tombstoned. The acceptance condition is retaining +`3-C`'s exact content; it need not remain the current leaf. Both Vaults +converged to those bytes. This observed outcome is limited to that mergeable +fixture and is not a universal no-data-loss guarantee. + +When changes cannot be combined safely, as in the tested competing whole-line +replacements, sensible merge does not combine the bodies. Both versions remain available for the existing +manual conflict dialogue, even when sensible merge is enabled. + +An interruption in the database/provenance interval is expected to be very +rare, especially when combined with a remote descendant arriving in that +interval; the field frequency is unmeasured. Start-up replays restored work, +scans, and commits pending local file work before marking the plug-in ready; +ordinary replication proceeds after that ready boundary. This does not claim +remote synchronisation before start-up readiness. The accepted design adds no +cross-store transaction, second journal, or new persistent state to close the +interval. + ## Operations while a conflict exists - Editing a file writes a child of its recorded or uniquely reconstructed displayed revision. diff --git a/manifest.json b/manifest.json index 267d35b5..efe56799 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.40", + "version": "1.0.41", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index e17dd5e7..5393a72f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.40", + "version": "1.0.41", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.40", + "version": "1.0.41", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -55,7 +55,7 @@ "@types/pouchdb-replication": "^6.4.7", "@types/transform-pouch": "^1.0.6", "@typescript-eslint/parser": "8.69.0", - "@vitest/coverage-v8": "^4.1.11", + "@vitest/coverage-v8": "^4.1.12", "@vrtmrz/obsidian-test-session": "0.2.6", "dotenv-cli": "^11.0.0", "esbuild": "0.28.1", @@ -90,8 +90,8 @@ "tsx": "^4.21.0", "typescript": "6.0.3", "typescript-eslint": "8.69.0", - "vite": "^8.0.16", - "vitest": "^4.1.11", + "vite": "^8.1.6", + "vitest": "^4.1.12", "yaml": "^2.8.2" } }, @@ -1945,22 +1945,25 @@ } }, "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", - "integrity": "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.6.tgz", + "integrity": "sha512-mRyDmEw9/fCDXCGfL1sbgIw/GUK0PVxRW4YwgPN61F3LQ+fSyoBFarJVzgGyhF6kRzEQZ0FejDqm4WLXSeqQmw==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@tybys/wasm-util": "^0.10.2" + "@tybys/wasm-util": "^0.10.3" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" }, "funding": { "type": "github", "url": "https://github.com/sponsors/Brooooooklyn" }, "peerDependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1" + "@emnapi/core": "^1.7.1 || 2.0.0-beta.2", + "@emnapi/runtime": "^1.7.1 || 2.0.0-beta.2" } }, "node_modules/@noble/curves": { @@ -2012,9 +2015,9 @@ "license": "MIT" }, "node_modules/@oxc-project/types": { - "version": "0.133.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", - "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", + "version": "0.139.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", + "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", "dev": true, "license": "MIT", "funding": { @@ -2222,9 +2225,9 @@ } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", - "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", + "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", "cpu": [ "arm64" ], @@ -2239,9 +2242,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", - "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", + "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", "cpu": [ "arm64" ], @@ -2256,9 +2259,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", - "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", + "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", "cpu": [ "x64" ], @@ -2273,9 +2276,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", - "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", + "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", "cpu": [ "x64" ], @@ -2290,9 +2293,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", - "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", + "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", "cpu": [ "arm" ], @@ -2307,13 +2310,16 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", - "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", + "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2324,13 +2330,16 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", - "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", + "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2341,13 +2350,16 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", - "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", + "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2358,13 +2370,16 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", - "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", + "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2375,13 +2390,16 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", - "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", + "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2392,13 +2410,16 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", - "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", + "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2409,9 +2430,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", - "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", + "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", "cpu": [ "arm64" ], @@ -2426,9 +2447,9 @@ } }, "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", - "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", + "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", "cpu": [ "wasm32" ], @@ -2436,52 +2457,18 @@ "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.10.0", - "@emnapi/runtime": "1.10.0", - "@napi-rs/wasm-runtime": "^1.1.4" + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" }, "engines": { "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", - "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", + "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", "cpu": [ "arm64" ], @@ -2496,9 +2483,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", - "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", + "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", "cpu": [ "x64" ], @@ -3347,9 +3334,9 @@ "license": "MIT" }, "node_modules/@tybys/wasm-util": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", - "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", + "integrity": "sha512-W3c4gRigFS0T/Ma4qIYF3GDAc5AQdHb1yL5znJT1Zv1YaD9Kitx656wBjvr19qbiosmZT8lWDM5BEMynUqX65A==", "dev": true, "license": "MIT", "optional": true, @@ -3963,14 +3950,14 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.11.tgz", - "integrity": "sha512-8MVGEFnJIcdGjcbfKmeq8z0pZHH0JlVtoVZH9Q/qwUp6wyFnEJUBMrw9DCaj+ra3vShGmhavjalMIhPNxZAUcw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.12.tgz", + "integrity": "sha512-6S7/iVknKglLAxMFxQGNppBI0iqJS2NgM6FBxavitj7KaDq/vywOWGf6dGu7F2L1Qy0G6S3uTp7aGshHrzK8JA==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", @@ -3984,8 +3971,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.1.11", - "vitest": "4.1.11" + "@vitest/browser": "4.1.12", + "vitest": "4.1.12" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -3994,16 +3981,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", - "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.12.tgz", + "integrity": "sha512-vFlTt+YEu9WBpNjdGayNmlMVzrWPi1+sv853/7t2FU7hpO5gAFBKHzQmkCq5Xjpr+evegF2994LQT3hF3mRlqQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -4012,13 +3999,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", - "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.12.tgz", + "integrity": "sha512-fKShNpKZzupWZuY0z7egydvL0sqW4INEZBEExMBMAa6izlENEVHwoXgdaTCaBZ5fwlr4xf2o1wXC83n9RpO+NA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.11", + "@vitest/spy": "4.1.12", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -4039,9 +4026,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", - "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.12.tgz", + "integrity": "sha512-XW9O8+Mxx4VpCtkYdMJLZQQwsVryP9wk7cPKDxtXFfwSak6yRh5+2fep/Y0XF/ElBvJHq8bcYdi6aeuai0opZA==", "dev": true, "license": "MIT", "dependencies": { @@ -4052,13 +4039,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", - "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.12.tgz", + "integrity": "sha512-NKlVKCr5QnCe8jijdMnmkpHHU+WTqiLFR6G0pHji9uRjXK9xJl1k1nTw6F3pOOj2PDcsDt2GKQMI7wIsaT491w==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "pathe": "^2.0.3" }, "funding": { @@ -4066,14 +4053,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", - "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.12.tgz", + "integrity": "sha512-ko1y1Qgb/slzQ9mr+b89Cuqud8WG1TKtsPD8r7ZUBdvVOGt85Af2HwCqHujrhwJq95hs/9RM7fcAPg57o1vlzw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/pretty-format": "4.1.12", + "@vitest/utils": "4.1.12", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -4082,9 +4069,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", - "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.12.tgz", + "integrity": "sha512-RGq9x4JNGNTZHEJxvN0oSiJjeSCExm0n2gZKKJZHY9cX5AVA6M7VJILBT3ZJ/tJ/0J9uWT7fETV9/btoi//vSg==", "dev": true, "license": "MIT", "funding": { @@ -4092,13 +4079,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", - "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.12.tgz", + "integrity": "sha512-Q3awXrwOfEV+qc3WmO5ONyE0yO7Q1gQ9/oyHxGLZ/pl7N3NO/ZztqQwKLuwMkV+CA1JYSIeO3gkHnRDIo4AVaA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", + "@vitest/pretty-format": "4.1.12", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -6377,9 +6364,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", - "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", "dev": true, "funding": [ { @@ -8814,6 +8801,19 @@ "dev": true, "license": "ISC" }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -9586,13 +9586,13 @@ } }, "node_modules/rolldown": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", - "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", + "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.133.0", + "@oxc-project/types": "=0.139.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -9602,21 +9602,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.3", - "@rolldown/binding-darwin-arm64": "1.0.3", - "@rolldown/binding-darwin-x64": "1.0.3", - "@rolldown/binding-freebsd-x64": "1.0.3", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", - "@rolldown/binding-linux-arm64-gnu": "1.0.3", - "@rolldown/binding-linux-arm64-musl": "1.0.3", - "@rolldown/binding-linux-ppc64-gnu": "1.0.3", - "@rolldown/binding-linux-s390x-gnu": "1.0.3", - "@rolldown/binding-linux-x64-gnu": "1.0.3", - "@rolldown/binding-linux-x64-musl": "1.0.3", - "@rolldown/binding-openharmony-arm64": "1.0.3", - "@rolldown/binding-wasm32-wasi": "1.0.3", - "@rolldown/binding-win32-arm64-msvc": "1.0.3", - "@rolldown/binding-win32-x64-msvc": "1.0.3" + "@rolldown/binding-android-arm64": "1.1.5", + "@rolldown/binding-darwin-arm64": "1.1.5", + "@rolldown/binding-darwin-x64": "1.1.5", + "@rolldown/binding-freebsd-x64": "1.1.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", + "@rolldown/binding-linux-arm64-gnu": "1.1.5", + "@rolldown/binding-linux-arm64-musl": "1.1.5", + "@rolldown/binding-linux-ppc64-gnu": "1.1.5", + "@rolldown/binding-linux-s390x-gnu": "1.1.5", + "@rolldown/binding-linux-x64-gnu": "1.1.5", + "@rolldown/binding-linux-x64-musl": "1.1.5", + "@rolldown/binding-openharmony-arm64": "1.1.5", + "@rolldown/binding-wasm32-wasi": "1.1.5", + "@rolldown/binding-win32-arm64-msvc": "1.1.5", + "@rolldown/binding-win32-x64-msvc": "1.1.5" } }, "node_modules/sade": { @@ -10465,19 +10465,6 @@ } } }, - "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/tinyrainbow": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", @@ -10847,16 +10834,16 @@ } }, "node_modules/vite": { - "version": "8.0.16", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", - "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", + "version": "8.1.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.6.tgz", + "integrity": "sha512-aRAHIxx5SW5zc1sMqFwJU/t5qaFfXb5Hj0J7W0VXihu/X99+346tz6080tO1T1xVIJORX/b3zVTXGFqUXh35uw==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.32.0", - "picomatch": "^4.0.4", - "postcss": "^8.5.15", - "rolldown": "1.0.3", + "picomatch": "^4.0.5", + "postcss": "^8.5.17", + "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "bin": { @@ -10873,7 +10860,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.1.18", + "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -10939,19 +10926,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/vite/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/vitefu": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.2.tgz", @@ -10973,19 +10947,19 @@ } }, "node_modules/vitest": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", - "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.12.tgz", + "integrity": "sha512-5eWdz2yb3ajxWF3C8C4TjkXl7FGm3n2E/VlazALrQHzK7nYMSGFnCZto0bbPNl77AwkOZVLk0YnBAejUu0NiWA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.11", - "@vitest/mocker": "4.1.11", - "@vitest/pretty-format": "4.1.11", - "@vitest/runner": "4.1.11", - "@vitest/snapshot": "4.1.11", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/expect": "4.1.12", + "@vitest/mocker": "4.1.12", + "@vitest/pretty-format": "4.1.12", + "@vitest/runner": "4.1.12", + "@vitest/snapshot": "4.1.12", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -11013,12 +10987,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.11", - "@vitest/browser-preview": "4.1.11", - "@vitest/browser-webdriverio": "4.1.11", - "@vitest/coverage-istanbul": "4.1.11", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/browser-playwright": "4.1.12", + "@vitest/browser-preview": "4.1.12", + "@vitest/browser-webdriverio": "^4.1.11", + "@vitest/coverage-istanbul": "4.1.12", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" @@ -11069,19 +11043,6 @@ "dev": true, "license": "MIT" }, - "node_modules/vitest/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/vuvuzela": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/vuvuzela/-/vuvuzela-1.0.3.tgz", @@ -11408,7 +11369,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.40-cli", + "version": "1.0.41-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11427,13 +11388,13 @@ }, "devDependencies": { "typescript": "6.0.3", - "vite": "^8.0.16", - "vitest": "^4.1.11" + "vite": "^8.1.6", + "vitest": "^4.1.12" } }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.40-webapp", + "version": "1.0.41-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11441,11 +11402,11 @@ "@sveltejs/vite-plugin-svelte": "^7.1.2", "svelte": "5.56.3", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } }, "src/apps/webpeer": { - "version": "1.0.40-webpeer", + "version": "1.0.41-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11456,7 +11417,7 @@ "svelte": "5.56.3", "svelte-check": "^4.6.0", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } } } diff --git a/package.json b/package.json index 1c1bea1a..d5927927 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.40", + "version": "1.0.41", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", @@ -127,7 +127,10 @@ "test:docker-all:start": "npm run test:docker-all:up && sleep 5 && npm run test:docker-all:init", "test:docker-all:stop": "npm run test:docker-all:down", "update-workspaces": "node update-workspaces.mjs", - "version": "node version-bump.mjs && node update-workspaces.mjs && npm run pretty:json && git add manifest.json versions.json src/apps/cli/package.json src/apps/webpeer/package.json src/apps/webapp/package.json" + "version": "node version-bump.mjs && node update-workspaces.mjs && npm run pretty:json && git add manifest.json versions.json src/apps/cli/package.json src/apps/webpeer/package.json src/apps/webapp/package.json", + "test:e2e:obsidian:startup-integrity": "tsx test/e2e-obsidian/scripts/startup-integrity.ts", + "test:e2e:obsidian:third-party-chunks": "tsx test/e2e-obsidian/scripts/third-party-chunks.ts", + "test:e2e:obsidian:upgrade-livesync-batching": "tsx test/e2e-obsidian/scripts/upgrade-livesync-batching.ts" }, "keywords": [], "author": "vorotamoroz", @@ -151,7 +154,7 @@ "@types/pouchdb-replication": "^6.4.7", "@types/transform-pouch": "^1.0.6", "@typescript-eslint/parser": "8.69.0", - "@vitest/coverage-v8": "^4.1.11", + "@vitest/coverage-v8": "^4.1.12", "@vrtmrz/obsidian-test-session": "0.2.6", "dotenv-cli": "^11.0.0", "esbuild": "0.28.1", @@ -186,8 +189,8 @@ "tsx": "^4.21.0", "typescript": "6.0.3", "typescript-eslint": "8.69.0", - "vite": "^8.0.16", - "vitest": "^4.1.11", + "vite": "^8.1.6", + "vitest": "^4.1.12", "yaml": "^2.8.2" }, "dependencies": { diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index e8d7ca9c..7f73f7f1 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.40-cli", + "version": "1.0.41-cli", "main": "dist/index.cjs", "type": "module", "scripts": { @@ -52,7 +52,7 @@ }, "devDependencies": { "typescript": "6.0.3", - "vite": "^8.0.16", - "vitest": "^4.1.11" + "vite": "^8.1.6", + "vitest": "^4.1.12" } } diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index 487af8bf..d0f4f5ce 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.40-webapp", + "version": "1.0.41-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { @@ -21,6 +21,6 @@ "@sveltejs/vite-plugin-svelte": "^7.1.2", "svelte": "5.56.3", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } } diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index 22594247..c7654ed5 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.40-webpeer", + "version": "1.0.41-webpeer", "type": "module", "scripts": { "dev": "vite", @@ -24,6 +24,6 @@ "svelte": "5.56.3", "svelte-check": "^4.6.0", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } } diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index fd543794..ae4790b3 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -98,6 +98,10 @@ The same scenario checks the five actions in Reporting Tools, the separate Check `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. +`startup-integrity` uses the registered Hatch integrity-check event with real Obsidian storage and a local database. It verifies that healthy files are inspected without reading their contents, that a controlled stale stat followed by matching real bytes does not produce a repair dialogue, and that a changed final stat leaves the check incomplete until a settled retry. The stat variations are injected; this scenario exercises the checker used during start-up rather than restarting the process. Natural configured start-up remains covered by `startup-scan`. + +`third-party-chunks` reflects synthetic externally generated text through the installed Commonlib and the real Obsidian file handler. It supplies a split surrogate pair at the file-access read boundary, with size checking enabled, then verifies exact Vault and on-disk text, unchanged Metadata, and matching persisted provenance. Complete-pair text and a binary attachment provide controls. The underlying Metadata and Chunks are real local database entries; this does not establish that CouchDB or a particular third-party application generates this chunk layout. No remote service is required. + `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. The principal entry points are: @@ -265,7 +269,13 @@ The workflow also opens the Customisation Sync dialogue. `--case=visibility` che `--case=automatic` separately checks that choosing **Automatic** while Hidden File Sync is disabled displays a Notice without changing modes, files, Metadata revisions, or processed markers. The fixture starts with a saved Automatic selection and requires it to survive start-up and both rejected actions. It also checks disabling the feature after opening **Initial Action**. After enablement, selecting a whole plug-in must initialise both **MAIN** and **DATA**, and transfer their files to the other Vault. Separate temporary Vaults exercise **Overwrite Remote**, **Overwrite Local**, and **Use newer**. The latter two start with different local and database contents; deterministic per-file modification times require **Use newer** to keep the newer database MAIN content and the newer local DATA content. -`test:e2e:obsidian:livesync-batch` uses two real Obsidian Vaults and a disposable local CouchDB to compare batching disabled, the older `batchSave` choice alone, and the explicit LiveSync batching choice. It covers repeated edits in both directions, template-like writes, deletion and recreation, incoming changes, maximum delay, concurrent paths, manual flush, and restart recovery before and after a database write. Restarts use the saved profile with natural plug-in start-up. `ISSUE1207_CASE_FILTER` selects case names; `ISSUE1207_RESTART_BUCKET=same|exact|different` exercises interrupted processing at the selected timestamp condition. `ISSUE1207_RESTART_SAVE_AFTER_EDIT=true` separately checks a settings save after the pending edit. Use `npm run test:e2e:obsidian:focused -- livesync-batch` to build and run the default matrix. This scenario requires a localhost CouchDB and is outside the complete local suite. +`test:e2e:obsidian:livesync-batch` uses two real Obsidian Vaults and a disposable local CouchDB to compare batching disabled, the older `batchSave` choice alone, and the explicit LiveSync batching choice. It covers repeated edits in both directions, template-like writes, deletion and recreation, incoming changes, maximum delay, concurrent paths, manual flush, and restart recovery before and after a local database write. Restarts use the saved profile with natural plug-in start-up. + +`ISSUE1207_CASE_FILTER` selects case names; `ISSUE1207_RESTART_BUCKET=same|exact|different` exercises interrupted processing at the selected timestamp condition. `ISSUE1207_RESTART_SAVE_AFTER_EDIT=true` separately checks a settings save after the pending edit. + +`ISSUE1207_RESTART_BEFORE_PROVENANCE=true` deliberately stops after the local database write and before device-local provenance is recorded. The case without a remote edit demonstrates replay creating an identical sibling and tombstoning one of the two branches; it must converge to one live leaf with exact saved content, and a second restart must add no revision. + +`ISSUE1207_RESTART_REMOTE_EDIT=mergeable|conflicting` requires `ISSUE1207_RESTART_BEFORE_PROVENANCE=true`. These cases persist the enabled test settings before editing so continuous replication remains active, receive a remote descendant before stopping, and restart the same profile. Both remote cases enable sensible merge. The pause is deliberately injected to observe the ordering; it is not a measure of field frequency. The `mergeable` case requires the final bytes to equal the incoming descendant exactly, with that revision retained in ancestry and further restart stable. The `conflicting` case uses a whole-line replacement and requires both bodies to remain for the manual conflict dialogue. The cases are named `enabled-before-provenance-mergeable-restart` and `enabled-before-provenance-conflicting-restart`. Use `npm run test:e2e:obsidian:focused -- livesync-batch` to build and run the default matrix. This scenario requires a localhost CouchDB and is outside the complete local suite. `test:e2e:obsidian:optional-file-save-completion` exercises Hidden File Sync and both Customisation Sync formats in one isolated real Obsidian Vault. It holds a local database write to verify that its event stays in the snapshot and that flush waits for the write, then checks unchanged notifications, subsequent edits, and repeated deletions. Injected save failures remain pending and must replay successfully before the next start reaches readiness. This local scenario disables replication and requires no remote service. It is outside `test:e2e:obsidian:local-suite`. @@ -281,6 +291,8 @@ The workflow also opens the Customisation Sync dialogue. `--case=visibility` che `test:e2e:obsidian:setting-markdown-roundtrip` generates Markdown with and without credentials through the real export command, copies each original file into a separate fresh Vault, and imports it through **Parse setting file** and the visible **Apply settings** dialogue. Each receiver starts with default plug-in settings and no connection profiles, with its own configuration encryption passphrase provided independently of the source through localStorage. The scenario verifies ordinary settings, conditional credentials and complete profiles, encrypted local persistence, and restoration after restarting the same Vault and profile without reseeding settings or localStorage. Imports without credentials may create new profiles through the existing legacy migration; those profiles must contain none of the omitted source credentials. This scenario requires no remote service and is included in the focused wrapper and local suite. +`test:e2e:obsidian:upgrade-livesync-batching` verifies the introduction of the separate LiveSync batching choice using the pinned, published 1.0.40 artefacts. It first saves and restarts the old release with legacy batching enabled and LiveSync mode selected. Replacing only the plug-in artefacts must retain the Vault, local database, device identity, remote profile, and existing Metadata revision, while leaving the new LiveSync batching choice disabled. The upgraded Vault and an independent current-version Vault exchange notes through CouchDB. Natural restarts retain both the disabled LiveSync choice and the original batching choice when switching to periodic mode. The source release is downloaded into the verified release cache; `E2E_LIVESYNC_TARGET_ARTIFACT_ROOT` selects an exact target build. This tests the update behaviour, while installation through BRAT remains a separate acceptance step. + `test:e2e:obsidian:upgrade-from-stable` is the release-acceptance upgrade workflow. It installs the exact published 0.25.83 artefacts into an isolated Vault, verifies their pinned SHA-256 values, and then replaces only the plug-in artefacts with the current target while retaining the same Vault and isolated Obsidian profile. The first run downloads the old release into the ignored `_testdata/releases` cache; every later run verifies the cached bytes before use. The workflow first exercises a non-empty legacy settings document which has no `isConfigured` or file-name case value. It verifies that 0.25.83 treats a default-equivalent document as unconfigured. That release can persist the inferred boolean during a later, unrelated settings-save event, so the runner accepts either an absent value or the inferred `false` on disk, then restores the same minimal pre-flag document deliberately before installing 1.0. The target independently proves its direct migration: the Vault remains unconfigured instead of receiving new-Vault recommendations, case-insensitive handling becomes explicit, no compatibility pause or acknowledgement marker is created while onboarding remains pending, and a second 1.0 start is idempotent. The absent marker is deliberately deferred while onboarding is pending; a later configured start records the current version if no other review is required. This fixture rewrite is limited to the missing-flag boundary; the configured transport upgrades use only state created and saved by 0.25.83 itself. diff --git a/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts index 6b249b38..1ccc0ab0 100644 --- a/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts +++ b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts @@ -180,12 +180,17 @@ async function waitFor( throw new Error(`State did not settle: ${JSON.stringify(state)}`); } -async function settled(pair: Pair, path: string, body: string): Promise<[Snapshot, Snapshot]> { +async function settled( + pair: Pair, + path: string, + body: string, + allowDeletedBranches = false +): Promise<[Snapshot, Snapshot]> { const ready = (state: Snapshot) => state.content === body && liveLeaves(state).length === 1 && liveLeaves(state)[0].body === body && - state.leaves.length === 1 && + (allowDeletedBranches || state.leaves.length === 1) && !state.waiting.some((item) => item.path === path); const deadline = Date.now() + 20000; let states: [Snapshot, Snapshot]; @@ -234,6 +239,15 @@ async function main(): Promise { await assertCouchDbReachable(config); const dbName = makeUniqueDatabaseName(config.dbPrefix, "live-batch"); const couch = { ...config, dbName }; + const remoteRestartEdit = process.env.ISSUE1207_RESTART_REMOTE_EDIT; + check( + !remoteRestartEdit || ["mergeable", "conflicting"].includes(remoteRestartEdit), + "Unknown remote edit for the provenance restart case" + ); + check( + !remoteRestartEdit || process.env.ISSUE1207_RESTART_BEFORE_PROVENANCE === "true", + "A remote restart edit requires the provenance gate" + ); const overrides = { doctorProcessedVersion: "1.0.0", notifyThresholdOfRemoteStorageSize: -1, @@ -248,7 +262,7 @@ async function main(): Promise { syncAfterMerge: false, checkConflictOnlyOnOpen: true, showMergeDialogOnlyOnActive: true, - disableMarkdownAutoMerge: true, + disableMarkdownAutoMerge: !remoteRestartEdit, resolveConflictsByNewerFile: false, writeDocumentsIfConflicted: false, writeLogToTheFile: false, @@ -259,8 +273,6 @@ async function main(): Promise { evidence.commonlibVersion = JSON.parse( await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8") ).version; - evidence.sourceBaseRevision = "73a14d4e4b0b414a1acfe170d01500abb3dca2e9"; - evidence.unreleasedImplementation = true; if (process.env.ISSUE1207_COMMONLIB_ARTEFACT) { evidence.commonlibArtefactSha256 = createHash("sha256") .update(new Uint8Array(readFileSync(process.env.ISSUE1207_COMMONLIB_ARTEFACT))) @@ -370,6 +382,31 @@ async function main(): Promise { type:item.type,path:item.args?.file?.path,oldPath:item.args?.oldPath}))};` ), }); + if (name === "enabled-settings-save-restart" || name.startsWith("enabled-before-provenance")) { + observations.push({ + name, + device, + revisionEvidence: await evaluate( + page, + ` + const path=${JSON.stringify(`${prefix}restart.md`)}; + const id=await core.services.path.path2id(path); + const leaves=await core.localDatabase.getRaw(id,{open_revs:'all',revs:true}); + const revisions=new Set(); + for(const leaf of leaves){const h=leaf.ok?._revisions; + for(let i=0;i<(h?.ids.length??0);i++)revisions.add((h.start-i)+'-'+h.ids[i]);} + const result=[]; + for(const rev of revisions){ + const raw=await core.localDatabase.getRaw(id,{rev,revs:true}).catch(error=>({error:String(error)})); + const doc=await core.localDatabase.getDBEntry(path,{rev,revs:true},false,true,true).catch(()=>false); + result.push({rev,deleted:raw._deleted??false,ctime:raw.ctime,mtime:raw.mtime, + size:raw.size,error:raw.error,body:doc?(Array.isArray(doc.data)?doc.data.join(''):doc.data):null}); + } + return {revisions:result,provenance:await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(path)}; + ` + ), + }); + } await page.screenshot({ path: join(diagnostics, `${name}-${device}-failure.png`) }); try { await evaluate(page, "await manager.waitForIdle();"); @@ -663,27 +700,61 @@ async function main(): Promise { const saveAfterEdit = process.env.ISSUE1207_RESTART_SAVE_AFTER_EDIT === "true"; const restartBucket = process.env.ISSUE1207_RESTART_BUCKET; + const stopBeforeProvenance = process.env.ISSUE1207_RESTART_BEFORE_PROVENANCE === "true"; + check(!stopBeforeProvenance || !restartBucket, "The provenance and timestamp gates are separate cases"); check( !restartBucket || ["same", "exact", "different"].includes(restartBucket), "Unknown restart timestamp condition" ); - const restartName = restartBucket - ? `enabled-inflight-restart-${restartBucket}` - : saveAfterEdit - ? "enabled-settings-save-restart" - : "enabled-restart"; + const restartName = stopBeforeProvenance + ? `enabled-before-provenance${remoteRestartEdit ? `-${remoteRestartEdit}` : ""}-restart` + : restartBucket + ? `enabled-inflight-restart-${restartBucket}` + : saveAfterEdit + ? "enabled-settings-save-restart" + : "enabled-restart"; await run(restartName, async () => { const path = `${prefix}restart.md`; + const beforeContent = + remoteRestartEdit === "mergeable" + ? "Heading\n\nLocal: original\n\nRemote: original\n\nFooter\n" + : "Before restart\n"; + const localContent = + remoteRestartEdit === "mergeable" + ? "Heading\n\nLocal: saved\n\nRemote: original\n\nFooter\n" + : "Retained after restart\n"; + const remoteContent = + remoteRestartEdit === "mergeable" + ? "Heading\n\nLocal: saved\n\nRemote: changed remotely\n\nFooter\n" + : "Remote edit after the durable save\n"; const beforeRestart = await withPair(async (pair) => { await options(pair.a, "enabled-batch", 5, 60); await options(pair.b, "normal"); - if (!saveAfterEdit) + // Remote cases keep continuous replication running while the local save is held. + if (!saveAfterEdit || remoteRestartEdit) await evaluate( pair.a, "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" ); - await create(pair.b, path, "Before restart\n"); - const initial = await settled(pair, path, "Before restart\n"); + await create(pair.b, path, beforeContent); + const initial = await settled(pair, path, beforeContent); + if (stopBeforeProvenance) { + await evaluate( + pair.a, + ` + const access=core.serviceModules.databaseFileAccess; + const original=access.storeWithBaseRevision.bind(access); + access.storeWithBaseRevision=async(file,...args)=>{ + const revision=await original(file,...args); + if(file.path===${JSON.stringify(path)}&&revision){ + globalThis.__issue1207Batch.beforeProvenanceRevision=revision; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + return revision; + }; + ` + ); + } if (restartBucket) { check(initial[0].databaseMtime !== null, "Missing initial database mtime"); const mtime = @@ -703,11 +774,27 @@ async function main(): Promise { }; await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),"Retained after restart\\n",{mtime:${mtime}});` ); - } else await modify(pair.a, path, "Retained after restart\n"); + } else await modify(pair.a, path, localContent); const pending = await waitFor(pair.a, path, (state) => state.waiting.some((item) => item.path === path) ); - if (saveAfterEdit) await evaluate(pair.a, "await core.services.setting.saveSettingData();"); + if (saveAfterEdit && !remoteRestartEdit) { + await evaluate(pair.a, "await core.services.setting.saveSettingData();"); + observations.push({ + phase: "settings-save-before-stop", + initial, + pending, + current: await Promise.all([snapshot(pair.a, path), snapshot(pair.b, path)]), + sourceProbe: await evaluate( + pair.a, + `return { + events:globalThis.__issue1207Batch.events, + changes:globalThis.__issue1207Batch.changes, + provenance:await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(${JSON.stringify(path)}) + };` + ), + }); + } if (restartBucket) await evaluate( pair.a, @@ -715,6 +802,82 @@ async function main(): Promise { while(!globalThis.__issue1207Batch.inflightPaused&&Date.now()setTimeout(resolve,10)); if(!globalThis.__issue1207Batch.inflightPaused) throw new Error('The in-flight test gate did not start');` ); + let durableBeforeProvenance: Snapshot | undefined; + let remoteAdvanced: Snapshot | undefined; + if (stopBeforeProvenance) { + await evaluate( + pair.a, + ` + const deadline=Date.now()+core.services.setting.currentSettings().batchSaveMinimumDelay*1000+5000; + while(!globalThis.__issue1207Batch.beforeProvenanceRevision&&Date.now()setTimeout(resolve,10)); + if(!globalThis.__issue1207Batch.beforeProvenanceRevision)throw new Error('The durable-write provenance gate did not start'); + ` + ); + durableBeforeProvenance = await snapshot(pair.a, path); + const provenance = await evaluate<{ revision: string }>( + pair.a, + `return await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(${JSON.stringify(path)});` + ); + check( + durableBeforeProvenance.rev !== initial[0].rev && + liveLeaves(durableBeforeProvenance)[0]?.body === localContent, + "The gated database write is not durable" + ); + check( + provenance?.revision === initial[0].rev, + "The fixture did not stop before provenance was updated" + ); + observations.push({ + phase: "durable-before-provenance", + state: durableBeforeProvenance, + provenance, + }); + if (remoteRestartEdit) { + const receivedOriginal = await waitFor( + pair.b, + path, + (state) => state.rev === durableBeforeProvenance!.rev && state.content === localContent + ); + await modify(pair.b, path, remoteContent); + remoteAdvanced = await waitFor( + pair.b, + path, + (state) => + liveLeaves(state).length === 1 && + liveLeaves(state)[0].body === remoteContent && + state.rev === liveLeaves(state)[0].rev && + state.content === remoteContent + ); + check( + liveLeaves(remoteAdvanced)[0].ancestry.includes(durableBeforeProvenance.rev!), + "The remote edit did not descend from the already durable local save" + ); + const localDbAdvanced = await waitFor( + pair.a, + path, + (state) => + state.rev === remoteAdvanced!.rev && + liveLeaves(state).length === 1 && + liveLeaves(state)[0].body === remoteContent + ); + const stillOldProvenance = await evaluate<{ revision: string }>( + pair.a, + `return await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(${JSON.stringify(path)});` + ); + check( + localDbAdvanced.content === localContent && stillOldProvenance.revision === initial[0].rev, + "The incoming DB change did not precede Vault reflection and provenance persistence" + ); + observations.push({ + phase: "remote-edit-local-db-ahead-of-storage", + receivedOriginal, + remoteAdvanced, + localDbAdvanced, + stillOldProvenance, + }); + } + } await evaluate(pair.a, "await manager._takeSnapshot();"); check(pending.rev === initial[0].rev, "Restart test did not retain a pending edit"); if (restartBucket) { @@ -746,16 +909,18 @@ async function main(): Promise { sameBucket, }); } - if (!saveAfterEdit && !restartBucket) { + if (stopBeforeProvenance || (!saveAfterEdit && !restartBucket)) { const strictState = await snapshot(pair.a, path); const saved = await evaluate>( pair.a, `return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));` ); - check( - strictState.rev === initial[0].rev && strictState.waiting.some((item) => item.path === path), - "Restart fixture flushed before shutdown" - ); + if (!stopBeforeProvenance) + check( + strictState.rev === initial[0].rev && + strictState.waiting.some((item) => item.path === path), + "Restart fixture flushed before shutdown" + ); check( saved.some((item) => item.path === path), "Restart fixture did not persist the pending event" @@ -778,12 +943,12 @@ async function main(): Promise { const settings=core.services.setting.currentSettings(); return {initialized:await core.kvDB.get('initialized'),restorationRequested:manager.snapShotRestored!==null, settings:{isConfigured:settings.isConfigured,liveSync:settings.liveSync,batchSave:settings.batchSave,batchSaveInLiveSync:settings.batchSaveInLiveSync, - suspendFileWatching:settings.suspendFileWatching}, + suspendFileWatching:settings.suspendFileWatching,disableMarkdownAutoMerge:settings.disableMarkdownAutoMerge}, events:events?.map(item=>({type:item.type,path:item.args?.file?.path, stat:item.args?.file?.stat,skipBatchWait:item.skipBatchWait}))};` ), }); - return { initial, pending }; + return { initial, pending, durableBeforeProvenance, remoteAdvanced }; }); await stop(slots[0]); const persistedSettings = JSON.parse( @@ -817,11 +982,82 @@ async function main(): Promise { events:events?.map(item=>({type:item.type,path:item.args?.file?.path,stat:item.args?.file?.stat}))};` ), }); - const result = await settled(pair, path, "Retained after restart\n"); + if (remoteRestartEdit === "conflicting") { + const preservesBothBodies = (state: Snapshot) => { + const leaves = liveLeaves(state); + return ( + leaves.length === 2 && + leaves.some((leaf) => leaf.body === localContent) && + leaves.some( + (leaf) => leaf.body === remoteContent && leaf.rev === beforeRestart.remoteAdvanced!.rev + ) + ); + }; + const result = await Promise.all([ + waitFor(pair.a, path, preservesBothBodies), + waitFor(pair.b, path, preservesBothBodies), + ]); + check( + result[0].content === localContent && result[1].content === remoteContent, + "An unresolved conflict overwrote one of the preserved Vault contents" + ); + const manualDialogue = pair.a.locator(".modal").filter({ hasText: "Conflicting changes" }); + await manualDialogue.waitFor({ state: "visible", timeout: 10000 }); + await pair.a.screenshot({ path: join(diagnostics, `${restartName}-manual-conflict.png`) }); + return { result, manualConflict: true, restoredGuard: result[0].shouldBatchSave }; + } + const expectedContent = remoteRestartEdit ? remoteContent : localContent; + const result = await settled(pair, path, expectedContent, saveAfterEdit || stopBeforeProvenance); check(result[0].shouldBatchSave, "The saved LiveSync batching choice was not restored"); + if (beforeRestart.remoteAdvanced) { + for (const state of result) + check( + liveLeaves(state)[0].ancestry.includes(beforeRestart.remoteAdvanced.rev!), + "Sensible merge lost the later remote revision's ancestry" + ); + check( + (await pair.a.locator(".modal").filter({ hasText: "Conflicting changes" }).count()) === 0, + "The mergeable recovery left a manual conflict dialogue" + ); + } + for (const [index, page] of [pair.a, pair.b].entries()) { + const provenance = await evaluate<{ revision: string }>( + page, + `const store=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1'); + const deadline=Date.now()+5000; + let record; + do { + record=await store.get(${JSON.stringify(path)}); + if(record?.revision===${JSON.stringify(result[index].rev)})return record; + await new Promise(resolve=>setTimeout(resolve,10)); + }while(Date.now() { + const result = await settled(pair, path, remoteRestartEdit ? remoteContent : localContent, true); + check( + result[0].rev === afterRestart.result[0].rev, + "A repeated restart created another revision after recovery had converged" + ); + check( + result[0].leaves.length === afterRestart.result[0].leaves.length, + "A repeated restart added another branch after recovery had converged" + ); + return result; + }); + } + return { beforeRestart, afterRestart, repeatedRestart }; }); await run("enabled-after-write-restart", async () => { diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 0a08e03d..8e394ccd 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -23,6 +23,7 @@ const focusedScenarios = new Set([ "p2p-pane", "replication-status", "vault-reflection", + "third-party-chunks", "couchdb-upload", "couchdb-http-cache", "chunk-fetch-retry", @@ -36,6 +37,7 @@ const focusedScenarios = new Set([ "p2p-setup-uri-workflow", "partial-startup-file-failure", "startup-scan", + "startup-integrity", "stale-file-restart", "folder-batch", "setup-uri-workflow", @@ -52,6 +54,7 @@ const focusedScenarios = new Set([ "setting-markdown-export", "setting-markdown-roundtrip", "upgrade-from-stable", + "upgrade-livesync-batching", ]); function usage(): string { diff --git a/test/e2e-obsidian/scripts/startup-integrity.ts b/test/e2e-obsidian/scripts/startup-integrity.ts new file mode 100644 index 00000000..f30bdd10 --- /dev/null +++ b/test/e2e-obsidian/scripts/startup-integrity.ts @@ -0,0 +1,396 @@ +/** Checks incomplete-document integrity against real Vault storage in Obsidian. */ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +const filePath = "startup-integrity.md"; +const fileContent = "startup integrity remains healthy\n"; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const vault = await createTemporaryVault("obsidian-startup-integrity-"); + let session: ObsidianLiveSyncSession | undefined; + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const settings = createE2eCouchDbPluginData( + // No replication is enabled; this scenario uses only the local database. + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "startup-integrity-local" }, + { + deviceAndVaultName: "startup-integrity-device", + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + } + ); + + async function evaluate(body: string): Promise { + if (!session) throw new Error("The Obsidian session is not running"); + return await evalObsidianJson( + cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync']?.core; + const sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms)); + const require=(condition,message)=>{if(!condition)throw new Error(message);}; + const waitFor=async(predicate,message,timeoutMs=15000)=>{ + const deadline=Date.now()+timeoutMs; + while(Date.now()(` + require(await core.services.fileProcessing.commitPendingFileEvents(), 'Initial local flush failed'); + const path=${JSON.stringify(filePath)}; + const expectedContent=${JSON.stringify(fileContent)}; + const storage=core.serviceModules.storageAccess; + const fileHandler=core.serviceModules.fileHandler; + const keyValueDB=core.services.keyValueDB.kvDB; + const confirm=core.services.UI.confirm; + const noticeGroups=core.services.context.noticeGroups; + const events=core.services.context.events; + require(storage&&typeof storage.statHidden==='function'&& + typeof storage.readHiddenFileBinary==='function', + 'The real StorageAccess methods were not available'); + require(fileHandler&&typeof fileHandler.storeFileToDB==='function', + 'The real file handler was not available'); + + const snapshot=async()=>{ + const meta=await core.localDatabase.getDBEntryMeta(path,undefined,true); + require(meta&&meta._rev,'The integrity fixture Metadata was missing'); + const doc=await core.localDatabase.getDBEntryFromMeta(meta); + require(doc&&doc.data!==undefined,'The integrity fixture Chunks could not be loaded'); + const content=Array.isArray(doc.data)?doc.data.join(''):doc.data; + require(typeof content==='string','The Markdown fixture did not load as text'); + const bytes=Array.from(new TextEncoder().encode(content)); + require(meta.size===bytes.length,'Metadata size did not match the stored bytes'); + require(doc.size===bytes.length,'Loaded document size did not match the stored bytes'); + return {revision:meta._rev,metadataSize:meta.size,documentSize:doc.size,content,bytes}; + }; + const assertUnchanged=(before,after,label)=>{ + require(before.revision===after.revision,label+': the Metadata revision changed'); + require(before.metadataSize===after.metadataSize,label+': the Metadata size changed'); + require(before.content===after.content,label+': the stored content changed'); + require(JSON.stringify(before.bytes)===JSON.stringify(after.bytes), + label+': the stored bytes changed'); + }; + const activeBytes=async()=>{ + const content=await app.vault.adapter.read(path); + return {content,bytes:Array.from(new TextEncoder().encode(content))}; + }; + const marker=async()=>await keyValueDB.get('checkIncompleteDocs')===true; + const activeState={current:null}; + const originalStat=storage.statHidden; + const originalRead=storage.readHiddenFileBinary; + const originalStore=fileHandler.storeFileToDB; + const originalDialogue=confirm.askSelectStringDialogue; + const ownStat=Object.prototype.hasOwnProperty.call(storage,'statHidden'); + const ownRead=Object.prototype.hasOwnProperty.call(storage,'readHiddenFileBinary'); + const ownStore=Object.prototype.hasOwnProperty.call(fileHandler,'storeFileToDB'); + const ownDialogue=Object.prototype.hasOwnProperty.call(confirm,'askSelectStringDialogue'); + const unhandledErrors=[]; + const onUnhandled=event=>{ + const reason=event.reason; + unhandledErrors.push(String(reason&&reason.message?reason.message:reason)); + event.preventDefault(); + }; + let installedStat=false; + let installedRead=false; + let installedStore=false; + let installedDialogue=false; + const restore=(owner,key,hadOwn,original)=>{ + if(hadOwn)owner[key]=original; + else delete owner[key]; + }; + + const onStat=async function(candidatePath){ + const actual=await originalStat.call(this,candidatePath); + const current=activeState.current; + if(!current||candidatePath!==path)return actual; + current.statCalls++; + require(actual&&actual.type==='file','StorageAccess did not report the fixture as a file'); + current.actualStats.push({size:actual.size,mtime:actual.mtime,ctime:actual.ctime}); + if(current.mode==='stat-read-race'&¤t.statCalls===1){ + const oldStat={...actual,size:actual.size+1}; + current.returnedStats.push({size:oldStat.size,mtime:oldStat.mtime,ctime:oldStat.ctime}); + return oldStat; + } + if(current.mode==='unstable-final-stat'&¤t.statCalls===2){ + const changedStat={...actual,mtime:actual.mtime+1}; + current.returnedStats.push({size:changedStat.size,mtime:changedStat.mtime,ctime:changedStat.ctime}); + return changedStat; + } + current.returnedStats.push({size:actual.size,mtime:actual.mtime,ctime:actual.ctime}); + return actual; + }; + const onRead=async function(candidatePath){ + const bytes=await originalRead.call(this,candidatePath); + const current=activeState.current; + if(current&&candidatePath===path){ + current.readCalls++; + current.readBytes=Array.from(new Uint8Array(bytes)); + } + return bytes; + }; + const onStore=async function(file,...args){ + const current=activeState.current; + if(current&&file?.path===path)current.repairCalls++; + return await originalStore.call(this,file,...args); + }; + const onDialogue=async function(message,buttons,...args){ + const current=activeState.current; + if(current){ + current.dialogueCalls++; + current.dialogueMessages.push(String(message)); + return Array.isArray(buttons)&&buttons.length>0?buttons[0]:false; + } + return await originalDialogue.call(this,message,buttons,...args); + }; + + const runRegisteredCheck=async current=>{ + const originalFinish=noticeGroups.finish; + const ownFinish=Object.prototype.hasOwnProperty.call(noticeGroups,'finish'); + let finished=false; + noticeGroups.finish=function(...args){ + try{return originalFinish.apply(this,args);} + finally{ + if(args[0]==='startup-integrity-check')finished=true; + } + }; + activeState.current=current; + try{ + events.emitEvent('request-run-fix-incomplete'); + await waitFor(()=>finished,'The registered incomplete-document check did not finish'); + }finally{ + activeState.current=null; + restore(noticeGroups,'finish',ownFinish,originalFinish); + } + }; + const makeObservation=mode=>({ + mode, + statCalls:0, + actualStats:[], + returnedStats:[], + readCalls:0, + readBytes:[], + dialogueCalls:0, + dialogueMessages:[], + repairCalls:0, + }); + + window.addEventListener('unhandledrejection',onUnhandled); + try{ + storage.statHidden=onStat; + installedStat=true; + storage.readHiddenFileBinary=onRead; + installedRead=true; + fileHandler.storeFileToDB=onStore; + installedStore=true; + confirm.askSelectStringDialogue=onDialogue; + installedDialogue=true; + + const firstStorageStat=await storage.statHidden(path); + require(firstStorageStat&&firstStorageStat.type==='file', + 'The real Vault file was not visible through StorageAccess'); + const firstVault=await activeBytes(); + const first=snapshot(); + const healthyBefore=await first; + require(firstVault.content===expectedContent&&healthyBefore.content===expectedContent, + 'The fixture content did not match between the Vault and Metadata'); + require(firstStorageStat.size===healthyBefore.bytes.length, + 'The real Vault size did not match the Metadata bytes'); + + await keyValueDB.set('checkIncompleteDocs',false); + const healthy=makeObservation('healthy'); + const healthyUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(healthy); + const healthyAfter=await snapshot(); + const healthyMarker=await marker(); + const healthyVaultAfter=await activeBytes(); + require(healthy.statCalls===2,'The healthy fixture was not checked with both StorageAccess stats'); + require(healthy.readCalls===0,'The healthy fixture was read only to determine its size'); + require(healthy.dialogueCalls===0,'The healthy fixture opened a repair dialogue'); + require(healthy.repairCalls===0,'The healthy fixture invoked repair'); + require(healthyMarker,'The healthy integrity check did not set its completion marker'); + require(unhandledErrors.length===healthyUnhandledBefore, + 'The healthy integrity check raised an unhandled error'); + require(healthyVaultAfter.content===expectedContent,'The healthy Vault file changed'); + assertUnchanged(healthyBefore,healthyAfter,'Healthy scan'); + const observations=[ + { + case:'healthy', + revision:healthyAfter.revision, + storedBytes:healthyAfter.bytes.length, + readCalls:healthy.readCalls, + statCalls:healthy.statCalls, + dialogueCalls:healthy.dialogueCalls, + repairCalls:healthy.repairCalls, + completionMarker:healthyMarker, + storedBytesUnchanged:true, + }, + ]; + + const raceBefore=await snapshot(); + await keyValueDB.set('checkIncompleteDocs',false); + const race=makeObservation('stat-read-race'); + const raceUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(race); + const raceAfter=await snapshot(); + const raceMarker=await marker(); + const raceVaultAfter=await activeBytes(); + require(race.statCalls===1,'The controlled stat/read race did not use one initial stat'); + require(race.actualStats[0]?.size===raceBefore.bytes.length, + 'The real StorageAccess stat did not see the Metadata size'); + require(race.returnedStats[0]?.size===raceBefore.bytes.length+1, + 'The checker did not see the controlled old mismatched size'); + require(race.readCalls===1,'The controlled mismatch did not read real Vault bytes'); + require(JSON.stringify(race.readBytes)===JSON.stringify(raceBefore.bytes), + 'The actual StorageAccess read did not match the database bytes'); + require(race.dialogueCalls===0,'The settled stat/read race opened a repair dialogue'); + require(race.repairCalls===0,'The settled stat/read race invoked repair'); + require(raceMarker,'The settled stat/read race did not set its completion marker'); + require(unhandledErrors.length===raceUnhandledBefore, + 'The settled stat/read race raised an unhandled error'); + require(raceVaultAfter.content===expectedContent,'The Vault file changed during the stat/read race'); + assertUnchanged(raceBefore,raceAfter,'Settled stat/read race'); + observations.push({ + case:'stat-read-race', + revision:raceAfter.revision, + storedBytes:raceAfter.bytes.length, + observedStatSize:race.returnedStats[0]?.size, + readCalls:race.readCalls, + readMatchedDatabase:true, + dialogueCalls:race.dialogueCalls, + repairCalls:race.repairCalls, + completionMarker:raceMarker, + storedBytesUnchanged:true, + }); + + const unstableBefore=await snapshot(); + await keyValueDB.set('checkIncompleteDocs',false); + const unstable=makeObservation('unstable-final-stat'); + const unstableUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(unstable); + await waitFor(()=>unhandledErrors.length>unstableUnhandledBefore, + 'The changed final stat did not reject the incomplete-document check'); + const unstableError=unhandledErrors.slice(unstableUnhandledBefore).join(' | '); + const unstableAfter=await snapshot(); + const unstableMarker=await marker(); + require(unstable.statCalls===2,'The changed final stat was not checked'); + require(unstable.returnedStats[1]?.mtime!==unstable.actualStats[1]?.mtime, + 'The final StorageAccess stat was not changed by the race fixture'); + require(unstable.readCalls===0,'The healthy path was read during the final-stat race'); + require(unstableError.includes('Storage changed during the incomplete document check'), + 'The final-stat race did not report the expected incomplete-check error'); + require(unstableMarker===false,'The incomplete check was marked complete after storage changed'); + require(unstable.dialogueCalls===0,'The final-stat race opened a repair dialogue'); + require(unstable.repairCalls===0,'The final-stat race invoked repair'); + assertUnchanged(unstableBefore,unstableAfter,'Final-stat race'); + observations.push({ + case:'unstable-final-stat', + revision:unstableAfter.revision, + statCalls:unstable.statCalls, + readCalls:unstable.readCalls, + reportedError:unstableError, + completionMarker:unstableMarker, + storedBytesUnchanged:true, + }); + + await keyValueDB.set('checkIncompleteDocs',false); + const retry=makeObservation('settled-retry'); + const retryUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(retry); + const retryAfter=await snapshot(); + const retryMarker=await marker(); + require(retry.statCalls===2,'The settled retry did not complete both StorageAccess stats'); + require(retry.readCalls===0,'The settled retry read healthy storage'); + require(retry.dialogueCalls===0,'The settled retry opened a repair dialogue'); + require(retry.repairCalls===0,'The settled retry invoked repair'); + require(retryMarker,'The settled retry did not set its completion marker'); + require(unhandledErrors.length===retryUnhandledBefore, + 'The settled retry raised an unhandled error'); + assertUnchanged(unstableBefore,retryAfter,'Settled retry'); + observations.push({ + case:'settled-retry', + revision:retryAfter.revision, + statCalls:retry.statCalls, + readCalls:retry.readCalls, + dialogueCalls:retry.dialogueCalls, + repairCalls:retry.repairCalls, + completionMarker:retryMarker, + storedBytesUnchanged:true, + }); + return JSON.stringify(observations); + }finally{ + window.removeEventListener('unhandledrejection',onUnhandled); + if(installedDialogue)restore(confirm,'askSelectStringDialogue',ownDialogue,originalDialogue); + if(installedStore)restore(fileHandler,'storeFileToDB',ownStore,originalStore); + if(installedRead)restore(storage,'readHiddenFileBinary',ownRead,originalRead); + if(installedStat)restore(storage,'statHidden',ownStat,originalStat); + } + `); + + console.log( + JSON.stringify( + { + ok: true, + bundleHash, + trigger: "registered-request-run-fix-incomplete-event", + naturalRestart: false, + observations, + }, + null, + 2 + ) + ); + } finally { + try { + await session?.app.stop(); + } finally { + await vault.dispose(); + } + } +} + +await main(); diff --git a/test/e2e-obsidian/scripts/third-party-chunks.ts b/test/e2e-obsidian/scripts/third-party-chunks.ts new file mode 100644 index 00000000..e781d74e --- /dev/null +++ b/test/e2e-obsidian/scripts/third-party-chunks.ts @@ -0,0 +1,269 @@ +/** Exercises synthetic split-surrogate input through real Obsidian file reflection. */ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + assertEqual, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const textPath = "synthetic-split-pair.md"; + const controlPath = "complete-pair.md"; + const binaryPath = "attachment.bin"; + const textContent = "prefix 🙂 suffix\n" + "normal chunk payload\n".repeat(50_000); + const controlContent = "prefix 🙂 suffix"; + const binaryBytes = Buffer.from([0, 255, 31, 128]); + const textSize = Buffer.byteLength(textContent, "utf8"); + const textHash = `sha256:${createHash("sha256").update(textContent, "utf8").digest("hex")}`; + const controlSize = Buffer.byteLength(controlContent, "utf8"); + const controlHash = `sha256:${createHash("sha256").update(controlContent, "utf8").digest("hex")}`; + const settings = createE2eCouchDbPluginData( + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "third-party-chunks-local" }, + { + customChunkSize: 0, + processSizeMismatchedFiles: false, + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + } + ); + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const vault = await createTemporaryVault("obsidian-third-party-chunks-"); + let session: ObsidianLiveSyncSession | undefined; + + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + pluginData: settings, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + lifecycle: { + beforeLaunch: async () => { + const registryPath = join(vault.userDataPath, "obsidian.json"); + const registry = JSON.parse(await readFile(registryPath, "utf8")); + registry.updateDisabled = true; + await writeFile(registryPath, JSON.stringify(registry)); + }, + }, + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + + const reflection = await evalObsidianJson<{ + sizeCheckingEnabled: boolean; + split: { + reflected: boolean; + metadataRevisionUnchanged: boolean; + chunkCount: number; + provenanceRevision: string; + contentSize: number; + contentHash: string; + }; + control: { + reflected: boolean; + content: string; + metadataRevisionUnchanged: boolean; + provenanceRevision: string; + contentSize: number; + contentHash: string; + }; + binary: { reflected: boolean; bytes: number[]; metadataRevisionUnchanged: boolean }; + }>( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const require=(condition,message)=>{if(!condition)throw new Error(message);};", + "const textPath='synthetic-split-pair.md';", + "const controlPath='complete-pair.md';", + "const binaryPath='attachment.bin';", + "const textContent='prefix 🙂 suffix\\n'+'normal chunk payload\\n'.repeat(50000);", + "const controlContent='prefix 🙂 suffix';", + "const binaryBytes=[0,255,31,128];", + "const store=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1');", + "require(core.settings.processSizeMismatchedFiles===false,'Size mismatch checking must remain enabled');", + "const seed=async(path,blob,type,datatype)=>{", + " require(!app.vault.getAbstractFileByPath(path),'Fixture already exists in the Vault: '+path);", + " const id=await core.services.path.path2id(path);", + " const now=Date.now();", + " const saved=await core.localDatabase.putDBEntry({_id:id,path,data:blob,ctime:now,mtime:now,", + " size:blob.size,children:[],datatype,type,eden:{} });", + " require(saved?.ok,'Could not create Metadata and Chunks: '+path);", + " const meta=await core.localDatabase.getDBEntryMeta(path,undefined,true);", + " require(meta&&meta._rev===saved.rev,'Could not reload the created Metadata: '+path);", + " return meta;", + "};", + "const textMeta=await seed(textPath,new Blob([textContent],{type:'text/plain'}),'plain','plain');", + "require((textMeta.children??[]).length>0,'The primary fixture did not create normal Chunks');", + "const savedText=await core.localDatabase.getDBEntry(textPath,{rev:textMeta._rev,revs:true},false,true,true);", + "const storedText=Array.isArray(savedText?.data)?savedText.data.join(''):savedText?.data;", + "require(storedText===textContent,'The seeded local database content did not match the joined text');", + "const controlMeta=await seed(controlPath,new Blob([controlContent],{type:'text/plain'}),'plain','plain');", + "const binaryMeta=await seed(binaryPath,new Blob([new Uint8Array(binaryBytes)],{type:'application/octet-stream'}),'newnote','newnote');", + "const initialTextRevision=textMeta._rev;", + "const initialControlRevision=controlMeta._rev;", + "const database=core.serviceModules.databaseFileAccess;", + "require(typeof database?.fetchEntryFromMeta==='function','The database file-access provider is missing');", + "const descriptor=Object.getOwnPropertyDescriptor(database,'fetchEntryFromMeta');", + "const original=database.fetchEntryFromMeta;", + "let supplied=0;", + "const pairIndex=textContent.indexOf('🙂');", + "const syntheticChunks=[textContent.slice(0,pairIndex+1),textContent.slice(pairIndex+1)];", + "require(syntheticChunks[0]==='prefix \\uD83D','The synthetic first chunk did not end with the high surrogate');", + "require(syntheticChunks[1].startsWith('\\uDE42 suffix'),'The synthetic second chunk did not start with the low surrogate');", + "database.fetchEntryFromMeta=async function(meta,...args){", + " const loaded=await original.call(database,meta,...args);", + " if(meta?._id!==textMeta._id||meta?.path!==textPath||!loaded)return loaded;", + " supplied++;", + " return {...loaded,data:syntheticChunks};", + "};", + "let reflectedText=false;", + "try{reflectedText=await core.serviceModules.fileHandler.dbToStorage(textMeta,null,true);}", + "finally{if(descriptor)Object.defineProperty(database,'fetchEntryFromMeta',descriptor);else delete database.fetchEntryFromMeta;}", + "require(supplied===1,'The synthetic provider was not used exactly once');", + "require(reflectedText,'Real Obsidian reflection rejected the joined text');", + "const textFile=app.vault.getAbstractFileByPath(textPath);", + "require(textFile,'The reflected text file is missing from the Vault');", + "const vaultText=await app.vault.read(textFile);", + "require(vaultText===textContent,'The Vault text does not match the joined input');", + "const textMetaAfter=await core.localDatabase.getDBEntryMeta(textPath,undefined,true);", + "require(textMetaAfter?._rev===initialTextRevision,'Text reflection changed the Metadata revision');", + "const textProvenance=await store.get(textPath);", + "require(textProvenance?.revision===initialTextRevision,'Text provenance has the wrong revision');", + `require(textProvenance?.contentSize===${textSize},'Text provenance has the wrong UTF-8 byte size');`, + `require(textProvenance?.contentHash===${JSON.stringify(textHash)},'Text provenance has the wrong content hash');`, + "const reflectedControl=await core.serviceModules.fileHandler.dbToStorage(controlMeta,null,true);", + "require(reflectedControl,'Real Obsidian reflection rejected the complete-pair control');", + "const controlFile=app.vault.getAbstractFileByPath(controlPath);", + "require(controlFile&&await app.vault.read(controlFile)===controlContent,'The complete-pair control content changed');", + "const controlVaultText=await app.vault.read(controlFile);", + "const controlMetaAfter=await core.localDatabase.getDBEntryMeta(controlPath,undefined,true);", + "require(controlMetaAfter?._rev===initialControlRevision,'Control reflection changed the Metadata revision');", + "const controlProvenance=await store.get(controlPath);", + "require(controlProvenance?.revision===initialControlRevision,'Control provenance has the wrong revision');", + `require(controlProvenance?.contentSize===${controlSize},'Control provenance has the wrong UTF-8 byte size');`, + `require(controlProvenance?.contentHash===${JSON.stringify(controlHash)},'Control provenance has the wrong content hash');`, + "const reflectedBinary=await core.serviceModules.fileHandler.dbToStorage(binaryMeta,null,true);", + "require(reflectedBinary,'Real Obsidian reflection rejected the binary attachment');", + "const binaryFile=app.vault.getAbstractFileByPath(binaryPath);", + "require(binaryFile,'The reflected binary attachment is missing from the Vault');", + "const binaryVaultBytes=Array.from(new Uint8Array(await app.vault.readBinary(binaryFile)));", + "require(JSON.stringify(binaryVaultBytes)===JSON.stringify(binaryBytes),'The Vault binary bytes changed');", + "const binaryMetaAfter=await core.localDatabase.getDBEntryMeta(binaryPath,undefined,true);", + "require(binaryMetaAfter?._rev===binaryMeta._rev,'Binary reflection changed the Metadata revision');", + "return JSON.stringify({sizeCheckingEnabled:!core.settings.processSizeMismatchedFiles,", + " split:{reflected:reflectedText,metadataRevisionUnchanged:textMetaAfter._rev===initialTextRevision,chunkCount:textMeta.children.length,", + " provenanceRevision:textProvenance.revision,contentSize:textProvenance.contentSize,contentHash:textProvenance.contentHash},", + " control:{reflected:reflectedControl,content:controlVaultText,", + " metadataRevisionUnchanged:controlMetaAfter._rev===initialControlRevision,provenanceRevision:controlProvenance.revision,", + " contentSize:controlProvenance.contentSize,contentHash:controlProvenance.contentHash},", + " binary:{reflected:reflectedBinary,bytes:binaryVaultBytes,metadataRevisionUnchanged:binaryMetaAfter._rev===binaryMeta._rev}});", + "})()", + ].join(""), + session.cliEnv + ); + + assertEqual(reflection.sizeCheckingEnabled, true, "Size mismatch checking was disabled."); + assertEqual(reflection.split.reflected, true, "Split-pair text was not reflected."); + assertEqual( + reflection.split.metadataRevisionUnchanged, + true, + "Split-pair reflection changed its Metadata revision." + ); + assertEqual(reflection.split.provenanceRevision.length > 0, true, "Split-pair provenance is missing."); + assertEqual(reflection.split.contentSize, textSize, "Split-pair provenance has the wrong UTF-8 byte size."); + assertEqual(reflection.split.contentHash, textHash, "Split-pair provenance has the wrong content hash."); + assertEqual(reflection.control.reflected, true, "Complete-pair control was not reflected."); + assertEqual(reflection.control.content, controlContent, "Complete-pair control content changed."); + assertEqual( + reflection.control.metadataRevisionUnchanged, + true, + "Control reflection changed its Metadata revision." + ); + assertEqual(reflection.control.contentSize, controlSize, "Control provenance has the wrong UTF-8 byte size."); + assertEqual(reflection.control.contentHash, controlHash, "Control provenance has the wrong content hash."); + assertEqual(reflection.binary.reflected, true, "Binary attachment was not reflected."); + if (!Buffer.from(reflection.binary.bytes).equals(binaryBytes)) { + throw new Error("The Obsidian Vault API returned different binary attachment bytes."); + } + assertEqual( + reflection.binary.metadataRevisionUnchanged, + true, + "Binary reflection changed its Metadata revision." + ); + + const diskText = await readFile(join(vault.path, textPath)); + assertEqual(diskText.toString("utf8"), textContent, "On-disk UTF-8 text differs from the joined input."); + assertEqual(diskText.byteLength, textSize, "On-disk UTF-8 text has the wrong byte length."); + const diskControl = await readFile(join(vault.path, controlPath)); + assertEqual( + diskControl.toString("utf8"), + controlContent, + "On-disk complete-pair control differs from the input." + ); + const diskBinary = await readFile(join(vault.path, binaryPath)); + if (!diskBinary.equals(binaryBytes)) throw new Error("On-disk binary attachment bytes changed."); + + console.log( + JSON.stringify( + { + ok: true, + bundleHash, + syntheticInput: + "A local fetchEntryFromMeta result split the joined text between the emoji surrogates; this does not establish that CouchDB or any named third-party producer emits lone-surrogate chunks.", + sizeCheckingEnabled: reflection.sizeCheckingEnabled, + splitText: { + reflected: reflection.split.reflected, + metadataRevisionUnchanged: reflection.split.metadataRevisionUnchanged, + realChunkCount: reflection.split.chunkCount, + vaultTextMatches: true, + onDiskUtf8Bytes: diskText.byteLength, + provenanceRevision: reflection.split.provenanceRevision, + contentSize: reflection.split.contentSize, + contentHash: reflection.split.contentHash, + }, + completePairControl: { + reflected: reflection.control.reflected, + metadataRevisionUnchanged: reflection.control.metadataRevisionUnchanged, + vaultAndDiskContentMatch: true, + provenanceContentSize: reflection.control.contentSize, + provenanceContentHash: reflection.control.contentHash, + }, + binaryAttachment: { + reflected: reflection.binary.reflected, + exactVaultBytes: true, + exactDiskBytes: true, + metadataRevisionUnchanged: reflection.binary.metadataRevisionUnchanged, + }, + }, + null, + 2 + ) + ); + } finally { + try { + await session?.app.stop(); + } finally { + await vault.dispose(); + } + } +} + +await main(); diff --git a/test/e2e-obsidian/scripts/upgrade-livesync-batching.ts b/test/e2e-obsidian/scripts/upgrade-livesync-batching.ts new file mode 100644 index 00000000..47e02759 --- /dev/null +++ b/test/e2e-obsidian/scripts/upgrade-livesync-batching.ts @@ -0,0 +1,312 @@ +/** Keeps the 1.0.40 batching choice and Vault state when installing the current release. */ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + loadCouchDbConfig, + makeUniqueDatabaseName, + waitForCouchDbDocs, +} from "../runner/couchdb.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + assertEqual, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + pushLocalChanges, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { ensurePinnedReleaseArtifact, type PinnedPluginRelease } from "../runner/releaseArtifact.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { readRuntimeUpgradeState, waitForPersistentNodeIdentity } from "../runner/upgradeWorkflow.ts"; +import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; + +const sourceRelease: PinnedPluginRelease = { + pluginId: "obsidian-livesync", + version: "1.0.40", + files: [ + { + name: "main.js", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/1.0.40/main.js", + sha256: "bdd6b18ffe36a6a72ce774e857889b70d379559e4a5a3ccafee357a42daca6e9", + }, + { + name: "manifest.json", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/1.0.40/manifest.json", + sha256: "8f8f57fb105aba0ce7f9ea5104dbed92fea837d49fdf27b4f02b6099ddf1ba3b", + }, + { + name: "styles.css", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/1.0.40/styles.css", + sha256: "4ba00bef417ee98d9a62dd9febb792dc6555949cfc9e8e069450ad699ab2b9f0", + }, + ], +}; + +type BatchState = { + batchSave: boolean; + batchSaveInLiveSync: boolean | null; + liveSync: boolean; + periodicReplication: boolean; + shouldBatchSave: boolean; + minimum: number; + maximum: number; +}; + +const beforePath = "upgrade-batching-before.md"; +const afterPath = "upgrade-batching-after.md"; +const returnPath = "upgrade-batching-return.md"; +const beforeContent = "Saved by the published 1.0.40 plug-in.\n"; +const afterContent = "Saved after upgrading with LiveSync batching still disabled.\n"; +const returnContent = "Returned by an independent current-release Vault.\n"; + +async function artefactHashes(root: string): Promise> { + return Object.fromEntries( + await Promise.all( + sourceRelease.files.map(async ({ name }) => [ + name, + createHash("sha256") + .update(new Uint8Array(await readFile(join(root, name)))) + .digest("hex"), + ]) + ) + ); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const sourceRoot = await ensurePinnedReleaseArtifact(sourceRelease); + const targetRoot = resolve(process.env.E2E_LIVESYNC_TARGET_ARTIFACT_ROOT?.trim() || process.cwd()); + const targetManifest = JSON.parse(await readFile(join(targetRoot, "manifest.json"), "utf8")); + if (targetManifest.id !== sourceRelease.pluginId || targetManifest.version === sourceRelease.version) { + throw new Error("The target must be a newer Self-hosted LiveSync build."); + } + const hashes = await artefactHashes(targetRoot); + const config = await loadCouchDbConfig(); + if (!["127.0.0.1", "localhost"].includes(new URL(config.uri).hostname)) { + throw new Error("Use a disposable local CouchDB for the upgrade scenario."); + } + const dbName = makeUniqueDatabaseName(config.dbPrefix, "upgrade-batching"); + const vault = await createTemporaryVault("obsidian-upgrade-batching-"); + const verifier = await createTemporaryVault("obsidian-upgrade-verifier-"); + let session: ObsidianLiveSyncSession | undefined; + let ownsDatabase = false; + const pluginData = createE2eCouchDbPluginData( + { ...config, dbName }, + { + batchSave: true, + batchSaveMinimumDelay: 3, + batchSaveMaximumDelay: 7, + liveSync: false, + periodicReplication: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + doctorProcessedVersion: "1.0.0", + } + ); + + async function stop(): Promise { + if (session) await session.app.stop(); + session = undefined; + } + + async function start(currentVault: TemporaryVault, artifactRoot: string, seed = false): Promise { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault: currentVault, + artifactRoot, + pluginStartup: seed ? undefined : "natural", + pluginData: seed ? pluginData : undefined, + localStorageEntries: seed ? createE2eObsidianDeviceLocalState(currentVault.name) : undefined, + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + } + + async function evaluate(body: string): Promise { + if (!session) throw new Error("No active Obsidian session"); + return await evalObsidianJson( + cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + return JSON.stringify(await(async()=>{${body}})()); + })()`, + session.cliEnv + ); + } + + async function batching(): Promise { + return await evaluate(` + const s=core.services.setting.currentSettings(); + return {batchSave:s.batchSave,batchSaveInLiveSync:s.batchSaveInLiveSync??null, + liveSync:s.liveSync,periodicReplication:s.periodicReplication, + shouldBatchSave:core.serviceModules.storageAccess.vaultManager.shouldBatchSave, + minimum:s.batchSaveMinimumDelay,maximum:s.batchSaveMaximumDelay}; + `); + } + + async function createNote(path: string, content: string): Promise { + await evaluate(`await app.vault.create(${JSON.stringify(path)},${JSON.stringify(content)}); return true;`); + const entry = await waitForLocalDatabaseEntry(cliBinary, session!.cliEnv, path); + if ((await batching()).liveSync) { + await waitForCouchDbDocs(config, dbName, (docs) => + docs.some((doc) => doc._id === entry.id && doc._rev === entry.rev) + ); + } else { + await pushLocalChanges(cliBinary, session!.cliEnv); + } + } + + async function assertNote(currentVault: TemporaryVault, path: string, content: string): Promise { + const actual = await evaluate(` + const until=Date.now()+15000; + while(Date.now()setTimeout(resolve,100)); + } + throw new Error('Expected Vault contents did not arrive: '+${JSON.stringify(path)}); + `); + assertEqual(actual, content, "The Obsidian Vault did not contain the expected text."); + assertEqual(await readFile(join(currentVault.path, path), "utf8"), content, "On-disk text differed."); + } + + function assertUpgradedBatching(state: BatchState, live: boolean): void { + assertEqual(state.batchSave, true, "The legacy batching choice changed during upgrade."); + assertEqual(state.batchSaveInLiveSync, false, "Upgrade unexpectedly enabled LiveSync batching."); + assertEqual(state.liveSync, live, "The selected synchronisation mode changed."); + assertEqual(state.shouldBatchSave, !live, "The retained choice applied to the wrong mode."); + assertEqual(state.minimum, 3, "The minimum batch delay changed."); + assertEqual(state.maximum, 7, "The maximum batch delay changed."); + } + + try { + await assertCouchDbReachable(config); + await createCouchDbDatabase(config, dbName); + ownsDatabase = true; + await start(vault, sourceRoot, true); + await prepareRemote(cliBinary, session!.cliEnv); + await createNote(beforePath, beforeContent); + await evaluate(` + const s=core.services.setting.currentSettings(); + s.liveSync=true; + await core.services.setting.saveSettingData(); + await core.services.control.applySettings(); + return true; + `); + await stop(); + + // Confirm that the old release itself persisted this state before replacing any artefacts. + await start(vault, sourceRoot); + const oldBatch = await batching(); + assertEqual(oldBatch.batchSave, true, "The old release did not retain its batching choice."); + assertEqual(oldBatch.batchSaveInLiveSync, null, "The old release unexpectedly owns the new setting."); + assertEqual(oldBatch.liveSync, true, "The old release did not retain LiveSync mode."); + assertEqual(oldBatch.shouldBatchSave, false, "The old release unexpectedly batches LiveSync mode."); + await waitForPersistentNodeIdentity(cliBinary, session!.cliEnv); + const before = await readRuntimeUpgradeState(cliBinary, session!.cliEnv); + const originalEntry = await waitForLocalDatabaseEntry(cliBinary, session!.cliEnv, beforePath); + await assertNote(vault, beforePath, beforeContent); + await stop(); + const savedOldData = JSON.parse( + await readFile(join(vault.path, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8") + ); + assertEqual( + Object.prototype.hasOwnProperty.call(savedOldData, "batchSaveInLiveSync"), + false, + "The old settings fixture contains the new option." + ); + + await start(vault, targetRoot); + const upgraded = await readRuntimeUpgradeState(cliBinary, session!.cliEnv); + assertEqual(upgraded.pluginVersion, targetManifest.version, "The requested target build was not loaded."); + assertEqual(upgraded.localDatabaseName, before.localDatabaseName, "Upgrade replaced the local database."); + assertEqual(upgraded.nodeId, before.nodeId, "Upgrade replaced the device identity."); + assertEqual( + upgraded.settings.activeConfigurationId, + before.settings.activeConfigurationId, + "Upgrade changed the remote profile." + ); + assertEqual(upgraded.settings.couchDB_DBNAME, dbName, "Upgrade selected another remote database."); + assertEqual(upgraded.settings.versionUpFlash, "", "Upgrade unexpectedly paused for compatibility review."); + const firstBatch = await batching(); + assertUpgradedBatching(firstBatch, true); + assertEqual( + (await waitForLocalDatabaseEntry(cliBinary, session!.cliEnv, beforePath)).rev, + originalEntry.rev, + "Upgrade rewrote existing Metadata." + ); + await assertNote(vault, beforePath, beforeContent); + await createNote(afterPath, afterContent); + await stop(); + + await start(verifier, targetRoot, true); + await prepareRemote(cliBinary, session!.cliEnv); + await pushLocalChanges(cliBinary, session!.cliEnv); + await assertNote(verifier, beforePath, beforeContent); + await assertNote(verifier, afterPath, afterContent); + await createNote(returnPath, returnContent); + await stop(); + + await start(vault, targetRoot); + assertUpgradedBatching(await batching(), true); + await assertNote(vault, returnPath, returnContent); + await evaluate(` + const s=core.services.setting.currentSettings(); + s.liveSync=false;s.periodicReplication=true;s.periodicReplicationInterval=60; + await core.services.setting.saveSettingData(); + await core.services.control.applySettings(); + return true; + `); + assertUpgradedBatching(await batching(), false); + await stop(); + await start(vault, targetRoot); + const finalBatch = await batching(); + assertUpgradedBatching(finalBatch, false); + assertEqual(finalBatch.periodicReplication, true, "The periodic mode choice did not survive restart."); + await assertNote(vault, returnPath, returnContent); + assertEqual( + JSON.stringify(await artefactHashes(targetRoot)), + JSON.stringify(hashes), + "Target artefacts changed during the test." + ); + console.log( + JSON.stringify( + { + passed: true, + sourceVersion: sourceRelease.version, + targetVersion: targetManifest.version, + sourceHashes: Object.fromEntries(sourceRelease.files.map((file) => [file.name, file.sha256])), + targetHashes: hashes, + oldBatch, + firstBatch, + finalBatch, + retainedDatabase: true, + retainedNodeIdentity: true, + retainedMetadataRevision: true, + twoWayTransfer: true, + naturalRestarts: true, + }, + null, + 2 + ) + ); + } finally { + await stop(); + await vault.dispose(); + await verifier.dispose(); + if (ownsDatabase) await deleteCouchDbDatabase(config, dbName); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/updates.md b/updates.md index 63cc06b6..2a288762 100644 --- a/updates.md +++ b/updates.md @@ -12,40 +12,64 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased -### Customisation Sync +## 1.0.41 -#### Fixed - -- We can reload the customisation list and apply every file in a multi-file item on the first attempt. (#1271) - - Reloading or clearing the list also prevents earlier pending updates from restoring removed files and items. - -### Setup tools - -#### Fixed - -- We can generate a Setup URI with prefixed connection and encryption variables, avoiding collisions with shell or system variables such as `hostname` and `USERNAME`. (#817, #1274) - - The generator reads `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE`. The previous names remain supported when the corresponding prefixed value is unset or blank. +26th October, 2026 ### Synchronisation #### New Feature -- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏩ status icon and a comparison count before the upload count. An unchanged count disappears after three seconds. -- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. Wider gaps between editor status items make the counts easier to read. The `livesync-stat-inactive` class lets CSS snippets control how inactive items disappear. -- We can enable **Batch database updates in LiveSync mode** to combine repeated local edits before sending them to other devices. - - The choice starts disabled, including when older settings contain the previously ignored batch option. Each mode retains its own choice and uses the same minimum and maximum delays. - - Batching delays remote visibility and reduces intermediate updates. It does not establish when a template has finished or prevent genuine concurrent edits. +- Rapid edits can produce fewer intermediate versions on other devices while LiveSync mode remains active. + - Enable **Batch database updates in LiveSync mode** to group successive local edits. + - Disabled by default; each synchronisation mode keeps its own choice. + - Updates reach other devices later. Batching does not detect template completion or prevent genuine concurrent-edit conflicts. +- CouchDB synchronisation no longer looks idle while it is comparing change history. + - A ⏩ icon and comparison count show activity before documents are transferred. + - This count represents history entries, not files remaining to synchronise. +- Status counts are easier to follow, with less movement as values change. + - CSS snippets can hide unwanted items or style individual `livesync-stat-` items, including inactive ones. #### Fixed -- Local changes remain available for recovery while their database save is running, including when the application stops after a batch delay ends. - - Manual synchronisation and settings application now wait for the accepted local saves to finish and report failures. Pending events retain the existing snapshot format. - - Hidden File Sync and Customisation Sync now report completion after saving, including delayed customisation updates. Failed saves remain pending, while unchanged notifications allow later edits to proceed. -- Start-up reconciliation checks content when modification times are equal, so an edit within the same timestamp interval is not mistaken for an unchanged file. - - Existing revision provenance preserves the correct edit branch. Unreadable or continually changing content remains unresolved rather than being reported as successfully reconciled. -- We receive a clear Notice when **Automatic** requires Hidden File Sync to be enabled on this device. Our existing mode remains unchanged, and hidden-file initialisation cannot start while the feature is disabled. (#1270) -- We can initialise a whole plug-in's **MAIN** and **DATA** selections using the chosen Automatic initial action. -- We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) +- Recent Vault edits remain recoverable after an interrupted local database save. + - This includes edits whose modification times compare equal to the database copy, and pending Hidden File Sync and Customisation Sync changes. +- Manual synchronisation no longer starts before already accepted local changes finish saving to the database. + - Settings application also waits for those saves. Failed saves stay pending and report a failure. +- Text from third-party tools is handled more robustly. + - Characters split across Chunks no longer trigger false corruption warnings. +- Start-up integrity checks use less memory for healthy Vault files. (#1283) +- Start-up checks avoid false repair warnings for files that change during inspection. (#1283) + - This applies when the final size matches the database and there are no conflicts. + +### Customisation Sync + +#### Fixed + +- Applying a multi-file customisation no longer skips files after the list is refreshed. (#1271) +- Choosing **Automatic** now explains a missing Hidden File Sync prerequisite instead of proceeding into a failed initialisation. (#1270) + - Enable and initialise Hidden File Sync on this device first. The current mode is kept when the prerequisite is missing. +- Initialising a whole plug-in in **Automatic** mode no longer omits its selected **MAIN** or **DATA** files. + +### Setup tools + +#### Fixed + +- Setup URI generation can avoid shell-variable collisions and unintended use of the operating-system username. (#817, #1274) + - Use `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE` for connection and encryption values. + - Existing scripts remain supported: the previous names are used when the corresponding prefixed value is unset or blank. + +### Miscellaneous + +- Dependencies have been updated, including security fixes. + +### Acknowledgements + +Thank you for your contributions! + +- @rwbru, for the original Setup URI environment-variable fix ([#817](https://github.com/vrtmrz/obsidian-livesync/pull/817)). +- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts ([#1274](https://github.com/vrtmrz/obsidian-livesync/pull/1274)). +- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib ([#1283](https://github.com/vrtmrz/obsidian-livesync/pull/1283), [Commonlib #184](https://github.com/vrtmrz/livesync-commonlib/pull/184)). ## 1.0.40 diff --git a/versions.json b/versions.json index 19b183c0..a68f2a1a 100644 --- a/versions.json +++ b/versions.json @@ -51,5 +51,6 @@ "1.0.37": "1.7.2", "1.0.38": "1.7.2", "1.0.39": "1.7.2", - "1.0.40": "1.7.2" + "1.0.40": "1.7.2", + "1.0.41": "1.7.2" } From e83aa892b4619ee0662e236125bf8daf5bb2615d Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 26 Oct 2026 15:46:55 +0000 Subject: [PATCH 25/28] Move Vite build-file copying into scripts Keep the Node.js build helper and its tests in the build-script directory recognised by Community review. Update the Vite import and unit CI path filters while preserving the copy behaviour and release artefacts. --- .github/workflows/unit-ci.yml | 2 ++ {utils => scripts}/copy-build-files.ts | 0 {utils => scripts}/copy-build-files.unit.spec.ts | 0 vite.config.ts | 2 +- 4 files changed, 3 insertions(+), 1 deletion(-) rename {utils => scripts}/copy-build-files.ts (100%) rename {utils => scripts}/copy-build-files.unit.spec.ts (100%) diff --git a/.github/workflows/unit-ci.yml b/.github/workflows/unit-ci.yml index 9df08f2d..80147fd4 100644 --- a/.github/workflows/unit-ci.yml +++ b/.github/workflows/unit-ci.yml @@ -13,6 +13,7 @@ on: - 'src/**' - 'test/**' - '_tools/**' + - 'scripts/**' - 'package.json' - 'package-lock.json' - 'tsconfig.json' @@ -51,6 +52,7 @@ on: - 'src/**' - 'test/**' - '_tools/**' + - 'scripts/**' - 'package.json' - 'package-lock.json' - 'tsconfig.json' diff --git a/utils/copy-build-files.ts b/scripts/copy-build-files.ts similarity index 100% rename from utils/copy-build-files.ts rename to scripts/copy-build-files.ts diff --git a/utils/copy-build-files.unit.spec.ts b/scripts/copy-build-files.unit.spec.ts similarity index 100% rename from utils/copy-build-files.unit.spec.ts rename to scripts/copy-build-files.unit.spec.ts diff --git a/vite.config.ts b/vite.config.ts index e4195f74..bb55bdd0 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -6,7 +6,7 @@ import path from "path"; import { fileURLToPath } from "node:url"; import fs from "node:fs"; import { platform } from "node:process"; -import { copyBuildFiles } from "./utils/copy-build-files"; +import { copyBuildFiles } from "./scripts/copy-build-files"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); function readJson(filePath: string) { From e46c05ddc20bf1761ae56d35ec3290e6aa1ed1b1 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 26 Oct 2026 18:15:51 +0000 Subject: [PATCH 26/28] Make strict startup content checks optional --- ...local_file_save_completion_and_batching.md | 54 +- docs/settings.md | 6 + package.json | 1 + .../SettingDialogue/AdvancedSettingSpecs.ts | 1 + .../SettingDialogue/PaneAdvanced.unit.spec.ts | 4 +- .../SettingDialogue/SettingSpec.unit.spec.ts | 14 +- .../SettingsPageCatalogue.unit.spec.ts | 1 + src/modules/main/ModuleLiveSyncMain.ts | 27 +- .../main/ModuleLiveSyncMain.unit.spec.ts | 98 +- test/e2e-obsidian/README.md | 2 + test/e2e-obsidian/scripts/run-focused.ts | 1 + .../scripts/startup-content-check.ts | 836 ++++++++++++++++++ updates.md | 7 + 13 files changed, 1029 insertions(+), 23 deletions(-) create mode 100644 test/e2e-obsidian/scripts/startup-content-check.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md index 33ee9b8c..bbf67b13 100644 --- a/docs/adr/2026_10_local_file_save_completion_and_batching.md +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -59,10 +59,12 @@ that evidence. The inspected Commonlib source is commit ## Decision -Make local file processing recoverable until its result is established, use -content and provenance to reconcile ambiguous timestamps, and enable batching -in LiveSync mode only through an explicit choice after those contracts pass -verification. Apply the recovery contract with batching enabled or disabled. +Make local file processing recoverable until its result is established, retain +strict content and provenance reconciliation for manual and recovery scans, and +enable batching in LiveSync mode only through an explicit choice after those +contracts pass verification. Ordinary Obsidian start-up may skip a body read +only under the exact-match conditions below. Apply the recovery contract with +batching enabled or disabled. The content-reconciliation change targets selected ordinary files in the Offline Scanner's `NEWER_WINS` / `sync-newer` action. Keep the existing action @@ -208,12 +210,14 @@ finishes; correctness must also follow from the persistent record and start-up reconciliation. Remote availability must not determine whether local pending work survives shutdown. -### 4. Reconcile content when timestamps are inconclusive +### 4. Preserve strict content reconciliation -Retain the existing timestamp tolerance for platforms with coarse modification -times. Timestamp equality, including exactly equal raw timestamps, does not -prove content equality. The Offline Scanner must establish one of the following -outcomes through the ordinary non-forced file handler and its provenance rules: +Strict content checking retains the timestamp and provenance reconciliation +contract for platforms with coarse modification times. Timestamp equality, +including exactly equal raw timestamps, does not prove content equality. With +strict checking enabled, the Offline Scanner must establish one of the +following outcomes through the ordinary non-forced file handler and its +provenance rules: | Evidence | Required outcome | | ----------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | @@ -223,7 +227,19 @@ outcomes through the ordinary non-forced file handler and its provenance rules: | Vault content has no provable origin and matches no current live leaf | Preserve it according to the existing independent-branch rules. Do not attach it to an arbitrary database winner. | | Required content or provenance cannot be read, or state changes during comparison | Retain unresolved work and report or re-evaluate it. Do not mark it reconciled merely because the timestamp comparison is `EVEN`. | -The table describes stable-state reconciliation, not atomic database and +The `strictContentCheck` scan option is strict when omitted. Ordinary Obsidian +start-up can opt out through **Strict content check on startup**, which is +disabled by default. With the option disabled, a body read may be skipped only +when the file's raw modification time, byte size, and valid device-local +provenance exactly match a conflict-free current revision and no file +processing is pending. All other files retain strict content reconciliation. +An unrecorded edit that preserves its exact size and modification time can +therefore remain undetected by ordinary start-up while the option is disabled. +Manual scans, Fetch, Rebuild, and other explicit recovery operations remain +strict. Pending-event replay, incoming-change protection, and deletion guards +remain active in both modes. + +The table describes stable-state strict reconciliation, not atomic database and provenance persistence. Replay in the pre-provenance gap may create an identical sibling; with no remote descendant, one of the identical branches is tombstoned, leaving one live leaf with the exact saved content. If replication has @@ -249,12 +265,14 @@ Obsidian start-up can report `completed-with-file-failures` and allow unaffected files to proceed; an unresolved pair must not become `completed` to achieve that result. Keep the stricter callers' existing completion requirements. -This deliberately extends the current equal-time scanner contract. Existing -provenance tests expect no database write during that scan, including for an -edited file. Preserve their unchanged-file and later incoming-protection -scenarios; change the edited-file expectation to require preservation during -the scan itself. Test both identical timestamps and distinct raw timestamps -within one comparison bucket. +Keep the original unchanged-file, edited-file, untracked-provenance, +incoming-protection, and concurrent-recheck fixtures under strict content +checking, including identical timestamps and distinct raw timestamps within +one comparison bucket. Do not weaken those strict-mode assertions to describe +the ordinary start-up opt-out. Test the opt-out separately: only the exact +stat, provenance, conflict-free, and no-pending-processing match may skip a +body read. Pending-event replay and incoming or deletion guards remain covered +with the opt-out enabled. ### 5. Make batch timing and activation explicit @@ -369,7 +387,9 @@ matrix is: | Handler returns false, throws, or snapshot persistence fails | Flush reports failure; pending work is retained; unrelated recoverable work can continue. | | Buffered or in-progress work exists with no waiting timer | Flush still waits for actual completion. | | A flush overlaps new typing or a rename/delete barrier | The captured work completes without waiting for later work or deadlocking. | -| Equal timestamps: unchanged, edited, untracked, pruned provenance, unavailable Chunks, or concurrent recheck changes | Preserve correct ancestry and bytes, avoid duplicate revisions, and distinguish unresolved comparisons from successful reconciliation. | +| Strict check: equal timestamps, unchanged or edited files, untracked or pruned provenance, unavailable Chunks, or concurrent recheck changes | Preserve correct ancestry and bytes, avoid duplicate revisions, and distinguish unresolved comparisons from successful reconciliation. | +| Start-up check disabled: exact raw time, size, and provenance match with no conflict or pending processing | The body read may be skipped; an unrecorded edit which preserves exact size and modification time remains an accepted limitation. | +| Pending-event replay and incoming or deletion guards, with either start-up setting | Keep replay and protection active; do not skip work or remove a file based only on matching statistics. | | Settings save followed by stop; ordinary unload; interruption during replay | No operation is forgotten because a settings callback or shutdown released its timer. | | Sustained edits and more paths than available processing slots | Observe the intentional-delay deadline and progress of unrelated ready files. | | Old settings, mode changes, explicit opt-in, and settings import | Existing LiveSync mode users retain immediate saving until they choose batching. | diff --git a/docs/settings.md b/docs/settings.md index b53a7da2..135351aa 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -989,6 +989,12 @@ If enabled, chunks will be split into no more than 100 items. However, dedupe is Setting key: useSegmenter If this enabled, chunks will be split into semantically meaningful segments. Not all platforms support this feature. +#### Strict content check on startup + +Setting key: strictContentCheckOnStartup + +When enabled, ordinary start-up compares file content with the local database even when the file's exact modification time and size match. When disabled (the default), this comparison can be skipped only when the modification time, size, and recorded device-local provenance match a conflict-free current revision, and no file processing is pending. An edit made outside the normal file-event path which preserves its exact modification time and size can therefore go undetected by ordinary start-up. Manual scans and explicit Fetch or Rebuild operations keep strict content checking. + ### 3. Transfer Tweak #### Fetch chunks on demand diff --git a/package.json b/package.json index d5927927..076bed16 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "test:e2e:obsidian:p2p-connection-check": "tsx test/e2e-obsidian/scripts/p2p-connection-check.ts", "test:e2e:obsidian:p2p-connection-check:services": "npm run test:e2e:obsidian:p2p-connection-check -- --manage-p2p", "test:e2e:obsidian:partial-startup-file-failure": "tsx test/e2e-obsidian/scripts/partial-startup-file-failure.ts", + "test:e2e:obsidian:startup-content-check": "tsx test/e2e-obsidian/scripts/startup-content-check.ts", "test:e2e:obsidian:startup-scan": "tsx test/e2e-obsidian/scripts/startup-scan.ts", "test:e2e:obsidian:stale-file-restart": "tsx test/e2e-obsidian/scripts/stale-file-restart.ts", "test:e2e:obsidian:folder-batch": "tsx test/e2e-obsidian/scripts/folder-batch.ts", diff --git a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts index 1ba23bb5..59e09c17 100644 --- a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts +++ b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts @@ -23,6 +23,7 @@ export function createAdvancedSettingSpecGroups({ control: { type: "dropdown", options: () => ChunkAlgorithmNames }, }, { key: "customChunkSize", control: { type: "number", min: 0, allowZero: true } }, + { key: "strictContentCheckOnStartup", control: { type: "toggle" } }, { key: "pathNormalizationVersion", holdValue: true, diff --git a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts index 859727cd..85597fbf 100644 --- a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts +++ b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts @@ -43,7 +43,7 @@ describe("paneAdvanced", () => { }; paneAdvanced.call(host as never, {} as HTMLElement, { addPanel } as never); - await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(10)); + await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(11)); expect(addPanel.mock.calls.map(([, heading]) => heading)).toEqual([ "Memory cache", @@ -56,6 +56,7 @@ describe("paneAdvanced", () => { panelElements.get("Local Database Tweak"), panelElements.get("Local Database Tweak"), panelElements.get("Local Database Tweak"), + panelElements.get("Local Database Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), @@ -67,6 +68,7 @@ describe("paneAdvanced", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "strictContentCheckOnStartup", "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", diff --git a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts index 62958bc4..7e957aeb 100644 --- a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts @@ -17,7 +17,7 @@ const rangeMessages = { }; describe("Advanced setting specifications", () => { - it("lists only the ten controls currently exposed by the Advanced page", () => { + it("lists only the eleven controls currently exposed by the Advanced page", () => { const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => true }); expect(groups.map(({ heading }) => heading)).toEqual([ @@ -30,6 +30,7 @@ describe("Advanced setting specifications", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "strictContentCheckOnStartup", "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", @@ -40,6 +41,17 @@ describe("Advanced setting specifications", () => { ]); }); + it("exposes strict content checking as an ungated Advanced toggle", () => { + const spec = createAdvancedSettingSpecGroups({ isCouchDB: () => false }) + .find(({ heading }) => heading === "Local Database Tweak") + ?.items.find(({ key }) => key === "strictContentCheckOnStartup"); + if (!spec) throw new Error("Strict content checking is not exposed on the Advanced page"); + + expect(spec.control).toEqual({ type: "toggle" }); + expect(spec.visible).toBeUndefined(); + expect(spec.disabled).toBeUndefined(); + }); + it("keeps path identity and the four remote chunk controls CouchDB-only", () => { let couchDB = false; const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => couchDB }); diff --git a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts index 32bf0e82..a2605f41 100644 --- a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts @@ -86,6 +86,7 @@ describe("settings page catalogue", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "strictContentCheckOnStartup", "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", diff --git a/src/modules/main/ModuleLiveSyncMain.ts b/src/modules/main/ModuleLiveSyncMain.ts index 7cdc656a..228f75db 100644 --- a/src/modules/main/ModuleLiveSyncMain.ts +++ b/src/modules/main/ModuleLiveSyncMain.ts @@ -19,6 +19,22 @@ import { initialiseWorkerModule } from "@vrtmrz/livesync-commonlib/compat/worker import { manifestVersion, packageVersion } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvVars"; import { VaultScanResults } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; +const OBSIDIAN_PLATFORMS = new Set([ + "android-app", + "ios", + "macos", + "mobile-app", + "mobile", + "safari", + "desktop", + "desktop-app", + "unknown-obsidian", +]); + +function isObsidianPlatform(platform: string): boolean { + return OBSIDIAN_PLATFORMS.has(platform); +} + export class ModuleLiveSyncMain extends AbstractModule { async _onLiveSyncReady() { if (!(await this.core.services.appLifecycle.onLayoutReady())) return false; @@ -44,13 +60,20 @@ export class ModuleLiveSyncMain extends AbstractModule { } } // Ordinary start-up may continue when individual files could not be - // processed. Explicit Fetch and Rebuild flows retain the strict default. + // processed. Explicit Fetch and Rebuild flows remain strict. const { ignoreSuspending = false, continueOnFileFailure = true } = this.core.startupDatabaseOptions; + // Only ordinary Obsidian start-up honours the opt-out. Keep other hosts + // and explicit operation modes strict. + const strictContentCheck = + !isObsidianPlatform(this.services.API.getPlatform()) || + this.settings.strictContentCheckOnStartup === true || + this.core.startupDatabaseOptions.continueOnFileFailure === false; const initialisationResult = await this.services.databaseEvents.initialiseDatabase( false, false, ignoreSuspending, - continueOnFileFailure + continueOnFileFailure, + { strictContentCheck } ); if (initialisationResult === VaultScanResults.FAILED) { this._log($msg("Ui.Common.LocalDatabaseInitialisationFailed"), LOG_LEVEL_NOTICE); diff --git a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts index bbb744ce..be9a9014 100644 --- a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts +++ b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts @@ -90,9 +90,11 @@ describe("ModuleLiveSyncMain", () => { }, }, services: { + API: { getPlatform: () => "desktop" }, databaseEvents: { initialiseDatabase }, }, settings: { + strictContentCheckOnStartup: false, suspendFileWatching: false, suspendParseReplicationResult: false, }, @@ -102,7 +104,7 @@ describe("ModuleLiveSyncMain", () => { const result = await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); expect(result).toBe(false); - expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true); + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true, { strictContentCheck: false }); expect(log).toHaveBeenCalledWith("Ui.Common.LocalDatabaseInitialisationFailed", LOG_LEVEL_NOTICE); }); @@ -120,6 +122,7 @@ describe("ModuleLiveSyncMain", () => { services: { appLifecycle }, }, services: { + API: { getPlatform: () => "desktop" }, appLifecycle, control: { applySettings: vi.fn(async () => undefined) }, databaseEvents: { initialiseDatabase }, @@ -152,6 +155,7 @@ describe("ModuleLiveSyncMain", () => { }, }, services: { + API: { getPlatform: () => "desktop" }, databaseEvents: { initialiseDatabase }, }, settings: { @@ -163,6 +167,96 @@ describe("ModuleLiveSyncMain", () => { await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); - expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false); + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false, { strictContentCheck: true }); + }); + + it("passes the strict content check setting for ordinary Obsidian start-up", async () => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: {}, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + API: { getPlatform: () => "desktop" }, + databaseEvents: { initialiseDatabase }, + }, + settings: { + strictContentCheckOnStartup: true, + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true, { strictContentCheck: true }); + }); + + it.each(["server", "browser"])( + "keeps the strict content check on non-Obsidian host start-up (%s)", + async (platform) => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: true }, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + API: { getPlatform: () => platform }, + databaseEvents: { initialiseDatabase }, + }, + settings: { + strictContentCheckOnStartup: false, + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true, { strictContentCheck: true }); + } + ); + + it("keeps explicit recovery startup options strict when the setting is off", async () => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: { + ignoreSuspending: false, + continueOnFileFailure: false, + }, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + API: { getPlatform: () => "desktop" }, + databaseEvents: { initialiseDatabase }, + }, + settings: { + strictContentCheckOnStartup: false, + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, false, { strictContentCheck: true }); }); }); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index ae4790b3..a8355261 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -100,6 +100,8 @@ The same scenario checks the five actions in Reporting Tools, the separate Check `startup-integrity` uses the registered Hatch integrity-check event with real Obsidian storage and a local database. It verifies that healthy files are inspected without reading their contents, that a controlled stale stat followed by matching real bytes does not produce a repair dialogue, and that a changed final stat leaves the check incomplete until a settled retry. The stat variations are injected; this scenario exercises the checker used during start-up rather than restarting the process. Natural configured start-up remains covered by `startup-scan`. +`startup-content-check` needs no remote service. It pre-creates a configurable set of Vault files, lets the plug-in populate the real local database and content provenance, then makes a same-size offline edit with the exact recorded raw mtime. A natural default-off cold restart must preserve the Vault edit without changing its database revision or branch; an explicit `scanVault(true)` must then detect it. After saving strict content checking as enabled, a second natural cold restart must detect another same-stat offline edit and update the current branch and provenance. The fixture reports a clear precision mismatch if the file system cannot restore the exact recorded mtime. Output includes the `main.js` SHA-256 and elapsed start-up timings; launch-to-core-ready timings include Obsidian and runner start-up, rather than measuring only the database scan. Run it with `npm run test:e2e:obsidian:focused -- startup-content-check`; `E2E_STARTUP_CHECK_FILE_COUNT` defaults to 20 and can be set to 5000 for a larger startup measurement. + `third-party-chunks` reflects synthetic externally generated text through the installed Commonlib and the real Obsidian file handler. It supplies a split surrogate pair at the file-access read boundary, with size checking enabled, then verifies exact Vault and on-disk text, unchanged Metadata, and matching persisted provenance. Complete-pair text and a binary attachment provide controls. The underlying Metadata and Chunks are real local database entries; this does not establish that CouchDB or a particular third-party application generates this chunk layout. No remote service is required. `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 8e394ccd..ffb805ed 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -38,6 +38,7 @@ const focusedScenarios = new Set([ "partial-startup-file-failure", "startup-scan", "startup-integrity", + "startup-content-check", "stale-file-restart", "folder-batch", "setup-uri-workflow", diff --git a/test/e2e-obsidian/scripts/startup-content-check.ts b/test/e2e-obsidian/scripts/startup-content-check.ts new file mode 100644 index 00000000..b648d166 --- /dev/null +++ b/test/e2e-obsidian/scripts/startup-content-check.ts @@ -0,0 +1,836 @@ +/** Checks the default and strict body checks across natural Obsidian restarts. */ +import { createHash } from "node:crypto"; +import { mkdir, readFile, stat, utimes, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { captureObsidianElement, captureObsidianPage, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +const targetPath = "startup-content-check-00000.md"; +const versions = [ + "Startup content check fixture body A\n", + "Startup content check fixture body B\n", + "Startup content check fixture body C\n", +] as const; + +type TargetState = { + rev: string; + rawMtime: number | null; + metadataMtime: number; + size: number; + documentSize: number; + dbContent: string; + vaultContent: string; + branchCount: number; + conflictCount: number; + provenanceRevision: string | null; + contentHash: string | null; + contentSize: number | null; + fileMtime: number | null; + adapterMtime: number | null; + storageMtime: number | null; + adapterSize: number | null; + storageSize: number | null; +}; + +type InitialState = { + fileCount: number; + indexedCount: number; + conflictCount: number; + pendingCount: number; + strictContentCheckOnStartup: boolean; + targetProvenanceValid: boolean; + target: TargetState; +}; + +type ManualScanState = TargetState & { + scanResult: boolean | string; + scanMs: number; + strictContentCheckOnStartup: boolean; +}; + +type StartupFailureCore = { + services?: { + database?: { isDatabaseReady?: () => boolean }; + appLifecycle?: { + isReady?: () => boolean; + isSuspended?: () => boolean; + getUnresolvedMessages?: () => Promise; + }; + }; +}; + +type StartupFailureGlobal = typeof globalThis & { + app?: { + commands?: { executeCommandById?: (commandId: string) => boolean }; + plugins?: { plugins?: Record }; + }; +}; + +const sensitiveDiagnosticLine = + /security seed|passphrase|password|credential|secret|access.?key|jwt.?key|authori[sz]ation|obsidian:\/\/setuplivesync|sls\+/iu; + +function redactDiagnosticLine(line: string): string { + if (sensitiveDiagnosticLine.test(line)) return "[REDACTED SENSITIVE LOG LINE]"; + return line.replace(/\bhttps?:\/\/[^/\s:@]+:[^@\s/]+@/giu, "https://[REDACTED]@"); +} + +function requireCondition(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function assertEqual(actual: unknown, expected: unknown, message: string): void { + if (actual !== expected) { + throw new Error(message + "\nExpected: " + String(expected) + "\nActual: " + String(actual)); + } +} + +function fileCountFromEnvironment(): number { + const raw = process.env.E2E_STARTUP_CHECK_FILE_COUNT ?? "20"; + if (!/^[1-9]\d*$/.test(raw)) { + throw new Error("E2E_STARTUP_CHECK_FILE_COUNT must be a positive integer; received " + raw + "."); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value > 5000) { + throw new Error("E2E_STARTUP_CHECK_FILE_COUNT must be between 1 and 5000; received " + raw + "."); + } + return value; +} + +function fixturePath(index: number): string { + return "startup-content-check-" + String(index).padStart(5, "0") + ".md"; +} + +function fixtureContent(index: number): string { + if (index === 0) return versions[0]; + return "Startup content check fixture " + String(index).padStart(5, "0") + "\n"; +} + +function sha256(content: string): string { + return "sha256:" + createHash("sha256").update(content).digest("hex"); +} + +function assertExactMtime(state: TargetState, label: string): void { + const expected = state.rawMtime; + if (expected === null || !Number.isFinite(expected)) { + throw new Error( + "Fixture precision mismatch during " + label + ": persisted provenance raw mtime was unavailable." + ); + } + for (const [name, actual] of [ + ["TFile", state.fileMtime], + ["Vault adapter", state.adapterMtime], + ["StorageAccess", state.storageMtime], + ] as const) { + if (actual !== expected) { + throw new Error( + "Fixture precision mismatch during " + + label + + ": " + + name + + " mtime " + + String(actual) + + " did not exactly match persisted raw mtime " + + expected + + "." + ); + } + } + assertEqual(state.metadataMtime, expected, label + ": Metadata and raw mtimes differed."); +} + +function requireRecordedMtime(state: TargetState, label: string): number { + if (state.rawMtime === null || !Number.isFinite(state.rawMtime)) { + throw new Error( + "Fixture precision mismatch before " + label + ": persisted provenance raw mtime was unavailable." + ); + } + return state.rawMtime; +} + +async function createFixtures(vaultPath: string, fileCount: number, fixtureMtime: number): Promise { + const paths = Array.from({ length: fileCount }, (_, index) => fixturePath(index)); + const exactMtime = new Date(fixtureMtime); + const batchSize = 100; + for (let offset = 0; offset < paths.length; offset += batchSize) { + await Promise.all( + paths.slice(offset, offset + batchSize).map(async (path, index) => { + const fileIndex = offset + index; + const fullPath = join(vaultPath, path); + await writeFile(fullPath, fixtureContent(fileIndex), "utf-8"); + await utimes(fullPath, exactMtime, exactMtime); + const created = await stat(fullPath); + if (created.mtimeMs !== fixtureMtime) { + throw new Error( + "Fixture precision mismatch before bootstrap: filesystem mtimeMs " + + created.mtimeMs + + " did not exactly match requested integer-millisecond mtime " + + fixtureMtime + + "." + ); + } + }) + ); + } + return paths; +} + +async function restoreOfflineEdit( + vaultPath: string, + content: string, + expectedMtime: number, + expectedSize: number, + label: string +): Promise { + const fullPath = join(vaultPath, targetPath); + if (Buffer.byteLength(content, "utf-8") !== expectedSize) { + throw new Error(label + ": the offline edit did not preserve the recorded byte size."); + } + if (!Number.isFinite(expectedMtime)) { + throw new Error(label + ": the persisted raw mtime was not a finite number."); + } + await writeFile(fullPath, content, "utf-8"); + const exactMtime = new Date(expectedMtime); + await utimes(fullPath, exactMtime, exactMtime); + const restored = await stat(fullPath); + if (restored.mtimeMs !== expectedMtime) { + throw new Error( + "Fixture precision mismatch before " + + label + + ": filesystem mtimeMs " + + restored.mtimeMs + + " did not exactly match persisted raw mtime " + + expectedMtime + + "." + ); + } + assertEqual(restored.size, expectedSize, label + ": the restored fixture byte size differed from Metadata."); +} + +async function main(): Promise { + const fileCount = fileCountFromEnvironment(); + if (fileCount >= 1000) { + process.env.E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS ??= "120000"; + process.env.E2E_OBSIDIAN_READY_TIMEOUT_MS ??= "120000"; + } + for (const content of versions) { + assertEqual( + Buffer.byteLength(content, "utf-8"), + Buffer.byteLength(versions[0], "utf-8"), + "The same-stat offline edit fixtures must have identical byte lengths." + ); + } + + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const vault = await createTemporaryVault("obsidian-startup-content-check-"); + const fixtureMtime = Math.floor(Date.now()); + const sessionState: { current: ObsidianLiveSyncSession | undefined } = { current: undefined }; + const pluginData = createE2eCouchDbPluginData( + // Replication is disabled; the scenario uses only the persisted local database. + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "startup-content-check-local" }, + { + deviceAndVaultName: "startup-content-check-device", + strictContentCheckOnStartup: false, + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + } + ); + + async function launch(pluginStartup: "controlled" | "natural", label: string, bootstrap = false): Promise { + console.log("Starting " + label + " with the real Obsidian process."); + const startedAt = Date.now(); + const options = { + binary, + cliBinary, + vault, + pluginStartup, + ...(bootstrap + ? { + pluginData, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + } + : {}), + }; + const currentSession = await startObsidianLiveSyncSession(options); + sessionState.current = currentSession; + const readiness = await waitForLiveSyncCoreReady(cliBinary, currentSession.cliEnv); + requireCondition( + readiness.databaseReady && readiness.appReady, + "Self-hosted LiveSync did not reach core readiness: " + JSON.stringify(readiness) + ); + const elapsedMs = Date.now() - startedAt; + console.log("Reached LiveSync core readiness for " + label + " in " + elapsedMs + " ms."); + return elapsedMs; + } + + async function evaluate(body: string): Promise { + const currentSession = sessionState.current; + if (!currentSession) throw new Error("The Obsidian session is not running."); + const script = [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync']?.core;", + "const require=(condition,message)=>{if(!condition)throw new Error(message);};", + "const contentOf=value=>Array.isArray(value)?value.join(''):value;", + "const targetState=async path=>{", + " require(core,'Self-hosted LiveSync did not load');", + " const meta=await core.localDatabase.getDBEntryMeta(path,{conflicts:true},true);", + " require(meta&&meta._rev,'The target Metadata was missing');", + " const document=await core.localDatabase.getDBEntryFromMeta(meta);", + " require(document&&document.data!==undefined,'The target Chunks could not be loaded');", + " const dbContent=contentOf(document.data);", + " require(typeof dbContent==='string','The text fixture did not load as text');", + " const file=app.vault.getAbstractFileByPath(path);", + " require(file&&file.stat,'The target Vault file was missing');", + " const adapterStat=await app.vault.adapter.stat(path);", + " const storage=core.serviceModules.storageAccess;", + " const storageStat=await storage.statHidden(path);", + " const provenance=await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(path);", + " return {rev:meta._rev,rawMtime:provenance?.observedStorageMtime??null,metadataMtime:Number(meta.mtime),", + " size:Number(meta.size),documentSize:Number(document.size),dbContent,", + " vaultContent:await app.vault.adapter.read(path),", + " branchCount:1+(meta._conflicts??[]).length,conflictCount:(meta._conflicts??[]).length,", + " provenanceRevision:provenance?.revision??null,contentHash:provenance?.contentHash??null,", + " contentSize:provenance?.contentSize??null,fileMtime:file.stat.mtime??null,", + " adapterMtime:adapterStat?.mtime??null,storageMtime:storageStat?.mtime??null,", + " adapterSize:adapterStat?.size??null,storageSize:storageStat?.size??null};", + "};", + body, + "})()", + ].join("\n"); + return await evalObsidianJson(cliBinary, script, currentSession.cliEnv); + } + + async function stopSession(): Promise { + const currentSession = sessionState.current; + if (!currentSession) return; + await currentSession.app.stop(); + sessionState.current = undefined; + } + + async function captureSettingScreenshots(): Promise<{ advancedPage: string; settingRow: string }> { + const currentSession = sessionState.current; + if (!currentSession) throw new Error("The Obsidian session is not running."); + const port = currentSession.remoteDebuggingPort; + const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e"; + const advancedPagePath = join(outputDirectory, "startup-content-check-setting-location.png"); + await mkdir(outputDirectory, { recursive: true }); + + await withObsidianPage(port, async (page) => { + const navigator = await openLiveSyncSettings(page); + const advanced = await navigator.openPage("Advanced"); + const section = advanced.getByText("Local Database Tweak", { exact: true }).first(); + await section.waitFor({ state: "visible" }); + await section.scrollIntoViewIfNeeded(); + const row = advanced.locator(".setting-item").filter({ + has: navigator.page.getByText("Strict content check on startup", { exact: true }), + }); + await row.waitFor({ state: "visible" }); + await row + .getByText( + "Compare file contents even when modification time and size match. This can slow startup in large vaults.", + { exact: true } + ) + .waitFor({ state: "visible" }); + assertEqual( + await row.locator(".checkbox-container.is-enabled").count(), + 0, + "The screenshot fixture toggle was not visibly OFF." + ); + await navigator.dialogue.screenshot({ + path: advancedPagePath, + animations: "disabled", + style: ".notice-container { visibility: hidden !important; }", + }); + }); + + const settingRowPath = await captureObsidianElement( + port, + "startup-content-check-setting-row.png", + async (page) => { + const navigator = await openLiveSyncSettings(page); + const advanced = await navigator.openPage("Advanced"); + const row = advanced.locator(".setting-item").filter({ + has: navigator.page.getByText("Strict content check on startup", { exact: true }), + }); + await row.waitFor({ state: "visible" }); + await row + .getByText( + "Compare file contents even when modification time and size match. This can slow startup in large vaults.", + { exact: true } + ) + .waitFor({ state: "visible" }); + assertEqual( + await row.locator(".checkbox-container.is-enabled").count(), + 0, + "The cropped screenshot fixture toggle was not visibly OFF." + ); + return row; + } + ); + + await withObsidianPage(port, async (page) => { + const navigator = await openLiveSyncSettings(page); + await navigator.close(); + }); + return { advancedPage: advancedPagePath, settingRow: settingRowPath }; + } + + async function captureFailureEvidence(error: unknown): Promise { + const currentSession = sessionState.current; + if (!currentSession) return; + const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e"; + const visibleScreenshotPath = join(outputDirectory, "startup-content-check-failure-visible.png"); + const screenshotPath = join(outputDirectory, "startup-content-check-failure.png"); + const snapshotPath = join(outputDirectory, "startup-content-check-failure.json"); + await mkdir(outputDirectory, { recursive: true }); + + const processOutput = currentSession.app.output(); + const snapshot: { + capturedAt: string; + failure: string; + screenshotPath: string; + visibleScreenshotPath: string; + rendererState: unknown; + recentLog: string[]; + logCaptureError?: string; + processOutput: { stdout: string; stderr: string }; + } = { + capturedAt: new Date().toISOString(), + failure: redactDiagnosticLine( + (error instanceof Error ? (error.stack ?? error.message) : String(error)).slice(0, 2000) + ), + screenshotPath, + visibleScreenshotPath, + rendererState: { unavailable: true }, + recentLog: [], + processOutput: { + stdout: processOutput.stdout + .split(/\r?\n/u) + .slice(-30) + .map(redactDiagnosticLine) + .join("\n") + .slice(-3000), + stderr: processOutput.stderr + .split(/\r?\n/u) + .slice(-30) + .map(redactDiagnosticLine) + .join("\n") + .slice(-3000), + }, + }; + + const capturedScreenshot = await captureObsidianPage( + currentSession.remoteDebuggingPort, + "startup-content-check-failure.png", + async (page) => { + await page.screenshot({ path: visibleScreenshotPath, fullPage: true }); + try { + snapshot.rendererState = await page.evaluate(async () => { + const host = globalThis as StartupFailureGlobal; + const core = host.app?.plugins?.plugins?.["obsidian-livesync"]?.core; + let unresolvedMessages: string[] = []; + let unresolvedMessagesError: string | undefined; + try { + const messages = await core?.services?.appLifecycle?.getUnresolvedMessages?.(); + unresolvedMessages = (messages ?? []) + .flat() + .filter((message) => message !== undefined && message !== null) + .map((message) => String(message).slice(0, 300)) + .slice(-20); + } catch (diagnosticError) { + unresolvedMessagesError = + diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError); + } + return { + url: location.href, + title: document.title, + bodyText: (document.body?.innerText ?? "").slice(0, 1200), + notices: Array.from(document.querySelectorAll(".notice")) + .slice(0, 10) + .map((notice) => (notice.textContent ?? "").trim().slice(0, 300)), + coreAvailable: core !== undefined, + databaseReady: core?.services?.database?.isDatabaseReady?.() ?? false, + appReady: core?.services?.appLifecycle?.isReady?.() ?? false, + suspended: core?.services?.appLifecycle?.isSuspended?.() ?? false, + unresolvedMessages, + unresolvedMessagesError, + }; + }); + } catch (snapshotError) { + snapshot.rendererState = { + error: snapshotError instanceof Error ? snapshotError.message : String(snapshotError), + }; + } + + try { + const opened = await page.evaluate( + (commandId) => + (globalThis as StartupFailureGlobal).app?.commands?.executeCommandById?.(commandId) === + true, + "obsidian-livesync:view-log" + ); + if (!opened) throw new Error("The Show log command was not registered."); + const logPane = page.locator(".logpane"); + await logPane.waitFor({ state: "visible", timeout: 3000 }); + snapshot.recentLog = (await logPane.locator(".log pre").allTextContents()) + .slice(-30) + .map((line) => redactDiagnosticLine(line.slice(0, 800))); + } catch (logError) { + snapshot.logCaptureError = logError instanceof Error ? logError.message : String(logError); + } + await writeFile(snapshotPath, JSON.stringify(snapshot, null, 2) + "\n", "utf8"); + console.error("Startup content check failure diagnostics: " + snapshotPath); + } + ); + console.error("Startup content check failure screenshot: " + capturedScreenshot); + console.error("Startup content check visible-page screenshot: " + visibleScreenshotPath); + } + + const countJson = JSON.stringify(fileCount); + const targetPathJson = JSON.stringify(targetPath); + + try { + await createFixtures(vault.path, fileCount, fixtureMtime); + // The controlled bootstrap establishes the isolated profile and compatibility marker. + // The two compared starts below are natural Obsidian cold restarts. + const bootstrapMs = await launch("controlled", "controlled bootstrap", true); + const initial = await evaluate( + [ + "require(core,'Self-hosted LiveSync did not load');", + "require(core.services.setting.currentSettings().strictContentCheckOnStartup===false,", + " 'The bootstrap did not use strictContentCheckOnStartup=false');", + "require(await core.services.fileProcessing.commitPendingFileEvents(),", + " 'The bootstrap local file events did not flush');", + "const fixturePaths=new Set(Array.from({length:", + countJson, + "},(_,index)=>'startup-content-check-'+String(index).padStart(5,'0')+'.md'));", + "let indexedCount=0,conflictCount=0;", + "for await(const meta of core.localDatabase.findAllNormalDocs({conflicts:true})||[]){", + " if(!fixturePaths.has(meta.path))continue;", + " indexedCount++;", + " if((meta._conflicts??[]).length>0)conflictCount++;", + "}", + "const pending=await core.serviceModules.storageAccess.vaultManager.adapter.persistence.loadSnapshot()??[];", + "const target=await targetState(", + targetPathJson, + ");", + "return JSON.stringify({fileCount:", + countJson, + ",indexedCount,conflictCount,pendingCount:pending.length,", + "strictContentCheckOnStartup:core.services.setting.currentSettings().strictContentCheckOnStartup===true,", + "targetProvenanceValid:target.provenanceRevision===target.rev&&", + " typeof target.contentHash==='string'&&target.contentHash!==''&&target.contentSize===target.size,", + "target});", + ].join("\n") + ); + assertEqual(initial.fileCount, fileCount, "The reported initial fixture count differed."); + assertEqual(initial.indexedCount, fileCount, "The pre-created files were not all indexed on bootstrap."); + assertEqual(initial.conflictCount, 0, "The initial fixture unexpectedly contained conflict branches."); + assertEqual(initial.pendingCount, 0, "Pending storage work remained before the cold restart."); + assertEqual( + initial.strictContentCheckOnStartup, + false, + "The initial strict startup setting was unexpectedly enabled." + ); + assertEqual( + initial.targetProvenanceValid, + true, + "The target did not have current persisted content provenance." + ); + assertEqual( + initial.target.rawMtime, + fixtureMtime, + "The initial provenance did not retain the fixture's exact integer-millisecond mtime." + ); + assertEqual(initial.target.dbContent, versions[0], "The initial target body was not stored."); + assertEqual(initial.target.vaultContent, versions[0], "The initial target Vault body differed."); + assertEqual(initial.target.documentSize, initial.target.size, "The initial Chunk size differed from Metadata."); + assertEqual( + initial.target.adapterSize, + initial.target.size, + "The initial adapter byte size differed from Metadata." + ); + assertEqual( + initial.target.storageSize, + initial.target.size, + "The initial StorageAccess byte size differed from Metadata." + ); + assertEqual( + initial.target.contentHash, + sha256(versions[0]), + "The initial provenance hash did not match the body." + ); + assertEqual( + initial.target.contentSize, + Buffer.byteLength(versions[0], "utf-8"), + "The initial provenance byte size was wrong." + ); + assertEqual( + initial.target.provenanceRevision, + initial.target.rev, + "Initial provenance did not match the current branch." + ); + assertEqual(initial.target.conflictCount, 0, "The initial target had a conflict branch."); + assertEqual(initial.target.branchCount, 1, "The initial target did not have exactly one branch."); + assertExactMtime(initial.target, "bootstrap"); + + await stopSession(); + await restoreOfflineEdit( + vault.path, + versions[1], + requireRecordedMtime(initial.target, "default-off cold restart"), + initial.target.size, + "default-off cold restart" + ); + + const ordinaryStartupMs = await launch("natural", "default-off natural cold restart"); + const ordinary = await evaluate( + [ + "const state=await targetState(", + targetPathJson, + ");", + "const pending=await core.serviceModules.storageAccess.vaultManager.adapter.persistence.loadSnapshot()??[];", + "return JSON.stringify({...state,pendingCount:pending.length,strictContentCheckOnStartup:", + "core.services.setting.currentSettings().strictContentCheckOnStartup===true});", + ].join("\n") + ); + assertEqual( + ordinary.strictContentCheckOnStartup, + false, + "The natural default-off restart enabled strict body checks." + ); + assertEqual(ordinary.pendingCount, 0, "Pending storage work remained during the default-off restart."); + assertEqual(ordinary.rev, initial.target.rev, "The default-off restart changed the local Metadata revision."); + assertEqual(ordinary.dbContent, versions[0], "The default-off restart changed the local database body."); + assertEqual(ordinary.vaultContent, versions[1], "The default-off restart overwrote the offline Vault edit."); + assertEqual(ordinary.size, initial.target.size, "The default-off restart changed the Metadata byte size."); + assertEqual(ordinary.documentSize, initial.target.size, "The default-off restart changed the Chunk byte size."); + assertEqual( + ordinary.adapterSize, + initial.target.size, + "The default-off restart changed the adapter byte size." + ); + assertEqual( + ordinary.storageSize, + initial.target.size, + "The default-off restart changed the StorageAccess byte size." + ); + assertEqual( + ordinary.provenanceRevision, + initial.target.rev, + "The default-off restart changed persisted provenance." + ); + assertEqual( + ordinary.contentHash, + initial.target.contentHash, + "The default-off restart changed the recorded content hash." + ); + assertEqual( + ordinary.contentSize, + initial.target.contentSize, + "The default-off restart changed the recorded byte size." + ); + assertEqual(ordinary.conflictCount, 0, "The default-off restart created a conflict branch."); + assertEqual(ordinary.branchCount, 1, "The default-off restart changed the branch count."); + assertExactMtime(ordinary, "default-off natural restart"); + + const manual = await evaluate( + [ + "const startedAt=Date.now();", + "const scanResult=await core.services.vault.scanVault(true);", + "const scanMs=Date.now()-startedAt;", + "require(scanResult===true,'The explicit strict manual Vault scan did not complete');", + "const state=await targetState(", + targetPathJson, + ");", + "core.settings.strictContentCheckOnStartup=true;", + "await core.services.setting.saveSettingData();", + "require(core.services.setting.currentSettings().strictContentCheckOnStartup===true,", + " 'The strict startup setting did not persist in the active settings');", + "return JSON.stringify({...state,scanResult,scanMs,strictContentCheckOnStartup:", + "core.services.setting.currentSettings().strictContentCheckOnStartup===true});", + ].join("\n") + ); + assertEqual(manual.scanResult, true, "The explicit strict manual scan returned an unexpected result."); + assertEqual(manual.dbContent, versions[1], "The manual strict scan did not detect the same-stat offline edit."); + assertEqual(manual.vaultContent, versions[1], "The manual strict scan changed the Vault body."); + assertEqual(manual.size, initial.target.size, "The manual strict scan changed the Metadata byte size."); + assertEqual(manual.documentSize, initial.target.size, "The manual strict scan changed the Chunk byte size."); + assertEqual(manual.adapterSize, initial.target.size, "The manual strict scan changed the adapter byte size."); + assertEqual( + manual.storageSize, + initial.target.size, + "The manual strict scan changed the StorageAccess byte size." + ); + requireCondition( + manual.rev !== ordinary.rev, + "The manual strict scan did not create a revision for the offline edit." + ); + assertEqual( + manual.provenanceRevision, + manual.rev, + "The manual scan did not record provenance for the detected edit." + ); + assertEqual(manual.contentHash, sha256(versions[1]), "The manual scan recorded the wrong content hash."); + assertEqual(manual.contentSize, initial.target.size, "The manual scan recorded the wrong content size."); + assertEqual(manual.conflictCount, 0, "The manual strict scan created a conflict branch."); + assertEqual(manual.branchCount, 1, "The manual strict scan changed the branch count."); + assertEqual(manual.strictContentCheckOnStartup, true, "The strict startup setting was not saved."); + assertExactMtime(manual, "manual strict scan"); + + await stopSession(); + await restoreOfflineEdit( + vault.path, + versions[2], + requireRecordedMtime(manual, "strict cold restart"), + manual.size, + "strict cold restart" + ); + + const strictStartupMs = await launch("natural", "strict natural cold restart"); + const strict = await evaluate( + [ + "const state=await targetState(", + targetPathJson, + ");", + "const pending=await core.serviceModules.storageAccess.vaultManager.adapter.persistence.loadSnapshot()??[];", + "return JSON.stringify({...state,pendingCount:pending.length,strictContentCheckOnStartup:", + "core.services.setting.currentSettings().strictContentCheckOnStartup===true});", + ].join("\n") + ); + assertEqual(strict.strictContentCheckOnStartup, true, "The natural strict restart did not retain its setting."); + assertEqual(strict.pendingCount, 0, "Pending storage work remained during the strict restart."); + requireCondition( + strict.rev !== manual.rev, + "The strict natural restart did not detect the same-stat offline edit." + ); + assertEqual(strict.dbContent, versions[2], "The strict natural restart did not store the offline Vault body."); + assertEqual(strict.vaultContent, versions[2], "The strict natural restart changed the offline Vault body."); + assertEqual(strict.size, manual.size, "The strict natural restart changed the Metadata byte size."); + assertEqual(strict.documentSize, manual.size, "The strict natural restart changed the Chunk byte size."); + assertEqual(strict.adapterSize, manual.size, "The strict natural restart changed the adapter byte size."); + assertEqual(strict.storageSize, manual.size, "The strict natural restart changed the StorageAccess byte size."); + assertEqual( + strict.provenanceRevision, + strict.rev, + "The strict restart did not persist current file provenance." + ); + assertEqual(strict.contentHash, sha256(versions[2]), "The strict restart recorded the wrong content hash."); + assertEqual(strict.contentSize, manual.size, "The strict restart recorded the wrong byte size."); + assertEqual(strict.conflictCount, 0, "The strict natural restart created a conflict branch."); + assertEqual(strict.branchCount, 1, "The strict natural restart changed the branch count."); + assertExactMtime(strict, "strict natural restart"); + + const screenshotSettings = await evaluate<{ strictContentCheckOnStartup: boolean; useAdvancedMode: boolean }>( + [ + "core.settings.strictContentCheckOnStartup=false;", + "core.settings.useAdvancedMode=true;", + "await core.services.setting.saveSettingData();", + "const settings=core.services.setting.currentSettings();", + "return JSON.stringify({strictContentCheckOnStartup:settings.strictContentCheckOnStartup===true,", + "useAdvancedMode:settings.useAdvancedMode===true});", + ].join("\n") + ); + assertEqual( + screenshotSettings.strictContentCheckOnStartup, + false, + "The screenshot fixture did not reset strict content checking to OFF after the startup assertions." + ); + assertEqual( + screenshotSettings.useAdvancedMode, + true, + "The screenshot fixture did not expose the Advanced page." + ); + const settingScreenshots = await captureSettingScreenshots(); + + console.log( + JSON.stringify( + { + ok: true, + bundleSha256: bundleHash, + fixtureFileCount: fileCount, + localOnly: true, + bootstrapPluginStartup: "controlled", + measuredRestartsPluginStartup: "natural", + observations: { + bootstrap: { + indexedCount: initial.indexedCount, + targetProvenanceValid: initial.targetProvenanceValid, + conflictCount: initial.conflictCount, + pendingCount: initial.pendingCount, + }, + defaultOffRestart: { + revisionUnchanged: ordinary.rev === initial.target.rev, + databaseBodyUnchanged: ordinary.dbContent === versions[0], + offlineVaultBodyPreserved: ordinary.vaultContent === versions[1], + branchCount: ordinary.branchCount, + provenanceRevision: ordinary.provenanceRevision, + }, + manualStrictScan: { + detectedOfflineEdit: manual.dbContent === versions[1], + revisionChanged: manual.rev !== ordinary.rev, + branchCount: manual.branchCount, + provenanceRevision: manual.provenanceRevision, + scanMs: manual.scanMs, + }, + strictRestart: { + detectedOfflineEdit: strict.dbContent === versions[2], + revisionChanged: strict.rev !== manual.rev, + branchCount: strict.branchCount, + provenanceRevision: strict.provenanceRevision, + }, + }, + timing: { + scope: "Bootstrap and natural restart timings run from startObsidianLiveSyncSession() invocation through waitForLiveSyncCoreReady() completion. They include Obsidian launch, Vault open, plug-in startup, and runner readiness; they are not isolated scanner timings. manualStrictScanMs measures only the scanVault(true) call inside the running renderer.", + bootstrapMs, + ordinaryStartupMs, + manualStrictScanMs: manual.scanMs, + strictStartupMs, + }, + settingScreenshots, + }, + null, + 2 + ) + ); + } catch (error) { + try { + await captureFailureEvidence(error); + } catch (captureError) { + console.warn( + "Could not capture startup content check failure evidence: " + + (captureError instanceof Error ? captureError.message : String(captureError)) + ); + } + throw error; + } finally { + try { + await stopSession(); + } finally { + await vault.dispose(); + } + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/updates.md b/updates.md index 2a288762..55a40933 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Changed + +- Ordinary start-up can skip reading file content when the exact modification time, size, and recorded device-local provenance match a conflict-free revision, with no pending file processing. + - **Strict content check on startup** keeps full content comparisons enabled. The setting is disabled by default; an edit which preserves its exact size and modification time can go undetected by this start-up scan. Manual scans and explicit recovery operations remain strict. + ## 1.0.41 26th October, 2026 From 51a1f702a3143c4938378675681324f83a20960b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 03:33:57 +0000 Subject: [PATCH 27/28] Prepare 1.0.42 with optional start-up content checks --- devs.md | 8 +++-- docs/releases/1.0-previews.md | 63 ++++++++++++++++++++++++++++++++++- manifest.json | 2 +- package-lock.json | 18 +++++----- package.json | 4 +-- src/apps/cli/package.json | 2 +- src/apps/webapp/package.json | 2 +- src/apps/webpeer/package.json | 2 +- updates.md | 16 ++++----- versions.json | 3 +- 10 files changed, 92 insertions(+), 28 deletions(-) diff --git a/devs.md b/devs.md index 3786afa6..a9279f14 100644 --- a/devs.md +++ b/devs.md @@ -353,6 +353,7 @@ This policy governs plug-in pre-releases and their accompanying CLI images. For - Add only user-facing changes that help users understand what they gain, what has changed, or what they may need to do after updating. - Avoid listing purely internal refactors, maintenance chores, generated-file changes, and dependency updates unless they affect users; group and label them when they are included. - When preparing a release, replace `## Unreleased` with the target version heading (for example, `## 0.25.81`) and add a fresh empty `## Unreleased` section above it for the next cycle. +- For a stable release, review the complete user-facing changes since the previous stable release. Classify new features and changed behaviour relative to that stable release. Include changes from intervening pre-releases which never became stable, even when their original notes have already moved into a release history. - Review and polish the released section in the release PR before tagging, because the content is embedded into the plug-in and may be reused as the GitHub Release notes. - Keep approximately the five most recent published plug-in versions in the embedded `updates.md`. Move older published sections unchanged into the appropriate release-line archive under `docs/releases/`, and update the history references when rotating them. @@ -372,8 +373,9 @@ For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch- - Validate installation or update through BRAT, and cold-start Obsidian to confirm the plug-in loads. Verify ordinary bidirectional synchronisation and relevant regression scenarios in real Obsidian against the published artefacts, or a build with matching hashes. When automated scenarios establish that coverage, BRAT acceptance need only cover installation or update and start-up; manually check any remaining release-specific behaviour. Record the artefact hashes and validation evidence as described in the [real Obsidian test guide](test/e2e-obsidian/README.md#combined-setup-and-security-regression-checks). - After a hyphenated pre-release passes, keep its release pull request unmerged. Add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`, then close the release pull request only through a separate maintainer action. - After a stable version passes, remove the GitHub pre-release designation and make that exact release the latest stable release. Confirm that the release is no longer a pre-release, then mark its release pull request ready, merge the exact release commit into the selected base branch with a merge commit, and integrate that exact commit through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata. Create the stable CLI tag and publish its `latest` and major-minor image tags, if selected, through a separate maintainer gate. -- If BRAT validation fails, keep the release PR in draft and do not move published tags. Before preparing the next version, add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`. Keep only changes made after that tag under `## Unreleased`. Compare the historical section with `git show :updates.md`; do not merge the failed release PR or describe it as validated. The next release PR can then rotate only the correction notes while preserving the immutable release history. -- Prepare and publish the next patch or pre-release version from that reconciled base. Leave the failed release PR draft until it is deliberately closed as superseded under a separate maintainer action. +- If BRAT validation fails, keep the release PR in draft and leave its published tag and artefacts unchanged. Prepare the correction on a new branch descended from that exact release commit, retaining its existing changes and `versions.json` entry. Prepare the next patch or pre-release version on that branch, with its own version, tag, and release PR. The selected base branch remains unchanged until a successor stable release passes validation and is promoted. +- For the successor stable release, consolidate all user-facing changes since the last stable release, including changes from intervening pre-releases which never became stable. Preserve each published pre-release's exact tagged notes in the appropriate history under `docs/releases/`, and compare the archived section with `git show :updates.md`. The new release notes must describe the whole stable upgrade, including the correction. +- After the successor stable release is validated and promoted, merge its exact release commit and ancestry through the reviewed branch chain. Leave the failed release PR in draft until it is deliberately closed as superseded under a separate maintainer action. - A hyphenated version is rejected unless `prerelease=true`. A stable version staged with `prerelease=true` is rejected unless `publish_cli=false`. ### Release Cheat Sheet @@ -403,7 +405,7 @@ For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch- 9. Validate the published release through BRAT, including start-up, ordinary bidirectional synchronisation, and any release-specific regression scenario. 10. After a hyphenated pre-release passes, keep its release PR unmerged, reconcile its `versions.json` entry and exact release-note section into the selected base branch as metadata only, then close the PR through a separate maintainer action. 11. After a stable version passes, remove the pre-release designation and make the exact release the latest stable release. Confirm that promotion, then mark the release PR ready, merge the exact release commit into the selected base branch, and integrate it through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata, then publish stable CLI tags through a separate maintainer gate if selected. -12. If validation fails, leave the PR in draft and do not move the published tags. Reconcile the published version's `updates.md` section and `versions.json` entry into the base branch as metadata only, then prepare the next patch or pre-release version from the remaining `## Unreleased` entries. +12. If validation fails, leave the PR in draft and keep its published tag and artefacts unchanged. Start a new branch from that release commit, add the correction, and prepare the next patch or pre-release version. Retain the published version's `versions.json` entry and exact archived notes, and consolidate the successor stable release notes from the last stable release. Keep the base branch unchanged until the successor stable release is validated and promoted. ### CLI-only patch releases diff --git a/docs/releases/1.0-previews.md b/docs/releases/1.0-previews.md index 2ae2b583..bba8f0bb 100644 --- a/docs/releases/1.0-previews.md +++ b/docs/releases/1.0-previews.md @@ -1,9 +1,70 @@ # 1.0 preview release history -This document records the opt-in beta and release-candidate builds published before 1.0.0. Most users upgrading from 0.25.83 only need the consolidated [1.0.0 release notes](1.0.md#100). +This document records opt-in pre-release builds in the 1.0 release line. Consolidated notes for stable upgrades are available in the [current change log](../../updates.md) and the [1.0 release history](1.0.md). The prepared `1.0.0-rc.0` tag was not published as a plug-in release and is therefore omitted. +Version 1.0.41 was published for validation and remains a GitHub pre-release. Its changes and the subsequent start-up correction are consolidated in the [1.0.42 release notes](../../updates.md#1042). + +## 1.0.41 + +26th October, 2026 + +### Synchronisation + +#### New Feature + +- Rapid edits can produce fewer intermediate versions on other devices while LiveSync mode remains active. + - Enable **Batch database updates in LiveSync mode** to group successive local edits. + - Disabled by default; each synchronisation mode keeps its own choice. + - Updates reach other devices later. Batching does not detect template completion or prevent genuine concurrent-edit conflicts. +- CouchDB synchronisation no longer looks idle while it is comparing change history. + - A ⏩ icon and comparison count show activity before documents are transferred. + - This count represents history entries, not files remaining to synchronise. +- Status counts are easier to follow, with less movement as values change. + - CSS snippets can hide unwanted items or style individual `livesync-stat-` items, including inactive ones. + +#### Fixed + +- Recent Vault edits remain recoverable after an interrupted local database save. + - This includes edits whose modification times compare equal to the database copy, and pending Hidden File Sync and Customisation Sync changes. +- Manual synchronisation no longer starts before already accepted local changes finish saving to the database. + - Settings application also waits for those saves. Failed saves stay pending and report a failure. +- Text from third-party tools is handled more robustly. + - Characters split across Chunks no longer trigger false corruption warnings. +- Start-up integrity checks use less memory for healthy Vault files. (#1283) +- Start-up checks avoid false repair warnings for files that change during inspection. (#1283) + - This applies when the final size matches the database and there are no conflicts. + +### Customisation Sync + +#### Fixed + +- Applying a multi-file customisation no longer skips files after the list is refreshed. (#1271) +- Choosing **Automatic** now explains a missing Hidden File Sync prerequisite instead of proceeding into a failed initialisation. (#1270) + - Enable and initialise Hidden File Sync on this device first. The current mode is kept when the prerequisite is missing. +- Initialising a whole plug-in in **Automatic** mode no longer omits its selected **MAIN** or **DATA** files. + +### Setup tools + +#### Fixed + +- Setup URI generation can avoid shell-variable collisions and unintended use of the operating-system username. (#817, #1274) + - Use `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE` for connection and encryption values. + - Existing scripts remain supported: the previous names are used when the corresponding prefixed value is unset or blank. + +### Miscellaneous + +- Dependencies have been updated, including security fixes. + +### Acknowledgements + +Thank you for your contributions! + +- @rwbru, for the original Setup URI environment-variable fix ([#817](https://github.com/vrtmrz/obsidian-livesync/pull/817)). +- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts ([#1274](https://github.com/vrtmrz/obsidian-livesync/pull/1274)). +- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib ([#1283](https://github.com/vrtmrz/obsidian-livesync/pull/1283), [Commonlib #184](https://github.com/vrtmrz/livesync-commonlib/pull/184)). + ## 1.0.0-rc.1 27th July, 2026 diff --git a/manifest.json b/manifest.json index efe56799..e1929275 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.41", + "version": "1.0.42", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index 5393a72f..717ba6f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.41", + "version": "1.0.42", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.41", + "version": "1.0.42", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.1", + "@vrtmrz/livesync-commonlib": "0.2.2", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", @@ -4103,9 +4103,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.1.tgz", - "integrity": "sha512-fnQb2FcEG+NqIQX9KhTstea4B7C4VAL0g9ry5rp41qnJDuLaR5zfCz995L+l3Khyrp/9kcPw4VX5RUmV+8GVog==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.2.tgz", + "integrity": "sha512-+nl4ntmRtUmSUuh9mOpicE0U/wpzhSPSQQXy9eg/2xdTBL58jkrQ6bQjbVKEhqUj0bUvg0ESjZm8YX/x+4zXiA==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -11369,7 +11369,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.41-cli", + "version": "1.0.42-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11394,7 +11394,7 @@ }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.41-webapp", + "version": "1.0.42-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11406,7 +11406,7 @@ } }, "src/apps/webpeer": { - "version": "1.0.41-webpeer", + "version": "1.0.42-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, diff --git a/package.json b/package.json index 076bed16..84e21ce5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.41", + "version": "1.0.42", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", @@ -204,7 +204,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.1", + "@vrtmrz/livesync-commonlib": "0.2.2", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 7f73f7f1..43e21f1c 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.41-cli", + "version": "1.0.42-cli", "main": "dist/index.cjs", "type": "module", "scripts": { diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index d0f4f5ce..ccda9b89 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.41-webapp", + "version": "1.0.42-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index c7654ed5..c9047af9 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.41-webpeer", + "version": "1.0.42-webpeer", "type": "module", "scripts": { "dev": "vite", diff --git a/updates.md b/updates.md index 55a40933..0516dada 100644 --- a/updates.md +++ b/updates.md @@ -12,16 +12,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased -### Synchronisation +## 1.0.42 -#### Changed +27th October, 2026 -- Ordinary start-up can skip reading file content when the exact modification time, size, and recorded device-local provenance match a conflict-free revision, with no pending file processing. - - **Strict content check on startup** keeps full content comparisons enabled. The setting is disabled by default; an edit which preserves its exact size and modification time can go undetected by this start-up scan. Manual scans and explicit recovery operations remain strict. - -## 1.0.41 - -26th October, 2026 +Making content checks mandatory in the 1.0.41 pre-release came with a little too much overhead, so 1.0.42 makes them optional. This has turned into a slightly larger release. ### Synchronisation @@ -37,6 +32,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - Status counts are easier to follow, with less movement as values change. - CSS snippets can hide unwanted items or style individual `livesync-stat-` items, including inactive ones. +- We can choose stricter content checks at start-up. + - Enable **Strict content check on startup** to compare file contents even when their modification times and sizes match the recorded local database state. This can increase start-up time in large Vaults. + - The setting is disabled by default. Ordinary start-up can skip content reads when the exact modification time, file size, and recorded database revision match, with no conflicts or pending file changes. + - While the setting is disabled, an edit which preserves its exact size and modification time can go undetected during start-up. Manual scans and explicit recovery operations remain strict. + #### Fixed - Recent Vault edits remain recoverable after an interrupted local database save. diff --git a/versions.json b/versions.json index a68f2a1a..8ad7a151 100644 --- a/versions.json +++ b/versions.json @@ -52,5 +52,6 @@ "1.0.38": "1.7.2", "1.0.39": "1.7.2", "1.0.40": "1.7.2", - "1.0.41": "1.7.2" + "1.0.41": "1.7.2", + "1.0.42": "1.7.2" } From efa3e356b4f529a929df00ac24b1fb62e8ff2180 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 04:11:11 +0000 Subject: [PATCH 28/28] Use plain pull request references in 1.0.42 notes --- updates.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/updates.md b/updates.md index 0516dada..f5551d3b 100644 --- a/updates.md +++ b/updates.md @@ -74,9 +74,9 @@ Making content checks mandatory in the 1.0.41 pre-release came with a little too Thank you for your contributions! -- @rwbru, for the original Setup URI environment-variable fix ([#817](https://github.com/vrtmrz/obsidian-livesync/pull/817)). -- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts ([#1274](https://github.com/vrtmrz/obsidian-livesync/pull/1274)). -- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib ([#1283](https://github.com/vrtmrz/obsidian-livesync/pull/1283), [Commonlib #184](https://github.com/vrtmrz/livesync-commonlib/pull/184)). +- @rwbru, for the original Setup URI environment-variable fix (#817). +- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts (#1274). +- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib (#1283, Commonlib #184). ## 1.0.40