From 75b1cd85b948e2b7688855aeb3dbe8b36ccbaf9c Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 16 Oct 2026 11:12:41 +0000 Subject: [PATCH 01/68] Reflect deletions and renames with Commonlib 0.1.41 Use the published package for deleted Metadata reflection and retain the real Obsidian deletion, rename, and final-check restoration regression coverage. --- package-lock.json | 8 +- package.json | 2 +- test/e2e-obsidian/README.md | 3 +- test/e2e-obsidian/scripts/two-vault-sync.ts | 341 +++++++++++++++++--- updates.md | 7 + 5 files changed, 314 insertions(+), 47 deletions(-) diff --git a/package-lock.json b/package-lock.json index cc64cbc6..503413fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.40", + "@vrtmrz/livesync-commonlib": "0.1.41", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4389,9 +4389,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.40", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.40.tgz", - "integrity": "sha512-GVskcVFoQIpb45rR+QT9Wjtdqc6oAxYz5IB2WeJwusWZ8lTmqVrba+6YD9FTtsbzqbPOyJ/LdQxnET7SUexVtw==", + "version": "0.1.41", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.41.tgz", + "integrity": "sha512-ZJBRM+7HSX5jvbFZ45jlCHRkSwxf0kphN+zh3xs8sF1GQS40NEcGiM/CiXVDEMpE2YUlB5dtrUpdtJYu4l/rHA==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index c725c1f4..cefc2f1c 100644 --- a/package.json +++ b/package.json @@ -191,7 +191,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.40", + "@vrtmrz/livesync-commonlib": "0.1.41", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 4030b4f8..98d1f416 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -212,7 +212,7 @@ Set `E2E_OBSIDIAN_CUSTOM_HTTP_HANDLER=true` when the local Object Storage fixtur `E2E_OBSIDIAN_ONLY_SETUP_URI_GENERATION=true npm run test:e2e:obsidian:focused -- dialog-mounts` runs the Setup URI generation slice in an isolated real Obsidian Vault without a CouchDB service. It checks the Time-bound choice and displayed end time, the generated URI's round trip, Compatible encryption with the original passphrase, and desktop and mobile dialogue layout. Unit and protocol tests cover the time-window boundary without changing the host clock. -`test:e2e:obsidian:two-vault-sync` runs a two-vault note synchronisation workflow. It verifies note creation, update, ordinary rename, a case-only file name change within the same directory, deletion, and a separate encrypted round-trip with Path Obfuscation enabled. Its target-filter scenario confirms that one Vault receives and checkpoints a remote document without reflecting it, restarts with the same profile and filter, and then reflects the stored document after the filter is broadened through the settings service. Directory case changes deliberately remain outside the ordinary workflow because they require directory-aware rename handling. +`test:e2e:obsidian:two-vault-sync` runs a two-vault note synchronisation workflow. It verifies note creation, update, ordinary rename, a case-only file name change within the same directory, deletion, and a separate encrypted round-trip with Path Obfuscation enabled. Creation, update, deletion, and ordinary rename run with `deleteMetadataOfDeletedFiles` explicitly disabled and enabled, using separate paths for each setting. The receiver must contain the original file before deletion or rename, and the source must contain the expected logical deletion or PouchDB tombstone before the receiver reflects it. A further regression check delivers a real CouchDB restoration after the final deleted-winner index read, while ordinary incoming deletion reflection is in progress. The receiving Vault must retain the exact file bytes, emit no deletion event, and record the restored winning revision. The timing gate and its observers are removed after the check. Its target-filter scenario confirms that one Vault receives and checkpoints a remote document without reflecting it, restarts with the same profile and filter, and then reflects the stored document after the filter is broadened through the settings service. Directory case changes deliberately remain outside the ordinary workflow because they require directory-aware rename handling. During focused development, `E2E_OBSIDIAN_ONLY_PARENT_CASE_DELETION=true` runs an Issue #1168 check which renames `parent/test3` to `parent/Test3` through external `node:fs/promises.rename` while Vault A is open, and verifies that the note content, Metadata, and Chunk references are not logically deleted locally, remotely, or after restart. It accepts either case spelling on Vault B, so it does not provide directory rename support or exact case convergence between devices. The natural Obsidian event sequence and resulting database state are evidence for the selected build; an existing-version reproduction result must be reported separately from fixed-version safety evidence. @@ -335,6 +335,7 @@ A short physical-device check is optional when a specific concern remains about - `E2E_OBSIDIAN_FILE_TIMEOUT_MS`: timeout for waiting until a note created through Obsidian's vault API is reflected to disk. - `E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS`: timeout for waiting until Self-hosted LiveSync reports that its core lifecycle and local database are ready. - `E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS`: timeout for waiting until a file appears in Self-hosted LiveSync's local database. +- `E2E_OBSIDIAN_ONLY_METADATA_DELETION=true`: run only the creation, update, deletion, and ordinary rename checks with `deleteMetadataOfDeletedFiles` disabled and enabled, plus restoration during the final deletion check, in `two-vault-sync`. - `E2E_OBSIDIAN_ONLY_PARENT_CASE_DELETION=true`: run only the focused external parent-directory case-rename protection check in `two-vault-sync`. - `E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS`: timeout for waiting until CouchDB contains uploaded E2E documents. - `E2E_OBSIDIAN_REMOTE_ACTIVITY_TIMEOUT_MS`: timeout for an observed remote activity to enter or leave its status boundary; default is 30 seconds. diff --git a/test/e2e-obsidian/scripts/two-vault-sync.ts b/test/e2e-obsidian/scripts/two-vault-sync.ts index 2c2e8990..f5ef0174 100644 --- a/test/e2e-obsidian/scripts/two-vault-sync.ts +++ b/test/e2e-obsidian/scripts/two-vault-sync.ts @@ -10,6 +10,7 @@ import { createCouchDbDatabase, deleteCouchDbDatabase, fetchAllCouchDbDocs, + fetchCouchDbDocument, fetchCouchDbLocalDocs, loadCouchDbConfig, makeUniqueDatabaseName, @@ -33,6 +34,7 @@ import { import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; import { captureObsidianPage } from "../runner/ui.ts"; +import { FILE_REFLECTION_PROVENANCE_STORE } from "../../../src/serviceModules/FileReflectionProvenance.ts"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000"; process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "20000"; @@ -132,6 +134,12 @@ async function pathExists(vaultPath: string, path: string): Promise { } } +function metadataDeletionModePath(path: string, deleteMetadataOfDeletedFiles?: boolean): string { + if (deleteMetadataOfDeletedFiles === undefined) return path; + const prefix = "E2E/two-vault/"; + return `${prefix}delete-metadata-of-deleted-files-${deleteMetadataOfDeletedFiles}/${path.slice(prefix.length)}`; +} + async function installParentCaseEventObserver( cliBinary: string, env: NodeJS.ProcessEnv, @@ -230,6 +238,44 @@ async function waitForPathDeleted( throw new Error(`Timed out waiting for deleted file: ${join(vaultPath, path)}`); } +async function waitForLocalDeletedMetadataMarker( + cliBinary: string, + env: NodeJS.ProcessEnv, + id: string, + path: string, + deleteMetadataOfDeletedFiles: boolean +): Promise { + const timeoutMs = Number(process.env.E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS ?? 15000); + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const id=${JSON.stringify(id)};`, + `const path=${JSON.stringify(path)};`, + `const expectHardDeletion=${JSON.stringify(deleteMetadataOfDeletedFiles)};`, + `const timeoutMs=${JSON.stringify(timeoutMs)};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const deadline=Date.now()+timeoutMs;", + "let state={id,path,rowFound:false,hardDeleted:false,logicallyDeleted:false,documentPresent:false,documentPath:null};", + "while(Date.now()candidate.id===id||candidate.key===id);", + " const doc=row?.doc??null;", + " const hardDeleted=Boolean(row?.value?.deleted||doc?._deleted);", + " const logicallyDeleted=Boolean(doc?.deleted);", + " const documentPresent=Boolean(doc&&!doc._deleted);", + " state={id,path,rowFound:Boolean(row),hardDeleted,logicallyDeleted,documentPresent,documentPath:doc?.path??null};", + " const matches=expectHardDeletion?Boolean(row&&hardDeleted&&!documentPresent):Boolean(row&&!hardDeleted&&logicallyDeleted&&documentPresent&&doc.path===path);", + " if(matches) return JSON.stringify(state);", + " await new Promise((resolve)=>setTimeout(resolve,250));", + "}", + "throw new Error(`Timed out waiting for the expected local Metadata deletion marker: ${JSON.stringify(state)}`);", + "})()", + ].join(""), + env + ); +} + async function waitForExactObsidianPath( cliBinary: string, env: NodeJS.ProcessEnv, @@ -458,6 +504,7 @@ async function startConfiguredSession( binary: context.binary, cliBinary: context.cliBinary, vault, + artifactRoot: process.env.E2E_OBSIDIAN_ARTIFACT_ROOT, startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), pluginData: createE2eCouchDbPluginData(couchDbSettings, persistedOverrides), }); @@ -692,81 +739,269 @@ async function deleteRevisionAndReflect( ); } -async function runCreateUpdateDelete( +async function runMetadataDeletionRestoration( context: RunnerContext, vaultA: TemporaryVault, vaultB: TemporaryVault ): Promise { - const createdContent = "# Created on A\n\nThis note should appear on B.\n"; - let session = await startConfiguredSession(context, vaultA); - await writeNoteViaObsidian(context.cliBinary, session.cliEnv, createPath, createdContent); - await uploadNote(context, session, createPath); + const path = "E2E/two-vault/delete-metadata-of-deleted-files-true/restored-during-deletion.md"; + const content = "# Restored note\n\nThese bytes must survive the final deletion check.\n"; + const overrides = { deleteMetadataOfDeletedFiles: true }; + let session = await startConfiguredSession(context, vaultA, overrides); + await writeNoteViaObsidian(context.cliBinary, session.cliEnv, path, content); + const entry = await uploadNote(context, session, path); + const originalMetadata = await fetchCouchDbDocument(context.couchDb, context.dbName, entry.id); await stopTrackedSession(context, session); - session = await startConfiguredSession(context, vaultB); + session = await startConfiguredSession(context, vaultB, overrides); await syncAndApply(context, session); - const createdOnB = await waitForPathContent(vaultB.path, createPath, (content) => content === createdContent); + await waitForPathContent(vaultB.path, path, (value) => value === content); + await stopTrackedSession(context, session); + + session = await startConfiguredSession(context, vaultA, overrides); + await deleteNoteViaObsidian(context.cliBinary, session.cliEnv, path); + await pushLocalChanges(context.cliBinary, session.cliEnv); + await waitForLocalDeletedMetadataMarker(context.cliBinary, session.cliEnv, entry.id, path, true); + await stopTrackedSession(context, session); + + session = await startConfiguredSession(context, vaultB, overrides); + await evalObsidianJson( + context.cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(path)};`, + `const content=${JSON.stringify(content)};`, + `const originalMetadata=${JSON.stringify(originalMetadata)};`, + `const storeName=${JSON.stringify(FILE_REFLECTION_PROVENANCE_STORE)};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const access=core.serviceModules.databaseFileAccess;", + "const db=core.localDatabase.localDatabase;", + "const store=core.services.keyValueDB.openSimpleStore(storeName);", + "const base=await store.get(path);", + "if(base?.revision!==originalMetadata._rev) throw new Error('The receiving Vault has no matching displayed base revision');", + "const fetchEntryMeta=access.fetchEntryMeta;", + "const allDocs=db.allDocs;", + "let latestReads=0;", + "let restoredRevision='';", + "let gateError='';", + "let gatePhase='index';", + "let indexReads=0;", + "let deletionEvents=0;", + "const ref=app.vault.on('delete',(file)=>{if(file.path===path) deletionEvents++;});", + "try{", + // Keep the production reflection path; control only the arrival of a + // real remote restoration after the final deleted-winner index read. + "access.fetchEntryMeta=async function(file,rev,...args){", + " const target=typeof file==='string'?file:file?.path;", + " if(target===path&&rev===undefined&&++latestReads===2){", + " db.allDocs=async function(...options){", + " indexReads++;", + " try{", + " const snapshot=await allDocs.apply(this,options);", + " if(options[0]?.keys?.length!==1||options[0].keys[0]!==originalMetadata._id) return snapshot;", + " db.allDocs=allDocs;", + " const row=snapshot.rows[0];", + " if(!row?.value?.deleted) throw new Error('The final lookup did not observe a PouchDB tombstone');", + " const replicator=core.services.replicator.getActiveReplicator();", + " gatePhase='connection';", + " const connection=await replicator.connectRemoteCouchDBWithSetting(core.services.setting.currentSettings(),false,false,true);", + " if(typeof connection==='string') throw new Error('Could not open the restoration fixture connection');", + " let restorationSource;", + " try{", + " const remoteRow=(await connection.db.allDocs({keys:[originalMetadata._id]})).rows[0];", + " if(remoteRow?.value?.rev!==row.value.rev||!remoteRow.value.deleted) throw new Error(`Remote deletion differs from the snapshot: ${JSON.stringify({local:row.value,remote:remoteRow?.value})}`);", + " gatePhase='restoration source';", + " restorationSource=new db.constructor(`e2e-deletion-restoration-${Date.now()}`,{adapter:db.adapter});", + " await restorationSource.replicate.from(connection.db,{doc_ids:[originalMetadata._id]});", + " const restored=await restorationSource.put({...originalMetadata,_rev:row.value.rev,deleted:false,mtime:Number(originalMetadata.mtime)+10000});", + " gatePhase='restoration replication';", + " await restorationSource.replicate.to(connection.db,{doc_ids:[originalMetadata._id]});", + " await db.replicate.from(connection.db,{doc_ids:[originalMetadata._id]});", + " const winner=await db.get(originalMetadata._id);", + " if(winner._rev!==restored.rev||winner._deleted||winner.deleted) throw new Error('The restoration did not become the local winning revision');", + " restoredRevision=restored.rev;", + " }finally{try{if(restorationSource) await restorationSource.destroy();}finally{await connection.close();}}", + " return snapshot;", + " }catch(error){gateError=JSON.stringify({phase:gatePhase,name:error?.name,message:error?.message,status:error?.status,reason:error?.reason});throw error;}", + " };", + " }", + " return await fetchEntryMeta.call(this,file,rev,...args);", + "};", + "await core.services.fileProcessing.commitPendingFileEvents();", + "if(!(await core.services.replication.replicate(true))) throw new Error('Finite replication did not complete');", + "const deadline=Date.now()+15000;", + "while(Date.now()0) throw new Error('The restored note was deleted during the final deletion check');", + " const file=app.vault.getAbstractFileByPath(path);", + " if(!file) throw new Error('The restored note is missing from the receiving Vault');", + " const record=await store.get(path);", + " if(record?.revision===restoredRevision){", + " if(await app.vault.read(file)!==content) throw new Error('The restored Vault bytes changed');", + " const winner=await db.get(originalMetadata._id,{conflicts:true});", + " if(winner._rev!==restoredRevision||winner._deleted||winner.deleted||(winner._conflicts??[]).length>0) throw new Error('The restored winning revision was not retained');", + " return JSON.stringify({restoredRevision,deletionEvents});", + " }", + " }", + " await new Promise((resolve)=>setTimeout(resolve,100));", + "}", + "throw new Error(`The restoration race did not settle: ${JSON.stringify({latestReads,indexReads,restoredRevision,deletionEvents})}`);", + "}finally{access.fetchEntryMeta=fetchEntryMeta;db.allDocs=allDocs;app.vault.offref(ref);}", + "})()", + ].join(""), + session.cliEnv + ); + assertEqual( + await readVaultFile(vaultB.path, path), + content, + "Restored file bytes did not survive deletion reflection." + ); + await stopTrackedSession(context, session); + console.log("A restoration replicated during the final deletion check preserved the receiving Vault file."); +} + +async function runCreateUpdateDelete( + context: RunnerContext, + vaultA: TemporaryVault, + vaultB: TemporaryVault, + deleteMetadataOfDeletedFiles?: boolean +): Promise { + const deletionOverrides = deleteMetadataOfDeletedFiles === undefined ? {} : { deleteMetadataOfDeletedFiles }; + const currentCreatePath = metadataDeletionModePath(createPath, deleteMetadataOfDeletedFiles); + const currentUpdatePath = metadataDeletionModePath(updatePath, deleteMetadataOfDeletedFiles); + const currentDeletePath = metadataDeletionModePath(deletePath, deleteMetadataOfDeletedFiles); + const createdContent = "# Created on A\n\nThis note should appear on B.\n"; + let session = await startConfiguredSession(context, vaultA, deletionOverrides); + await writeNoteViaObsidian(context.cliBinary, session.cliEnv, currentCreatePath, createdContent); + await uploadNote(context, session, currentCreatePath); + await stopTrackedSession(context, session); + + session = await startConfiguredSession(context, vaultB, deletionOverrides); + await syncAndApply(context, session); + const createdOnB = await waitForPathContent( + vaultB.path, + currentCreatePath, + (content) => content === createdContent + ); await stopTrackedSession(context, session); assertEqual(createdOnB, createdContent, "Created note did not round-trip to the second vault."); const initialUpdateContent = "# Update target\n\nInitial content.\n"; const updatedContent = "# Update target\n\nUpdated content from A.\n"; - session = await startConfiguredSession(context, vaultA); - await writeNoteViaObsidian(context.cliBinary, session.cliEnv, updatePath, initialUpdateContent); - await uploadNote(context, session, updatePath); - await writeNoteViaObsidian(context.cliBinary, session.cliEnv, updatePath, updatedContent); - await uploadNote(context, session, updatePath); + session = await startConfiguredSession(context, vaultA, deletionOverrides); + await writeNoteViaObsidian(context.cliBinary, session.cliEnv, currentUpdatePath, initialUpdateContent); + await uploadNote(context, session, currentUpdatePath); + await writeNoteViaObsidian(context.cliBinary, session.cliEnv, currentUpdatePath, updatedContent); + await uploadNote(context, session, currentUpdatePath); await stopTrackedSession(context, session); - session = await startConfiguredSession(context, vaultB); + session = await startConfiguredSession(context, vaultB, deletionOverrides); await syncAndApply(context, session); - const updatedOnB = await waitForPathContent(vaultB.path, updatePath, (content) => content === updatedContent); + const updatedOnB = await waitForPathContent( + vaultB.path, + currentUpdatePath, + (content) => content === updatedContent + ); await stopTrackedSession(context, session); assertEqual(updatedOnB, updatedContent, "Updated note content did not round-trip to the second vault."); const deleteContent = "# Delete target\n\nThis note should be removed from B.\n"; - session = await startConfiguredSession(context, vaultA); - await writeNoteViaObsidian(context.cliBinary, session.cliEnv, deletePath, deleteContent); - await uploadNote(context, session, deletePath); + session = await startConfiguredSession(context, vaultA, deletionOverrides); + await writeNoteViaObsidian(context.cliBinary, session.cliEnv, currentDeletePath, deleteContent); + await uploadNote(context, session, currentDeletePath); await stopTrackedSession(context, session); - session = await startConfiguredSession(context, vaultB); + session = await startConfiguredSession(context, vaultB, deletionOverrides); await syncAndApply(context, session); - await waitForPathContent(vaultB.path, deletePath, (content) => content === deleteContent); + await waitForPathContent(vaultB.path, currentDeletePath, (content) => content === deleteContent); await stopTrackedSession(context, session); - session = await startConfiguredSession(context, vaultA); - await deleteNoteViaObsidian(context.cliBinary, session.cliEnv, deletePath); + session = await startConfiguredSession(context, vaultA, deletionOverrides); + const deleteEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, currentDeletePath); + await deleteNoteViaObsidian(context.cliBinary, session.cliEnv, currentDeletePath); await pushLocalChanges(context.cliBinary, session.cliEnv); + if (deleteMetadataOfDeletedFiles !== undefined) { + await waitForLocalDeletedMetadataMarker( + context.cliBinary, + session.cliEnv, + deleteEntry.id, + currentDeletePath, + deleteMetadataOfDeletedFiles + ); + } await stopTrackedSession(context, session); - session = await startConfiguredSession(context, vaultB); + session = await startConfiguredSession(context, vaultB, deletionOverrides); await syncAndApply(context, session); - await waitForPathDeleted(vaultB.path, deletePath); + await waitForPathDeleted(vaultB.path, currentDeletePath); await stopTrackedSession(context, session); - console.log("Two-vault note creation, update, and deletion round-tripped."); + const deletionModeLabel = + deleteMetadataOfDeletedFiles === undefined + ? "" + : ` with deleteMetadataOfDeletedFiles=${deleteMetadataOfDeletedFiles}`; + console.log(`Two-vault note creation, update, and deletion round-tripped${deletionModeLabel}.`); } -async function runRename(context: RunnerContext, vaultA: TemporaryVault, vaultB: TemporaryVault): Promise { +async function runRename( + context: RunnerContext, + vaultA: TemporaryVault, + vaultB: TemporaryVault, + deleteMetadataOfDeletedFiles?: boolean +): Promise { + const deletionOverrides = deleteMetadataOfDeletedFiles === undefined ? {} : { deleteMetadataOfDeletedFiles }; + const currentRenameFromPath = metadataDeletionModePath(renameFromPath, deleteMetadataOfDeletedFiles); + const currentRenameToPath = metadataDeletionModePath(renameToPath, deleteMetadataOfDeletedFiles); const renamedContent = "# Rename target\n\nThis note should move from A to B.\n"; - let session = await startConfiguredSession(context, vaultA); - await writeNoteViaObsidian(context.cliBinary, session.cliEnv, renameFromPath, renamedContent); - await uploadNote(context, session, renameFromPath); - await renameNoteViaObsidian(context.cliBinary, session.cliEnv, renameFromPath, renameToPath); - await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, renameToPath); - await pushLocalChanges(context.cliBinary, session.cliEnv); + let session = await startConfiguredSession(context, vaultA, deletionOverrides); + await writeNoteViaObsidian(context.cliBinary, session.cliEnv, currentRenameFromPath, renamedContent); + await uploadNote(context, session, currentRenameFromPath); await stopTrackedSession(context, session); - session = await startConfiguredSession(context, vaultB); + session = await startConfiguredSession(context, vaultB, deletionOverrides); await syncAndApply(context, session); - const renamedOnB = await waitForPathContent(vaultB.path, renameToPath, (content) => content === renamedContent); - await waitForPathDeleted(vaultB.path, renameFromPath); + const originalOnB = await waitForPathContent( + vaultB.path, + currentRenameFromPath, + (content) => content === renamedContent + ); + await stopTrackedSession(context, session); + assertEqual(originalOnB, renamedContent, "Original note content did not reach the second vault before rename."); + + session = await startConfiguredSession(context, vaultA, deletionOverrides); + const renamedEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, currentRenameFromPath); + await renameNoteViaObsidian(context.cliBinary, session.cliEnv, currentRenameFromPath, currentRenameToPath); + await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, currentRenameToPath); + await pushLocalChanges(context.cliBinary, session.cliEnv); + if (deleteMetadataOfDeletedFiles !== undefined) { + await waitForLocalDeletedMetadataMarker( + context.cliBinary, + session.cliEnv, + renamedEntry.id, + currentRenameFromPath, + deleteMetadataOfDeletedFiles + ); + } + await stopTrackedSession(context, session); + + session = await startConfiguredSession(context, vaultB, deletionOverrides); + await syncAndApply(context, session); + const renamedOnB = await waitForPathContent( + vaultB.path, + currentRenameToPath, + (content) => content === renamedContent + ); + await waitForPathDeleted(vaultB.path, currentRenameFromPath); await stopTrackedSession(context, session); assertEqual(renamedOnB, renamedContent, "Renamed note content did not round-trip to the second vault."); - console.log("Two-vault note rename round-tripped."); + const deletionModeLabel = + deleteMetadataOfDeletedFiles === undefined + ? "" + : ` with deleteMetadataOfDeletedFiles=${deleteMetadataOfDeletedFiles}`; + console.log(`Two-vault note rename round-tripped${deletionModeLabel}.`); } async function runCaseOnlyRename( @@ -1593,32 +1828,56 @@ async function main(): Promise { console.log(`Temporary CouchDB database: ${dbName}`); console.log(`Temporary encrypted CouchDB database: ${encryptedDbName}`); + const onlyMetadataDeletion = process.env.E2E_OBSIDIAN_ONLY_METADATA_DELETION === "true"; const onlyParentCaseDeletion = process.env.E2E_OBSIDIAN_ONLY_PARENT_CASE_DELETION === "true"; const onlyIndependentIds = process.env.E2E_OBSIDIAN_ONLY_INDEPENDENT_IDS === "true"; const onlyDifferentChunkIdKeys = process.env.E2E_OBSIDIAN_ONLY_DIFFERENT_CHUNK_ID_KEYS === "true"; - if (onlyParentCaseDeletion) { + if (onlyMetadataDeletion) { + await runMetadataDeletionRestoration(context, vaultA, vaultB); + for (const deleteMetadataOfDeletedFiles of [false, true]) { + await runCreateUpdateDelete(context, vaultA, vaultB, deleteMetadataOfDeletedFiles); + await runRename(context, vaultA, vaultB, deleteMetadataOfDeletedFiles); + } + } + if (!onlyMetadataDeletion && onlyParentCaseDeletion) { await runParentCaseDeletionProtection(context, vaultA, vaultB); } const onlyConflictOperations = process.env.E2E_OBSIDIAN_ONLY_CONFLICT_OPERATIONS === "true"; - if (!onlyParentCaseDeletion && !onlyConflictOperations && !onlyIndependentIds && !onlyDifferentChunkIdKeys) { - await runCreateUpdateDelete(context, vaultA, vaultB); - await runRename(context, vaultA, vaultB); + if ( + !onlyMetadataDeletion && + !onlyParentCaseDeletion && + !onlyConflictOperations && + !onlyIndependentIds && + !onlyDifferentChunkIdKeys + ) { + await runMetadataDeletionRestoration(context, vaultA, vaultB); + for (const deleteMetadataOfDeletedFiles of [false, true]) { + await runCreateUpdateDelete(context, vaultA, vaultB, deleteMetadataOfDeletedFiles); + await runRename(context, vaultA, vaultB, deleteMetadataOfDeletedFiles); + } await runCaseOnlyRename(context, vaultA, vaultB); if (process.env.E2E_OBSIDIAN_INCLUDE_MARKDOWN_CONFLICT === "true") { await runMarkdownAutoMerge(context, vaultA, vaultB); } } if ( + !onlyMetadataDeletion && !onlyParentCaseDeletion && (onlyConflictOperations || process.env.E2E_OBSIDIAN_INCLUDE_CONFLICT_OPERATIONS === "true") ) { await runConflictTimeStorageOperations(context, vaultA, vaultB); } - if (!onlyParentCaseDeletion && !onlyConflictOperations && !onlyIndependentIds && !onlyDifferentChunkIdKeys) { + if ( + !onlyMetadataDeletion && + !onlyParentCaseDeletion && + !onlyConflictOperations && + !onlyIndependentIds && + !onlyDifferentChunkIdKeys + ) { await runTargetMismatch(context, vaultA, vaultB); await runEncryptedRoundTrip(encryptedContext, encryptedVaultA, encryptedVaultB); } - if (!onlyParentCaseDeletion && !onlyConflictOperations) { + if (!onlyMetadataDeletion && !onlyParentCaseDeletion && !onlyConflictOperations) { if (onlyDifferentChunkIdKeys) { await runDifferentChunkIdKeysRoundTrip(independentContext, independentVaultA, independentVaultB); } else { diff --git a/updates.md b/updates.md index d362d1a4..a47aeb2d 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We can synchronise file deletions and renames when **Do not keep metadata of deleted files.** is enabled. + - A file restored during the final deletion check remains in the receiving Vault. + ### CLI #### Fixed From d31ad562113b61633e95dafa714fd7caeb1280a5 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 08:23:08 +0000 Subject: [PATCH 02/68] Fix tweak recovery for effective encryption settings --- docs/design_docs/tweak_compatibility.md | 7 + .../ModuleResolveMismatchedTweaks.ts | 29 +++- ...ModuleResolveMismatchedTweaks.unit.spec.ts | 124 ++++++++++++++++++ test/e2e-obsidian/README.md | 2 +- .../scripts/tweak-compatibility.ts | 40 +++++- updates.md | 7 + 6 files changed, 199 insertions(+), 10 deletions(-) diff --git a/docs/design_docs/tweak_compatibility.md b/docs/design_docs/tweak_compatibility.md index 93b89c5c..1e9c931b 100644 --- a/docs/design_docs/tweak_compatibility.md +++ b/docs/design_docs/tweak_compatibility.md @@ -65,6 +65,13 @@ for a decision and before applying it. A changed target or changed settings discard that decision. The signature used for this check can contain sensitive configuration and must not be logged, persisted, or included in diagnostics. +Central replication can assess an effective value which differs from a saved +preference, such as internal Metadata encryption whose prerequisites are inactive. +Recovery accepts the current saved or effective snapshot without treating that +representation difference as a settings change. Applying assessed changes retains +saved preferences for settings whose effective values already agree. The full +settings signature still detects a preference changed while awaiting a decision. + The active publication reservation is not held while waiting for the dialogue. Remote writes use the failed attempt's publication guard. Fetch and Rebuild use their existing owners and propagate failure without claiming a successful diff --git a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts index c5889223..0294b7d1 100644 --- a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts +++ b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts @@ -16,11 +16,12 @@ import { AbstractModule } from "@/modules/AbstractModule.ts"; import { $msg, translateIfAvailable } from "@/common/translation"; import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; import type { LiveSyncCore } from "@/main.ts"; -import { REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.const"; +import { REMOTE_COUCHDB, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.const"; import { withOwnedRemoteResource } from "@/common/ownedRemoteResource"; import { CENTRAL_COMPATIBILITY_REJECTION_REASONS, REMOTE_RESOURCE_KINDS, + usesEncryptedInternalMetadata, type ReplicationAttemptFailure, type ReplicatorInstance, } from "@vrtmrz/livesync-commonlib/replication"; @@ -62,10 +63,14 @@ function definedTweaks(values: TweakValues): TweakValues { ); } -function settingsAfterAdoption(assessment: TweakAssessment, direction: "adoptPreferred" | "adoptCurrent"): TweakValues { +function settingsAfterAdoption( + assessment: TweakAssessment, + direction: "adoptPreferred" | "adoptCurrent", + currentSettings: TweakValues = assessment.currentValues +): TweakValues { const comparedKeys = new Set(assessment.entries.map((entry) => entry.key)); const source = direction === "adoptPreferred" ? assessment.preferredValues : assessment.currentValues; - const target = direction === "adoptPreferred" ? assessment.currentValues : assessment.preferredValues; + const target = direction === "adoptPreferred" ? currentSettings : assessment.preferredValues; const recommendations = Object.fromEntries( Object.entries(source).filter(([key, value]) => !comparedKeys.has(key) && value !== undefined) ); @@ -100,7 +105,9 @@ function resolutionSettingsSignature(settings: ObsidianLiveSyncSettings): string export class ModuleResolvingMismatchedTweaks extends AbstractModule { private requiresIdConfigurationReview(assessment: TweakAssessment): boolean { - if (!assessment.entries.some(({ key, relation }) => key === "idDerivationVersion" && relation === "different")) { + if ( + !assessment.entries.some(({ key, relation }) => key === "idDerivationVersion" && relation === "different") + ) { return false; } Logger( @@ -208,7 +215,8 @@ export class ModuleResolvingMismatchedTweaks extends AbstractModule { ): Promise<[TweakValues | boolean, boolean]> { if (assessment.alignment === "matched") return [false, false]; if (this.requiresIdConfigurationReview(assessment)) return [false, false]; - const acceptedSettings = settingsAfterAdoption(assessment, "adoptPreferred"); + // Retain saved preferences when only their effective representation differs from the assessment. + const acceptedSettings = settingsAfterAdoption(assessment, "adoptPreferred", this.settings); const autoAcceptSide = await this._shouldAutoAcceptCompatibleLossy(assessment); if (autoAcceptSide === "REMOTE") return [acceptedSettings, false]; if (autoAcceptSide === "CURRENT") return [true, false]; @@ -279,8 +287,15 @@ export class ModuleResolvingMismatchedTweaks extends AbstractModule { const signature = resolutionSettingsSignature(this.settings); const publication = await this.services.replicator.acquireActiveReplicatorContext(); if (resolutionSettingsSignature(this.settings) !== signature) return "IGNORE"; - const currentTweaks = JSON.stringify(extractObject(TweakValuesTemplate, this.settings)); - if (JSON.stringify(extractObject(TweakValuesTemplate, assessment.currentValues)) !== currentTweaks) { + const currentTweaks = extractObject(TweakValuesTemplate, this.settings); + const effectiveTweaks = { + ...currentTweaks, + encryptInternalMetadata: + this.settings.remoteType === REMOTE_COUCHDB && usesEncryptedInternalMetadata(this.settings), + }; + const assessedTweaks = JSON.stringify(extractObject(TweakValuesTemplate, assessment.currentValues)); + // Replication supplies effective values; legacy callers can supply saved preferences. + if (assessedTweaks !== JSON.stringify(currentTweaks) && assessedTweaks !== JSON.stringify(effectiveTweaks)) { return "IGNORE"; } const [conf, rebuildRequired] = await this.services.tweakValue.checkAndAskResolvingMismatched( diff --git a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts index d0a02bb8..b8fb8623 100644 --- a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts +++ b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts @@ -74,6 +74,130 @@ function createModule(settingsOverride: Partial = {}) { } describe("ModuleResolvingMismatchedTweaks", () => { + it.each([ + { label: "automatic alignment with E2EE disabled", autoAcceptCompatibleTweak: true, encrypt: false }, + { label: "manual review with E2EE disabled", autoAcceptCompatibleTweak: false, encrypt: false }, + { + label: "automatic alignment with Property Encryption disabled", + autoAcceptCompatibleTweak: true, + encrypt: true, + }, + ])( + "resolves an effective chunk mismatch while retaining an inactive encryption preference: $label", + async ({ autoAcceptCompatibleTweak, encrypt }) => { + const { module, core, askSelectStringDialogue, reinitialise } = createModule({ + autoAcceptCompatibleTweak, + encrypt, + usePathObfuscation: false, + encryptInternalMetadata: true, + customChunkSize: 60, + tweakModified: 100, + }); + // CouchDB assesses the effective encryption value, while the saved preference remains enabled. + const current = { ...extractObject(TweakValuesTemplate, core.settings), encryptInternalMetadata: false }; + const preferred = { ...current, customChunkSize: 0, tweakModified: 200 }; + const assessment = assessTweakCompatibility(current, preferred); + core._services.tweakValue = { + checkAndAskResolvingMismatched: module._checkAndAskResolvingMismatchedTweaks.bind(module), + }; + askSelectStringDialogue.mockResolvedValueOnce("Apply settings to this device"); + const updateRemote = vi.fn(async () => true); + + await expect(module._askResolvingMismatchedTweaks(preferred, updateRemote, assessment)).resolves.toBe( + "CHECKAGAIN" + ); + + expect(core.settings).toMatchObject({ customChunkSize: 0, encryptInternalMetadata: true }); + expect(core._services.setting.saveSettingData).toHaveBeenCalledOnce(); + expect(reinitialise).toHaveBeenCalledOnce(); + expect(updateRemote).toHaveBeenCalledOnce(); + expect(askSelectStringDialogue).toHaveBeenCalledTimes(autoAcceptCompatibleTweak ? 0 : 1); + } + ); + + it.each(["settings", "publication"] as const)( + "discards an effective assessment when %s change while awaiting a decision", + async (changed) => { + const { module, core, askSelectStringDialogue, reinitialise } = createModule({ + autoAcceptCompatibleTweak: false, + encrypt: false, + encryptInternalMetadata: true, + customChunkSize: 60, + }); + const current = { ...extractObject(TweakValuesTemplate, core.settings), encryptInternalMetadata: false }; + const preferred = { ...current, customChunkSize: 0 }; + const assessment = assessTweakCompatibility(current, preferred); + core._services.tweakValue = { + checkAndAskResolvingMismatched: module._checkAndAskResolvingMismatchedTweaks.bind(module), + }; + askSelectStringDialogue.mockImplementation(async () => { + if (changed === "settings") { + // The effective value stays false, but the user's saved preference has changed. + core.settings.encryptInternalMetadata = false; + } else { + core._services.replicator.acquireActiveReplicatorContext.mockResolvedValue({}); + } + return "Apply settings to this device"; + }); + const updateRemote = vi.fn(async () => true); + + await expect(module._askResolvingMismatchedTweaks(preferred, updateRemote, assessment)).resolves.toBe( + "IGNORE" + ); + + expect(askSelectStringDialogue).toHaveBeenCalledOnce(); + expect(core.settings.customChunkSize).toBe(60); + expect(core._services.setting.saveSettingData).not.toHaveBeenCalled(); + expect(reinitialise).not.toHaveBeenCalled(); + expect(updateRemote).not.toHaveBeenCalled(); + } + ); + + it("discards an effective assessment of an earlier chunk setting before opening a dialogue", async () => { + const { module, core, askSelectStringDialogue } = createModule({ + autoAcceptCompatibleTweak: false, + encrypt: false, + encryptInternalMetadata: true, + customChunkSize: 60, + }); + const current = { ...extractObject(TweakValuesTemplate, core.settings), encryptInternalMetadata: false }; + const preferred = { ...current, customChunkSize: 0 }; + const assessment = assessTweakCompatibility(current, preferred); + core.settings.customChunkSize = 72; + core._services.tweakValue = { + checkAndAskResolvingMismatched: module._checkAndAskResolvingMismatchedTweaks.bind(module), + }; + const updateRemote = vi.fn(async () => true); + + await expect(module._askResolvingMismatchedTweaks(preferred, updateRemote, assessment)).resolves.toBe("IGNORE"); + + expect(askSelectStringDialogue).not.toHaveBeenCalled(); + expect(core.settings.customChunkSize).toBe(72); + expect(core._services.setting.saveSettingData).not.toHaveBeenCalled(); + expect(updateRemote).not.toHaveBeenCalled(); + }); + + it("applies an explicit decision when effective internal Metadata encryption differs", async () => { + const { module, core, askSelectStringDialogue } = createModule({ + autoAcceptCompatibleTweak: true, + encrypt: true, + usePathObfuscation: true, + E2EEAlgorithm: "v2", + encryptInternalMetadata: true, + }); + const preferred = { ...extractObject(TweakValuesTemplate, core.settings), encryptInternalMetadata: false }; + core._services.tweakValue = { + checkAndAskResolvingMismatched: module._checkAndAskResolvingMismatchedTweaks.bind(module), + }; + askSelectStringDialogue.mockResolvedValueOnce("Apply settings to this device"); + + await expect(module._askResolvingMismatchedTweaks(preferred, async () => true)).resolves.toBe("CHECKAGAIN"); + + expect(askSelectStringDialogue).toHaveBeenCalledOnce(); + expect(core.settings.encryptInternalMetadata).toBe(false); + expect(core._services.setting.saveSettingData).toHaveBeenCalledOnce(); + }); + it.each([0, 1] as const)( "keeps ID configuration %s when automatically aligning Chunk settings", async (idDerivationVersion) => { diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 4030b4f8..5f582dd4 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -187,7 +187,7 @@ LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) -- npm run test:e2e:obsidian:cli-to-obsidian-sync ``` -`test:e2e:obsidian:tweak-compatibility` exercises the mismatch dialogue against temporary CouchDB databases. Build the plug-in first. The scenario removes the legacy filename-case value from the remote preferred settings, applies compatible differences through the ordinary settings action, and verifies note synchronisation and a subsequent restart. Separate true/false and true/missing filename-case mismatches check that Fetch remains required. The true/missing case also fetches the files and verifies a subsequent synchronisation attempt. A final case changes the remote while the dialogue is open, selects the stale Fetch action, and checks that the current settings, existing local file, and documents in both remotes remain unchanged. +`test:e2e:obsidian:tweak-compatibility` exercises the mismatch dialogue against temporary CouchDB databases. Build the plug-in first. The scenario removes the legacy filename-case value from the remote preferred settings, applies compatible differences through the ordinary settings action, and verifies note synchronisation and a subsequent restart. An enabled but inactive internal Metadata encryption preference must neither suppress the `Sync now` dialogue nor be lost when applying the remote chunk settings and restarting. Separate true/false and true/missing filename-case mismatches check that Fetch remains required. The true/missing case also fetches the files and verifies a subsequent synchronisation attempt. A final case changes the remote while the dialogue is open, selects the stale Fetch action, and checks that the current settings, existing local file, and documents in both remotes remain unchanged. `test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service and starts with isolated Object Storage settings and the device-local compatibility acknowledgement already in place, keeping the scenario focused on upload rather than unconfigured start-up or setup. It confirms those settings through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. Adapter tests separately observe an in-progress SDK command, while this real-runtime workflow verifies the resulting request counters advance and rebalance. diff --git a/test/e2e-obsidian/scripts/tweak-compatibility.ts b/test/e2e-obsidian/scripts/tweak-compatibility.ts index ba083278..d975e225 100644 --- a/test/e2e-obsidian/scripts/tweak-compatibility.ts +++ b/test/e2e-obsidian/scripts/tweak-compatibility.ts @@ -6,6 +6,8 @@ * in the chunk size and V2 customisation setting. Applying the ordinary * remote settings action must permit a fresh synchronisation without a Fetch. A * Separate true/false and true/missing case controls keep Fetch required. + * An enabled but inactive internal Metadata encryption preference must survive + * ordinary setting adoption and restart, and must not suppress the Sync now dialogue. */ import { readFile } from "node:fs/promises"; import { join } from "node:path"; @@ -129,6 +131,21 @@ async function replicateOnce(cliBinary: string, env: NodeJS.ProcessEnv): Promise ); } +async function runSyncNow(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const command=app.commands.commands['obsidian-livesync:livesync-replicate'];", + "if(typeof command?.callback!=='function') throw new Error('Sync now command is unavailable');", + "await command.callback();", + "return JSON.stringify(true);", + "})()", + ].join(""), + env + ); +} + async function selectCompatibilityAction( port: number, action: string, @@ -220,17 +237,23 @@ async function prepareConfiguredSession( async function readTweakState( cliBinary: string, env: NodeJS.ProcessEnv -): Promise<{ customChunkSize: unknown; usePluginSyncV2: unknown; handleFilenameCaseSensitive: unknown }> { +): Promise<{ + customChunkSize: unknown; + usePluginSyncV2: unknown; + handleFilenameCaseSensitive: unknown; + encryptInternalMetadata: unknown; +}> { return await evalObsidianJson<{ customChunkSize: unknown; usePluginSyncV2: unknown; handleFilenameCaseSensitive: unknown; + encryptInternalMetadata: unknown; }>( cliBinary, [ "(()=>{", "const settings=app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings();", - "return JSON.stringify({customChunkSize:settings.customChunkSize,usePluginSyncV2:settings.usePluginSyncV2,handleFilenameCaseSensitive:settings.handleFilenameCaseSensitive});", + "return JSON.stringify({customChunkSize:settings.customChunkSize,usePluginSyncV2:settings.usePluginSyncV2,handleFilenameCaseSensitive:settings.handleFilenameCaseSensitive,encryptInternalMetadata:settings.encryptInternalMetadata});", "})()", ].join(""), env @@ -360,12 +383,20 @@ async function main(): Promise { await removeLegacyCasePreference(couchDb, dbName); targetSession = await prepareConfiguredSession(binary, cli.binary, targetVault, settings, { + encrypt: false, + encryptInternalMetadata: true, customChunkSize: 60, usePluginSyncV2: true, handleFilenameCaseSensitive: false, autoAcceptCompatibleTweak: false, }); await prepareRemote(cli.binary, targetSession.cliEnv); + const commandPromise = runSyncNow(cli.binary, targetSession.cliEnv); + await selectCompatibilityAction(targetSession.remoteDebuggingPort, dismissAction); + await commandPromise; + const dismissedState = await readTweakState(cli.binary, targetSession.cliEnv); + assertEqual(dismissedState.customChunkSize, 60, "Dismissing Sync now changed the chunk size."); + assertEqual(dismissedState.encryptInternalMetadata, true, "Dismissing Sync now changed the saved preference."); const targetReplicationPromise = replicateOnce(cli.binary, targetSession.cliEnv); await selectCompatibilityAction(targetSession.remoteDebuggingPort, applySettingsAction); const targetReplication = await targetReplicationPromise; @@ -383,6 +414,11 @@ async function main(): Promise { await waitForLiveSyncCoreReady(cli.binary, targetSession.cliEnv); await assertE2eCompatibilityMarker(cli.binary, targetSession.cliEnv); const restartedState = await readTweakState(cli.binary, targetSession.cliEnv); + assertEqual( + restartedState.encryptInternalMetadata, + true, + "The inactive internal Metadata encryption preference was not retained after adoption and restart." + ); assertEqual( restartedState.customChunkSize, 0, diff --git a/updates.md b/updates.md index 31d529ec..9ed3abb4 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We can resolve configuration mismatches when **Encrypt internal file Properties** is enabled but its encryption prerequisites are inactive. (#1180) + - Compatible chunk settings can be aligned automatically, or reviewed through `Sync now`. Applying those settings preserves our inactive encryption preference. + ## 1.0.37 16th October, 2026 From 83d40a561f86f1f13378b686cdd0f56e18124619 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 08:43:04 +0000 Subject: [PATCH 03/68] Preserve adopted encryption settings when prerequisites change --- docs/design_docs/tweak_compatibility.md | 6 ++- .../ModuleResolveMismatchedTweaks.ts | 11 ++++- ...ModuleResolveMismatchedTweaks.unit.spec.ts | 41 +++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/design_docs/tweak_compatibility.md b/docs/design_docs/tweak_compatibility.md index 1e9c931b..965e5f3b 100644 --- a/docs/design_docs/tweak_compatibility.md +++ b/docs/design_docs/tweak_compatibility.md @@ -69,8 +69,10 @@ Central replication can assess an effective value which differs from a saved preference, such as internal Metadata encryption whose prerequisites are inactive. Recovery accepts the current saved or effective snapshot without treating that representation difference as a settings change. Applying assessed changes retains -saved preferences for settings whose effective values already agree. The full -settings signature still detects a preference changed while awaiting a decision. +saved preferences only when they leave the adopted effective values unchanged. +If adoption activates encryption prerequisites, the adopted encryption value +takes precedence. The full settings signature still detects a preference changed +while awaiting a decision. The active publication reservation is not held while waiting for the dialogue. Remote writes use the failed attempt's publication guard. Fetch and Rebuild use diff --git a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts index 0294b7d1..4053b6c9 100644 --- a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts +++ b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts @@ -215,8 +215,17 @@ export class ModuleResolvingMismatchedTweaks extends AbstractModule { ): Promise<[TweakValues | boolean, boolean]> { if (assessment.alignment === "matched") return [false, false]; if (this.requiresIdConfigurationReview(assessment)) return [false, false]; - // Retain saved preferences when only their effective representation differs from the assessment. + // Retain saved preferences only when they leave the adopted effective values unchanged. + const assessedSettings = settingsAfterAdoption(assessment, "adoptPreferred"); const acceptedSettings = settingsAfterAdoption(assessment, "adoptPreferred", this.settings); + if ( + this.settings.remoteType === REMOTE_COUCHDB && + usesEncryptedInternalMetadata({ ...this.settings, ...acceptedSettings }) !== + usesEncryptedInternalMetadata({ ...this.settings, ...assessedSettings }) + ) { + // Adopting encryption prerequisites can activate a previously inactive preference. + acceptedSettings.encryptInternalMetadata = assessedSettings.encryptInternalMetadata; + } const autoAcceptSide = await this._shouldAutoAcceptCompatibleLossy(assessment); if (autoAcceptSide === "REMOTE") return [acceptedSettings, false]; if (autoAcceptSide === "CURRENT") return [true, false]; diff --git a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts index b8fb8623..f52f5c85 100644 --- a/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts +++ b/src/modules/coreFeatures/ModuleResolveMismatchedTweaks.unit.spec.ts @@ -14,6 +14,7 @@ import { CENTRAL_COMPATIBILITY_REJECTION_REASONS, REMOTE_RESOURCE_KINDS, USER_INITIATED_REPLICATION_AUTHORITY, + usesEncryptedInternalMetadata, type ReplicationAttemptFailure, } from "@vrtmrz/livesync-commonlib/replication"; @@ -74,6 +75,46 @@ function createModule(settingsOverride: Partial = {}) { } describe("ModuleResolvingMismatchedTweaks", () => { + it.each([ + { label: "E2EE", encrypt: false, usePathObfuscation: true }, + { label: "Property Encryption", encrypt: true, usePathObfuscation: false }, + ])( + "uses the remote Metadata encryption preference when adoption enables $label", + async ({ encrypt, usePathObfuscation }) => { + const { module, core, askSelectStringDialogue } = createModule({ + autoAcceptCompatibleTweak: false, + encrypt, + usePathObfuscation, + E2EEAlgorithm: "v2", + encryptInternalMetadata: true, + idDerivationVersion: 0, + }); + const current = { + ...extractObject(TweakValuesTemplate, core.settings), + encryptInternalMetadata: usesEncryptedInternalMetadata(core.settings), + }; + const preferred = { ...current, encrypt: true, usePathObfuscation: true, encryptInternalMetadata: false }; + const assessment = assessTweakCompatibility(current, preferred); + core._services.tweakValue = { + checkAndAskResolvingMismatched: module._checkAndAskResolvingMismatchedTweaks.bind(module), + }; + core.rebuilder = { $fetchLocal: vi.fn(async () => undefined) }; + askSelectStringDialogue.mockResolvedValueOnce("Apply settings to this device, and fetch again"); + const updateRemote = vi.fn(async () => true); + + await expect(module._askResolvingMismatchedTweaks(preferred, updateRemote, assessment)).resolves.toBe( + "CHECKAGAIN" + ); + + expect(core.rebuilder.$fetchLocal).toHaveBeenCalledOnce(); + expect(core.settings).toMatchObject({ encrypt: true, usePathObfuscation: true }); + expect(usesEncryptedInternalMetadata(core.settings)).toBe(false); + expect(updateRemote).toHaveBeenCalledWith( + expect.objectContaining({ encrypt: true, usePathObfuscation: true, encryptInternalMetadata: false }) + ); + } + ); + it.each([ { label: "automatic alignment with E2EE disabled", autoAcceptCompatibleTweak: true, encrypt: false }, { label: "manual review with E2EE disabled", autoAcceptCompatibleTweak: false, encrypt: false }, From 456a742a72e3ad2c30496201b2f515349bd09928 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 09:11:17 +0000 Subject: [PATCH 04/68] Cover tweak recovery continuity in real Obsidian --- devs.md | 2 +- test/e2e-obsidian/README.md | 8 +- .../scripts/tweak-compatibility.ts | 166 +++++++++++++++++- 3 files changed, 170 insertions(+), 6 deletions(-) diff --git a/devs.md b/devs.md index bbeaf1be..a539c013 100644 --- a/devs.md +++ b/devs.md @@ -344,7 +344,7 @@ For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch- - For a hyphenated pre-release, run finalisation with `prerelease=true`; CLI publication remains optional. For a stable version awaiting BRAT validation, use `prerelease=true` and `publish_cli=false`. - Approve the `Release Obsidian Plugin` workflow for the `release` environment, then inspect the generated draft GitHub Release. When a hyphenated pre-release includes the CLI, confirm that it received only its immutable version and SHA-qualified image tags. - Publish the draft as a GitHub pre-release without replacing the latest stable release. Keep its release pull request in draft and leave its base branch unchanged throughout BRAT validation. Record that state in the pull request. -- Validate the published release through BRAT. Confirm start-up, ordinary bidirectional synchronisation, and any regression scenario relevant to the release. +- Validate installation or update through BRAT, and cold-start Obsidian to confirm the plug-in loads. Verify ordinary bidirectional synchronisation and relevant regression scenarios in real Obsidian against the published artefacts, or a build with matching hashes. When automated scenarios establish that coverage, BRAT acceptance need only cover installation or update and start-up; manually check any remaining release-specific behaviour. Record the artefact hashes and validation evidence as described in the [real Obsidian test guide](test/e2e-obsidian/README.md#combined-setup-and-security-regression-checks). - After a hyphenated pre-release passes, keep its release pull request unmerged. Add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`, then close the release pull request only through a separate maintainer action. - After a stable version passes, remove the GitHub pre-release designation and make that exact release the latest stable release. Confirm that the release is no longer a pre-release, then mark its release pull request ready, merge the exact release commit into the selected base branch with a merge commit, and integrate that exact commit through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata. Create the stable CLI tag and publish its `latest` and major-minor image tags, if selected, through a separate maintainer gate. - If BRAT validation fails, keep the release PR in draft and do not move published tags. Before preparing the next version, add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`. Keep only changes made after that tag under `## Unreleased`. Compare the historical section with `git show :updates.md`; do not merge the failed release PR or describe it as validated. The next release PR can then rotate only the correction notes while preserving the immutable release history. diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 5f582dd4..a0df4f1f 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -189,6 +189,10 @@ LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) -- `test:e2e:obsidian:tweak-compatibility` exercises the mismatch dialogue against temporary CouchDB databases. Build the plug-in first. The scenario removes the legacy filename-case value from the remote preferred settings, applies compatible differences through the ordinary settings action, and verifies note synchronisation and a subsequent restart. An enabled but inactive internal Metadata encryption preference must neither suppress the `Sync now` dialogue nor be lost when applying the remote chunk settings and restarting. Separate true/false and true/missing filename-case mismatches check that Fetch remains required. The true/missing case also fetches the files and verifies a subsequent synchronisation attempt. A final case changes the remote while the dialogue is open, selects the stale Fetch action, and checks that the current settings, existing local file, and documents in both remotes remain unchanged. +Three independent databases cover automatic chunk-setting alignment and adoption which enables either E2EE or Property Encryption. Automatic alignment retains the inactive internal Metadata encryption preference. Both encryption cases require the visible Fetch action and adopt the remote internal Metadata encryption preference without unexpectedly enabling it. Each case checks the local and remote settings, real file contents, restart persistence, and a return transfer to the source Vault. The encryption cases verify file reflection immediately after Fetch, before a fresh synchronisation attempt. + +Set `E2E_LIVESYNC_TARGET_ARTIFACT_ROOT` to the directory containing the exact release artefacts to run this scenario against them. The runner records SHA-256 hashes of `main.js`, `manifest.json`, and `styles.css`; retain the output with the release validation evidence. A passing run against a local build establishes that build's behaviour. It covers the published artefacts only when all three hashes match, or when the scenario is rerun against those published files. + `test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service and starts with isolated Object Storage settings and the device-local compatibility acknowledgement already in place, keeping the scenario focused on upload rather than unconfigured start-up or setup. It confirms those settings through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. Adapter tests separately observe an in-progress SDK command, while this real-runtime workflow verifies the resulting request counters advance and rebalance. Set `E2E_OBSIDIAN_INDEPENDENT_IDS=true` to run the same upload with E2EE, Path Obfuscation, and a separately derived ID key. The scenario verifies the local document and Chunk ID shapes before the Journal transfer. @@ -296,7 +300,7 @@ Build the current plug-in once, then run these focused scenarios sequentially wi The setup-tool contract suite also checks ID recovery and explicit legacy IDs in both URI modes. `dialog-mounts` covers the availability dialogue and setup choices on desktop and emulated mobile. These automated scenarios remove the need to repeat every decision path manually during BRAT acceptance. -BRAT acceptance validates the exact published artefacts: install or update through BRAT, cold-start Obsidian, and exchange one note in each direction. Build behaviour can be checked before publication using the exact reviewed build; record its identity and avoid repeating the same decision paths during BRAT acceptance. +BRAT acceptance validates installation of the exact published artefacts: install or update through BRAT, and cold-start Obsidian to confirm the plug-in loads. When the release's relevant regression scenarios and two-way synchronisation have passed in real Obsidian against those artefacts, or a build with matching hashes, these functional checks need not be repeated manually. Otherwise, exchange one note in each direction and check any remaining release-specific regression during BRAT acceptance. Record the artefact identity and the automated coverage used to select the manual checks. A short physical-device check is optional when a specific concern remains about localised time text, input, clipboard interaction, or responsiveness. The camera and operating-system dispatch paths are unchanged by this integration and do not require routine revalidation. Emulated mobile establishes layout and interaction, but does not establish native device performance or verify the published installation path. @@ -329,7 +333,7 @@ A short physical-device check is optional when a specific concern remains about - `E2E_LIVESYNC_CLI_TIMEOUT_MS`: timeout for each official LiveSync CLI invocation in the CLI-to-Obsidian compatibility check; default is 60 seconds. - `LIVESYNC_CLI_COMMAND`: optional LiveSync CLI executable and prefix arguments used by the CLI-to-Obsidian compatibility check. The default is the locally built CLI. - `E2E_LIVESYNC_SOURCE_ARTIFACT_ROOT`: optional cache directory containing the exact pinned 0.25.83 plug-in artefacts. Cached files are always checksum-verified. -- `E2E_LIVESYNC_TARGET_ARTIFACT_ROOT`: directory containing the built 1.0 target `main.js`, `manifest.json`, and `styles.css`; default is the repository root. +- `E2E_LIVESYNC_TARGET_ARTIFACT_ROOT`: directory containing the target `main.js`, `manifest.json`, and `styles.css` for upgrade and tweak-compatibility scenarios; default is the repository root. - `E2E_OBSIDIAN_ARTIFACT_ROOT`: directory containing the plug-in artefact installed by a direct scenario invocation; default is the repository root. - `E2E_OBSIDIAN_ARTIFACT_REVISION`: exact source commit recorded by the Security Seed reconnect result when `E2E_OBSIDIAN_ARTIFACT_ROOT` is a downloaded artefact rather than a Git worktree. - `E2E_OBSIDIAN_FILE_TIMEOUT_MS`: timeout for waiting until a note created through Obsidian's vault API is reflected to disk. diff --git a/test/e2e-obsidian/scripts/tweak-compatibility.ts b/test/e2e-obsidian/scripts/tweak-compatibility.ts index d975e225..14ce5f6d 100644 --- a/test/e2e-obsidian/scripts/tweak-compatibility.ts +++ b/test/e2e-obsidian/scripts/tweak-compatibility.ts @@ -4,13 +4,16 @@ * The source Vault creates a remote preferred profile with a missing legacy * filename-case value. The target has the effective false value, but differs * in the chunk size and V2 customisation setting. Applying the ordinary - * remote settings action must permit a fresh synchronisation without a Fetch. A + * remote settings action must permit a fresh synchronisation without a Fetch. * Separate true/false and true/missing case controls keep Fetch required. * An enabled but inactive internal Metadata encryption preference must survive * ordinary setting adoption and restart, and must not suppress the Sync now dialogue. + * Automatic alignment and adoption which enables an encryption prerequisite also + * exercise two-way transfer and restart against independent remote databases. */ +import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { MILESTONE_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { evalObsidianJson } from "../runner/cli.ts"; import { @@ -54,6 +57,7 @@ const sourceNotePath = "E2E/tweak-compatibility/source.md"; const restartedNotePath = "E2E/tweak-compatibility/restarted.md"; const sourceNoteContent = `# Tweak compatibility source\n\n${"source-content ".repeat(1200)}\n`; const restartedNoteContent = `# Tweak compatibility restart\n\n${"restart-content ".repeat(1200)}\n`; +const artifactRoot = resolve(process.env.E2E_LIVESYNC_TARGET_ARTIFACT_ROOT?.trim() || process.cwd()); type MilestoneDocument = CouchDbDocument & { tweak_values?: Record; @@ -208,6 +212,7 @@ async function startSession( binary, cliBinary, vault, + artifactRoot, startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), ...(pluginData === undefined ? {} : { pluginData }), localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), @@ -242,18 +247,24 @@ async function readTweakState( usePluginSyncV2: unknown; handleFilenameCaseSensitive: unknown; encryptInternalMetadata: unknown; + encrypt: unknown; + usePathObfuscation: unknown; + tweakModified: unknown; }> { return await evalObsidianJson<{ customChunkSize: unknown; usePluginSyncV2: unknown; handleFilenameCaseSensitive: unknown; encryptInternalMetadata: unknown; + encrypt: unknown; + usePathObfuscation: unknown; + tweakModified: unknown; }>( cliBinary, [ "(()=>{", "const settings=app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings();", - "return JSON.stringify({customChunkSize:settings.customChunkSize,usePluginSyncV2:settings.usePluginSyncV2,handleFilenameCaseSensitive:settings.handleFilenameCaseSensitive,encryptInternalMetadata:settings.encryptInternalMetadata});", + "return JSON.stringify({customChunkSize:settings.customChunkSize,usePluginSyncV2:settings.usePluginSyncV2,handleFilenameCaseSensitive:settings.handleFilenameCaseSensitive,encryptInternalMetadata:settings.encryptInternalMetadata,encrypt:settings.encrypt,usePathObfuscation:settings.usePathObfuscation,tweakModified:settings.tweakModified});", "})()", ].join(""), env @@ -337,12 +348,158 @@ async function verifyStaleTargetChoice( ); } +async function verifyRecoveryContinuity( + binary: string, + cliBinary: string, + couchDb: CouchDbConfig, + scenario: "automatic" | "encrypt" | "usePathObfuscation" +): Promise { + const automatic = scenario === "automatic"; + const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, `tweak-${scenario.toLowerCase()}`); + const settings = couchDbSettings(couchDb, dbName); + const sourceVault = await createTemporaryVault(); + const targetVault = await createTemporaryVault(); + const sourceSettings = { + encrypt: !automatic, + usePathObfuscation: !automatic, + encryptInternalMetadata: false, + passphrase: "tweak-recovery-e2e-secret", + E2EEAlgorithm: "v2", + idDerivationVersion: 0, + customChunkSize: 0, + autoAcceptCompatibleTweak: false, + }; + const expectedState = { + encrypt: !automatic, + usePathObfuscation: !automatic, + encryptInternalMetadata: automatic, + customChunkSize: 0, + }; + let session: ObsidianLiveSyncSession | undefined; + const assertAdoptedSettings = async () => { + assert(session, "The recovery scenario has no active Obsidian session."); + const state = await readTweakState(cliBinary, session.cliEnv); + for (const [key, value] of Object.entries(expectedState)) { + assertEqual(state[key as keyof typeof state], value, `${scenario}: incorrect adopted ${key}.`); + } + const milestone = (await fetchCouchDbDocument(couchDb, dbName, milestoneId)) as MilestoneDocument; + const preferred = milestone.tweak_values?.PREFERRED; + assert(isRecord(preferred), `${scenario}: the remote preferred profile is missing.`); + for (const [key, value] of Object.entries({ ...expectedState, encryptInternalMetadata: false })) { + assertEqual(preferred[key], value, `${scenario}: incorrect remote ${key}.`); + } + }; + + try { + await createCouchDbDatabase(couchDb, dbName); + console.log(`Recovery continuity scenario: ${scenario} (${dbName})`); + session = await prepareConfiguredSession(binary, cliBinary, sourceVault, settings, sourceSettings); + await prepareRemote(cliBinary, session.cliEnv); + await writeNote(cliBinary, session.cliEnv, sourceNotePath, sourceNoteContent); + await waitForLocalDatabaseEntry(cliBinary, session.cliEnv, sourceNotePath); + assertEqual( + (await replicateOnce(cliBinary, session.cliEnv)).succeeded, + true, + `${scenario}: source upload failed.` + ); + await session.app.stop(); + session = undefined; + + session = await prepareConfiguredSession(binary, cliBinary, targetVault, settings, { + ...sourceSettings, + encryptInternalMetadata: true, + ...(automatic ? { customChunkSize: 60, autoAcceptCompatibleTweak: true } : { [scenario]: false }), + }); + await prepareRemote(cliBinary, session.cliEnv); + const initialState = await readTweakState(cliBinary, session.cliEnv); + assertEqual(initialState.encryptInternalMetadata, true, `${scenario}: the saved preference is not enabled.`); + // The target was configured last. Make the remote profile newer so automatic + // alignment must adopt its chunk setting, rather than publish the target's. + if (automatic) { + assertEqual(initialState.customChunkSize, 60, "The automatic alignment fixture has no chunk mismatch."); + const milestone = (await fetchCouchDbDocument(couchDb, dbName, milestoneId)) as MilestoneDocument; + const preferred = milestone.tweak_values?.PREFERRED; + assert(isRecord(preferred), "The automatic alignment fixture has no remote preferred profile."); + assertEqual(preferred.customChunkSize, 0, "The remote fixture has the wrong chunk setting."); + preferred.tweakModified = + (typeof initialState.tweakModified === "number" ? initialState.tweakModified : 0) + 1; + await putCouchDbDocument(couchDb, dbName, milestone); + } else { + assertEqual(initialState[scenario], false, `${scenario}: the encryption prerequisite is already enabled.`); + } + const adoption = replicateOnce(cliBinary, session.cliEnv); + if (!automatic) { + await selectCompatibilityAction( + session.remoteDebuggingPort, + applySettingsWithFetchAction, + applySettingsAction + ); + } + await adoption; + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + await assertAdoptedSettings(); + if (!automatic) { + // The real Fetch must reflect the source before a subsequent replication + // could hide a failure to rebuild or decrypt the local database. + await waitForPathContent(targetVault.path, sourceNotePath, sourceNoteContent); + } + assertEqual( + (await replicateOnce(cliBinary, session.cliEnv)).succeeded, + true, + `${scenario}: synchronisation after adoption failed.` + ); + await waitForPathContent(targetVault.path, sourceNotePath, sourceNoteContent); + await session.app.stop(); + session = undefined; + + session = await startSession(binary, cliBinary, targetVault); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + await assertE2eCompatibilityMarker(cliBinary, session.cliEnv); + await assertAdoptedSettings(); + await writeNote(cliBinary, session.cliEnv, restartedNotePath, restartedNoteContent); + await waitForLocalDatabaseEntry(cliBinary, session.cliEnv, restartedNotePath); + assertEqual( + (await replicateOnce(cliBinary, session.cliEnv)).succeeded, + true, + `${scenario}: upload after restart failed.` + ); + await assertAdoptedSettings(); + await session.app.stop(); + session = undefined; + + session = await startSession(binary, cliBinary, sourceVault); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + assertEqual( + (await replicateOnce(cliBinary, session.cliEnv)).succeeded, + true, + `${scenario}: return transfer failed.` + ); + await waitForPathContent(sourceVault.path, restartedNotePath, restartedNoteContent); + console.log(`Recovery continuity passed: ${scenario}, persisted settings, restart, and two-way content.`); + } finally { + if (session) await session.app.stop().catch(() => undefined); + await Promise.all([sourceVault.dispose(), targetVault.dispose()]); + if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { + await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => { + console.warn(error instanceof Error ? error.message : error); + }); + } + } +} + async function main(): Promise { const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); if (!cli.binary) { throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); } + console.log(`Plug-in artefacts: ${artifactRoot}`); + for (const name of ["main.js", "manifest.json", "styles.css"]) { + const hash = createHash("sha256") + .update(new Uint8Array(await readFile(join(artifactRoot, name)))) + .digest("hex"); + console.log(`SHA-256 ${name}: ${hash}`); + } const couchDb = await loadCouchDbConfig(); const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "tweak-compatibility"); @@ -505,6 +662,9 @@ async function main(): Promise { controlSession = undefined; console.log("Tweak compatibility also rejected a stale Fetch choice after the remote changed."); + for (const scenario of ["automatic", "encrypt", "usePathObfuscation"] as const) { + await verifyRecoveryContinuity(binary, cli.binary, couchDb, scenario); + } } finally { if (sourceSession) await sourceSession.app.stop().catch(() => undefined); if (targetSession) await targetSession.app.stop().catch(() => undefined); From f47db4f256dd14d0b15ee46d52c4fb3dfc0bb79d Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 13:24:53 +0000 Subject: [PATCH 05/68] Preserve unchanged-file reflection after revision history pruning --- devs.md | 12 ++ package-lock.json | 8 +- package.json | 2 +- .../cli/serviceModules/CLIServiceModules.ts | 1 + .../serviceModules/FSAPIServiceModules.ts | 1 + .../ReviewHarness/reviewHarnessContract.ts | 8 + .../reviewHarnessContract.unit.spec.ts | 6 + .../ReviewHarness/reviewHarnessController.ts | 3 + .../reviewHarnessController.unit.spec.ts | 42 +++++ .../reviewHarnessFileContentHashBenchmark.ts | 149 ++++++++++++++++++ ...rnessFileContentHashBenchmark.unit.spec.ts | 134 ++++++++++++++++ ...wHarnessFileContentHashBenchmarkRuntime.ts | 20 +++ ...leContentHashBenchmarkRuntime.unit.spec.ts | 15 ++ src/main.ts | 1 + src/serviceFeatures/useReviewHarness.ts | 7 + test/e2e-obsidian/README.md | 2 +- test/e2e-obsidian/scripts/review-harness.ts | 56 +++++++ .../scripts/stale-file-restart.ts | 66 ++++++-- updates.md | 9 ++ 19 files changed, 524 insertions(+), 18 deletions(-) create mode 100644 src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts create mode 100644 src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts create mode 100644 src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts create mode 100644 src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts diff --git a/devs.md b/devs.md index bbeaf1be..e5648637 100644 --- a/devs.md +++ b/devs.md @@ -245,6 +245,18 @@ Where `performance.memory` is available, the report includes approximate JavaScr The developer-only adapter in `src/features/ReviewHarness/reviewHarnessIdBenchmarkRuntime.ts` imports `HashManager` from Commonlib's public `/hashing` entry. Compilation, packed-package checks, and runtime tests cover this boundary. The algorithms remain owned by Commonlib. +#### File content hash performance measurements + +Commonlib owns the algorithm choice and provenance contract. Its [file-reflection content hash decision](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/file-reflection-content-hash.md) records the SHA-256 rationale, xxHash64 comparison, measured costs, and conditions for reconsideration. This section describes the host-owned measurement procedure. + +Choose **Run** beside **File content hash performance** in the Review Harness. **Automatic** does not run this measurement; **Full review** includes it. + +The measurement compares the Self-hosted LiveSync SHA-256 file content hash helper with `xxHash64` for fixed in-memory `Blob` inputs of 4 KiB, 64 KiB, 1 MiB, and 50 MiB. Each input begins with the same deterministic byte sequence. Input construction is excluded from the timed samples; `xxHash64` engine initialisation is timed and reported separately. One 4-KiB warm-up call per algorithm is also measured separately and excluded from samples. SHA-256 uses the same helper as the product's file content hash path; `xxHash64` timing includes `Blob.arrayBuffer()` and `h64Raw`. Calls run one at a time, alternate algorithm order between samples, and yield to the interface between calls. The 50-MiB input uses the same three samples as the other sizes. + +The report gives the median and range for each algorithm and input size. Where `performance.memory` is available, it also records approximate JavaScript heap samples before, during, and after measurement. These samples may include other Obsidian activity and garbage collection; they are neither total process RAM nor a guaranteed peak. Unsupported devices explicitly report that heap measurements are unavailable. + +The scenario uses only fixed in-memory inputs. It does not read or write the Vault, database, settings, remote storage, or user content. It does not expose a hash-method setting or use a fixed performance threshold. The developer-only adapter in `src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts` imports `computeFileContentHash` from Commonlib's public `/hashing` entry and `xxhashNew()` from `octagonal-wheels`. The benchmark compares the product helper with that package implementation; it does not change the hash method used by LiveSync. + #### Logs - Use `this._log(msg, LOG_LEVEL_INFO)` in modules (automatically prefixes with module name) diff --git a/package-lock.json b/package-lock.json index 6aadafc4..e607283a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.40", + "@vrtmrz/livesync-commonlib": "0.1.42", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4389,9 +4389,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.40", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.40.tgz", - "integrity": "sha512-GVskcVFoQIpb45rR+QT9Wjtdqc6oAxYz5IB2WeJwusWZ8lTmqVrba+6YD9FTtsbzqbPOyJ/LdQxnET7SUexVtw==", + "version": "0.1.42", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.42.tgz", + "integrity": "sha512-LyWl0Mkrj9UsQXV0nOUaYJybcN3wLkpAAxCeHMZYOEizaUTQ7ezN+FgzN+OQxDRA8UXWnoatxh719NWzOwBpBw==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 16dc3c0b..fb7edb0b 100644 --- a/package.json +++ b/package.json @@ -191,7 +191,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.40", + "@vrtmrz/livesync-commonlib": "0.1.42", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/src/apps/cli/serviceModules/CLIServiceModules.ts b/src/apps/cli/serviceModules/CLIServiceModules.ts index 2430cfaa..e87f864e 100644 --- a/src/apps/cli/serviceModules/CLIServiceModules.ts +++ b/src/apps/cli/serviceModules/CLIServiceModules.ts @@ -102,6 +102,7 @@ export function initialiseServiceModulesCLI( events: services.context.events, API: services.API, database: services.database, + path: services.path, appLifecycle: services.appLifecycle, setting: services.setting, remote: services.remote, diff --git a/src/apps/webapp/serviceModules/FSAPIServiceModules.ts b/src/apps/webapp/serviceModules/FSAPIServiceModules.ts index b9054cec..58b61d45 100644 --- a/src/apps/webapp/serviceModules/FSAPIServiceModules.ts +++ b/src/apps/webapp/serviceModules/FSAPIServiceModules.ts @@ -93,6 +93,7 @@ export function initialiseServiceModulesFSAPI( events: services.context.events, API: services.API, database: services.database, + path: services.path, appLifecycle: services.appLifecycle, setting: services.setting, remote: services.remote, diff --git a/src/features/ReviewHarness/reviewHarnessContract.ts b/src/features/ReviewHarness/reviewHarnessContract.ts index 3925857d..fa953f48 100644 --- a/src/features/ReviewHarness/reviewHarnessContract.ts +++ b/src/features/ReviewHarness/reviewHarnessContract.ts @@ -37,6 +37,14 @@ export const REVIEW_HARNESS_SCENARIOS = [ mode: "automatic", access: "read-only", }, + { + id: "file-content-hash-performance", + title: "File content hash performance", + description: + "Compares SHA-256 and xxHash64 on fixed in-memory Blob inputs of 4 KiB, 64 KiB, 1 MiB, and 50 MiB. Input construction is excluded from hash timings; engine initialisation and warm-up are reported separately. Keep Obsidian in the foreground.", + mode: "manual", + access: "read-only", + }, ] as const; export const REVIEW_HARNESS_SCENARIO_IDS = REVIEW_HARNESS_SCENARIOS.map(({ id }) => id); diff --git a/src/features/ReviewHarness/reviewHarnessContract.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessContract.unit.spec.ts index bc33c775..127a6c71 100644 --- a/src/features/ReviewHarness/reviewHarnessContract.unit.spec.ts +++ b/src/features/ReviewHarness/reviewHarnessContract.unit.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { NEW_VAULT_SETTINGS, type SettingsMigrationState } from "@vrtmrz/livesync-commonlib/settings"; import type { CompatibilityPause } from "@/common/databaseCompatibility.ts"; import { + REVIEW_HARNESS_SCENARIOS, REVIEW_HARNESS_SCENARIO_IDS, formatReviewHarnessReport, inspectCompatibilityReview, @@ -76,7 +77,12 @@ describe("Review Harness contract", () => { "compatibility-review", "vault-round-trip", "id-generation-performance", + "file-content-hash-performance", ]); + expect(REVIEW_HARNESS_SCENARIOS.find(({ id }) => id === "file-content-hash-performance")).toMatchObject({ + mode: "manual", + access: "read-only", + }); }); it("checks that an existing Vault exposes the seven synchronisation choices as booleans", () => { diff --git a/src/features/ReviewHarness/reviewHarnessController.ts b/src/features/ReviewHarness/reviewHarnessController.ts index 12c0e370..2bde2f58 100644 --- a/src/features/ReviewHarness/reviewHarnessController.ts +++ b/src/features/ReviewHarness/reviewHarnessController.ts @@ -22,6 +22,7 @@ export interface ReviewHarnessRuntime { openCompatibilityReview(): Promise; runVaultRoundTrip(): Promise; runIdBenchmark(): Promise; + runFileContentHashBenchmark(): Promise; readContinuation(): string | null; writeContinuation(value: string): void; deleteContinuation(): void; @@ -162,6 +163,8 @@ export class ReviewHarnessController { result = await this.runtime.runVaultRoundTrip(); } else if (id === "id-generation-performance") { result = await this.runtime.runIdBenchmark(); + } else if (id === "file-content-hash-performance") { + result = await this.runtime.runFileContentHashBenchmark(); } else { const inspection = this.inspectCompatibilityReview(); result = diff --git a/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts index dee43186..cce28869 100644 --- a/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts +++ b/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts @@ -85,6 +85,11 @@ function createRuntime(): ReviewHarnessRuntime & { detail: "ID generation measurements completed.", observations: ["Chunk 256 B: 1000 IDs total=43.00 ms; per ID=0.0430 ms"], })), + runFileContentHashBenchmark: vi.fn(async () => ({ + status: "passed" as const, + detail: "File content hash measurements completed.", + observations: ["SHA-256, 4 KiB: median=1.00 ms; range=1.00–1.00 ms"], + })), readContinuation() { return this.continuation; }, @@ -121,6 +126,7 @@ describe("ReviewHarnessController", () => { expect(controller.snapshot().results["settings-lifecycle"].status).toBe("passed"); expect(controller.snapshot().results["vault-round-trip"].status).toBe("idle"); expect(controller.snapshot().results["compatibility-review"].status).toBe("idle"); + expect(controller.snapshot().results["file-content-hash-performance"].status).toBe("idle"); }); it("runs the Vault fixture scenario only when it is selected explicitly", async () => { @@ -185,6 +191,42 @@ describe("ReviewHarnessController", () => { expect(runtime.reportError).toHaveBeenCalledOnce(); }); + it("runs file content hash measurements on request without running them automatically", async () => { + const runtime = createRuntime(); + const controller = new ReviewHarnessController(runtime); + + await controller.runAutomaticScenarios(); + expect(runtime.runFileContentHashBenchmark).not.toHaveBeenCalled(); + + await controller.runScenario("file-content-hash-performance"); + + expect(runtime.runFileContentHashBenchmark).toHaveBeenCalledOnce(); + expect(controller.snapshot().results["file-content-hash-performance"]).toMatchObject({ status: "passed" }); + expect(runtime.runVaultRoundTrip).not.toHaveBeenCalled(); + }); + + it("includes file content hash measurements in Full review", async () => { + const runtime = createRuntime(); + const controller = new ReviewHarnessController(runtime); + + await controller.runAllScenarios(); + + expect(runtime.runFileContentHashBenchmark).toHaveBeenCalledOnce(); + expect(controller.snapshot().results["file-content-hash-performance"].status).toBe("passed"); + }); + + it("excludes an unexpected file content hash error from the copied report", async () => { + const runtime = createRuntime(); + runtime.runFileContentHashBenchmark = vi.fn().mockRejectedValue(new Error("private benchmark content")); + const controller = new ReviewHarnessController(runtime); + + await controller.runScenario("file-content-hash-performance"); + + expect(controller.snapshot().results["file-content-hash-performance"].status).toBe("failed"); + expect(controller.createReport()).not.toContain("private benchmark content"); + expect(runtime.reportError).toHaveBeenCalledOnce(); + }); + it("does not overlap an ID measurement with another scenario", async () => { const runtime = createRuntime(); let finish!: () => void; diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts new file mode 100644 index 00000000..7ff2d604 --- /dev/null +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts @@ -0,0 +1,149 @@ +import type { ReviewHarnessScenarioResult } from "./reviewHarnessTypes"; + +export const REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS = [ + { label: "4 KiB", bytes: 4 * 1024 }, + { label: "64 KiB", bytes: 64 * 1024 }, + { label: "1 MiB", bytes: 1024 * 1024 }, + { label: "50 MiB", bytes: 50 * 1024 * 1024 }, +] as const; + +const CONTENT_BLOCK_BYTES = 64 * 1024; +const SAMPLE_COUNT = 3; + +export interface FileContentHashBenchmarkOperations { + readonly xxHashInitialisationMs: number; + createInput(bytes: number): Blob; + sha256(content: Blob): Promise<`sha256:${string}`>; + xxHash64(content: Blob): Promise; +} + +type BenchmarkPerformance = Pick & { + readonly memory?: { readonly usedJSHeapSize: number }; +}; + +type HashAlgorithm = "SHA-256" | "xxHash64"; + +function readHeap(clock: BenchmarkPerformance): number | undefined { + try { + const bytes = clock.memory?.usedJSHeapSize; + return typeof bytes === "number" && Number.isFinite(bytes) && bytes >= 0 ? bytes : undefined; + } catch { + return undefined; + } +} + +function summary(samples: readonly number[]): string { + const sorted = [...samples].sort((a, b) => a - b); + const middle = sorted[Math.floor(sorted.length / 2)]; + return `median=${middle.toFixed(2)} ms; range=${sorted[0].toFixed(2)}–${sorted[sorted.length - 1].toFixed(2)} ms`; +} + +function hashKey(hash: `sha256:${string}` | bigint): string { + if (typeof hash === "bigint") return hash.toString(16); + return hash; +} + +export function createFixedFileContentHashInput(bytes: number): Blob { + const block = new Uint8Array(CONTENT_BLOCK_BYTES); + for (let index = 0; index < block.length; index++) block[index] = (index * 31 + 17) & 0xff; + + const parts: BlobPart[] = []; + for (let remaining = bytes; remaining > 0; remaining -= CONTENT_BLOCK_BYTES) { + parts.push(block.subarray(0, Math.min(remaining, CONTENT_BLOCK_BYTES))); + } + return new Blob(parts); +} + +export async function runReviewHarnessFileContentHashBenchmark( + operations: FileContentHashBenchmarkOperations, + clock: BenchmarkPerformance = performance, + yieldControl: () => Promise = () => new Promise((resolve) => window.setTimeout(resolve, 0)) +): Promise { + if (!Number.isFinite(operations.xxHashInitialisationMs) || operations.xxHashInitialisationMs < 0) { + throw new Error("The xxHash64 initialisation time is invalid."); + } + + const before = readHeap(clock); + let highest = before; + const sampleHeap = () => { + const value = readHeap(clock); + if (value !== undefined) highest = Math.max(highest ?? value, value); + return value; + }; + + const inputs: { readonly label: string; readonly bytes: number; readonly content: Blob }[] = []; + for (const { label, bytes } of REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS) { + await yieldControl(); + inputs.push({ label, bytes, content: operations.createInput(bytes) }); + sampleHeap(); + } + + const hash = (algorithm: HashAlgorithm, content: Blob) => + algorithm === "SHA-256" ? operations.sha256(content) : operations.xxHash64(content); + const warmupTimes = new Map(); + const signatures = new Map(); + const checkHash = (algorithm: HashAlgorithm, label: string, result: `sha256:${string}` | bigint): void => { + const value = hashKey(result); + const key = `${algorithm}:${label}`; + const previous = signatures.get(key); + if (previous !== undefined && previous !== value) { + throw new Error(`The ${algorithm} result changed for fixed in-memory content.`); + } + signatures.set(key, value); + }; + + for (const algorithm of ["SHA-256", "xxHash64"] as const) { + await yieldControl(); + const started = clock.now(); + const result = await hash(algorithm, inputs[0].content); + const elapsed = clock.now() - started; + checkHash(algorithm, inputs[0].label, result); + warmupTimes.set(algorithm, elapsed); + sampleHeap(); + } + + const observations = [ + "Fixed in-memory Blob inputs contain the same deterministic byte sequence at 4 KiB, 64 KiB, 1 MiB, and 50 MiB. Input construction is excluded from timings.", + `xxHash64 engine initialisation: ${operations.xxHashInitialisationMs.toFixed(2)} ms; measured separately.`, + `Warm-up on 4 KiB, one call per algorithm: SHA-256=${warmupTimes.get("SHA-256")!.toFixed(2)} ms; xxHash64=${warmupTimes.get("xxHash64")!.toFixed(2)} ms; excluded from samples.`, + "Each algorithm has three samples per input. SHA-256 uses the Self-hosted LiveSync file content hash helper; xxHash64 timings include Blob.arrayBuffer() and h64Raw. Calls run serially, with alternating algorithm order and a UI yield between calls.", + ]; + + for (const input of inputs) { + const samples: Record = { "SHA-256": [], xxHash64: [] }; + for (let sample = 0; sample < SAMPLE_COUNT; sample++) { + const order: readonly HashAlgorithm[] = + sample % 2 === 0 ? ["SHA-256", "xxHash64"] : ["xxHash64", "SHA-256"]; + for (const algorithm of order) { + await yieldControl(); + const started = clock.now(); + const result = await hash(algorithm, input.content); + const elapsed = clock.now() - started; + checkHash(algorithm, input.label, result); + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error(`The ${algorithm} measurement returned an invalid duration.`); + } + samples[algorithm].push(elapsed); + sampleHeap(); + } + } + for (const algorithm of ["SHA-256", "xxHash64"] as const) { + observations.push(`${algorithm}, ${input.label}: ${summary(samples[algorithm])}.`); + } + } + + const after = sampleHeap(); + if (highest === undefined) { + observations.push("JavaScript heap: unavailable on this device."); + } else { + const mib = (bytes: number | undefined) => + bytes === undefined ? "unavailable" : `${(bytes / 1048576).toFixed(2)} MiB`; + observations.push( + `JavaScript heap: before=${mib(before)}; highest sampled=${mib(highest)}; after=${mib(after)}.` + ); + } + observations.push( + "Heap samples are approximate, may include other Obsidian work, and are affected by garbage collection. They are neither total app RAM nor a true peak." + ); + return { status: "passed", detail: "File content hash measurements completed.", observations }; +} diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts new file mode 100644 index 00000000..a3ffc39a --- /dev/null +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from "vitest"; +import { + REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS, + createFixedFileContentHashInput, + runReviewHarnessFileContentHashBenchmark, + type FileContentHashBenchmarkOperations, +} from "./reviewHarnessFileContentHashBenchmark"; + +function fixture() { + let elapsed = 0; + let activeCalls = 0; + let maximumConcurrentCalls = 0; + let yieldCount = 0; + const contentSizes = new WeakMap(); + const contentBySize = new Map(); + const callCounts = new Map(); + const observedInputs = new Map(); + const operations: FileContentHashBenchmarkOperations = { + xxHashInitialisationMs: 12.5, + createInput: (bytes) => { + elapsed += 10000; + const content = new Blob([new Uint8Array(4)]); + contentSizes.set(content, bytes); + contentBySize.set(bytes, content); + return content; + }, + sha256: async (content) => { + await measure("SHA-256", content); + return `sha256:${contentSizes.get(content)!}`; + }, + xxHash64: async (content) => { + await measure("xxHash64", content); + return BigInt(contentSizes.get(content)!); + }, + }; + + async function measure(algorithm: string, content: Blob): Promise { + activeCalls++; + maximumConcurrentCalls = Math.max(maximumConcurrentCalls, activeCalls); + const bytes = contentSizes.get(content)!; + const key = `${algorithm}:${bytes}`; + const count = callCounts.get(key) ?? 0; + callCounts.set(key, count + 1); + observedInputs.set(key, [...(observedInputs.get(key) ?? []), content]); + const durations = + bytes === 4 * 1024 + ? algorithm === "SHA-256" + ? [100, 1, 3, 5] + : [200, 11, 13, 15] + : algorithm === "SHA-256" + ? [1, 3, 5] + : [11, 13, 15]; + await Promise.resolve(); + elapsed += durations[count]; + activeCalls--; + } + + return { + operations, + now: () => elapsed, + yieldControl: async () => { + yieldCount++; + await Promise.resolve(); + }, + inputs: () => ({ contentBySize, callCounts, observedInputs, yieldCount, maximumConcurrentCalls }), + }; +} + +describe("Review Harness file content hash measurements", () => { + it("measures fixed Blob inputs with separate initialisation, warm-up, and sample ranges", async () => { + const f = fixture(); + const result = await runReviewHarnessFileContentHashBenchmark(f.operations, { now: f.now }, f.yieldControl); + const report = result.observations.join("\n"); + + expect(result.status).toBe("passed"); + expect(result.detail).toBe("File content hash measurements completed."); + expect([...f.inputs().contentBySize.keys()]).toEqual( + REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS.map(({ bytes }) => bytes) + ); + expect(Object.fromEntries(f.inputs().callCounts)).toEqual({ + "SHA-256:4096": 4, + "xxHash64:4096": 4, + ...Object.fromEntries( + REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS.slice(1).flatMap(({ bytes }) => [ + [`SHA-256:${bytes}`, 3], + [`xxHash64:${bytes}`, 3], + ]) + ), + }); + expect(f.inputs().yieldCount).toBe(30); + expect(f.inputs().maximumConcurrentCalls).toBe(1); + for (const { bytes } of REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS) { + for (const algorithm of ["SHA-256", "xxHash64"]) { + expect(f.inputs().observedInputs.get(`${algorithm}:${bytes}`)?.every( + (content) => content === f.inputs().contentBySize.get(bytes) + )).toBe(true); + } + } + expect(report).toContain("xxHash64 engine initialisation: 12.50 ms; measured separately."); + expect(report).toContain( + "Warm-up on 4 KiB, one call per algorithm: SHA-256=100.00 ms; xxHash64=200.00 ms; excluded from samples." + ); + expect(report).toContain("SHA-256, 4 KiB: median=3.00 ms; range=1.00–5.00 ms."); + expect(report).toContain("xxHash64, 4 KiB: median=13.00 ms; range=11.00–15.00 ms."); + expect(report).toContain("Each algorithm has three samples per input."); + expect(report).not.toContain("sha256:4096"); + expect(report).not.toContain("4096n"); + }); + + it("uses the same deterministic byte sequence for short and long inputs", async () => { + const shortInput = createFixedFileContentHashInput(4 * 1024); + const longInput = createFixedFileContentHashInput(64 * 1024); + + expect(shortInput.size).toBe(4 * 1024); + expect(longInput.size).toBe(64 * 1024); + expect(new Uint8Array(await longInput.slice(0, shortInput.size).arrayBuffer())).toEqual( + new Uint8Array(await shortInput.arrayBuffer()) + ); + }); + + it("propagates hash failures without turning a partial run into a passed measurement", async () => { + const f = fixture(); + const operations = { + ...f.operations, + sha256: async () => { + throw new Error("synthetic hashing failure"); + }, + }; + + await expect( + runReviewHarnessFileContentHashBenchmark(operations, { now: f.now }, f.yieldControl) + ).rejects.toThrow("synthetic hashing failure"); + }); +}); diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts new file mode 100644 index 00000000..8d6b8831 --- /dev/null +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts @@ -0,0 +1,20 @@ +import { computeFileContentHash } from "@vrtmrz/livesync-commonlib/hashing"; +import { xxhashNew } from "octagonal-wheels/hash/xxhash.js"; +import { + createFixedFileContentHashInput, + type FileContentHashBenchmarkOperations, +} from "./reviewHarnessFileContentHashBenchmark"; + +export async function createFileContentHashBenchmarkOperations( + clock: Pick = performance +): Promise { + const started = clock.now(); + const xxHash = await xxhashNew(); + const xxHashInitialisationMs = clock.now() - started; + return { + xxHashInitialisationMs, + createInput: createFixedFileContentHashInput, + sha256: (content) => computeFileContentHash(content), + xxHash64: async (content) => xxHash.h64Raw(new Uint8Array(await content.arrayBuffer())), + }; +} diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts new file mode 100644 index 00000000..a5d6db38 --- /dev/null +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { createFileContentHashBenchmarkOperations } from "./reviewHarnessFileContentHashBenchmarkRuntime"; + +describe("Review Harness file content hash benchmark adapter", () => { + it("uses the packaged SHA-256 helper and xxHash64 engine with fixed Blob content", async () => { + const operations = await createFileContentHashBenchmarkOperations(); + const content = operations.createInput(4 * 1024); + + expect(operations.xxHashInitialisationMs).toBeGreaterThanOrEqual(0); + expect(await operations.sha256(content)).toMatch(/^sha256:/u); + expect(typeof (await operations.xxHash64(content))).toBe("bigint"); + expect(await operations.sha256(content)).toBe(await operations.sha256(content)); + expect(await operations.xxHash64(content)).toBe(await operations.xxHash64(content)); + }); +}); diff --git a/src/main.ts b/src/main.ts index 7ea2f730..72f2ff08 100644 --- a/src/main.ts +++ b/src/main.ts @@ -115,6 +115,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { events: services.context.events, API: services.API, database: services.database, + path: services.path, appLifecycle: services.appLifecycle, setting: services.setting, remote: services.remote, diff --git a/src/serviceFeatures/useReviewHarness.ts b/src/serviceFeatures/useReviewHarness.ts index cb56eb93..99e2cd5a 100644 --- a/src/serviceFeatures/useReviewHarness.ts +++ b/src/serviceFeatures/useReviewHarness.ts @@ -17,6 +17,8 @@ import { import type { CompatibilityReviewController } from "./compatibilityReview"; import { runReviewHarnessIdBenchmark } from "@/features/ReviewHarness/reviewHarnessIdBenchmark"; import { createIdBenchmarkOperations } from "@/features/ReviewHarness/reviewHarnessIdBenchmarkRuntime"; +import { runReviewHarnessFileContentHashBenchmark } from "@/features/ReviewHarness/reviewHarnessFileContentHashBenchmark"; +import { createFileContentHashBenchmarkOperations } from "@/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime"; async function runVaultRoundTrip(plugin: ObsidianLiveSyncPlugin): Promise { const vault = plugin.app.vault; @@ -62,6 +64,11 @@ export function useReviewHarness( runVaultRoundTrip: () => runVaultRoundTrip(plugin), runIdBenchmark: async () => runReviewHarnessIdBenchmark(await createIdBenchmarkOperations(), activeWindow.performance), + runFileContentHashBenchmark: async () => + runReviewHarnessFileContentHashBenchmark( + await createFileContentHashBenchmarkOperations(activeWindow.performance), + activeWindow.performance + ), readContinuation: () => services.setting.getSmallConfig(REVIEW_HARNESS_STATE_KEY), writeContinuation: (value) => services.setting.setSmallConfig(REVIEW_HARNESS_STATE_KEY, value), deleteContinuation: () => services.setting.deleteSmallConfig(REVIEW_HARNESS_STATE_KEY), diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 4030b4f8..c9f2878e 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -87,7 +87,7 @@ The wrapper accepts only maintained real-Obsidian scenario names; run it with `- `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. -`stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file with exact provenance receives the newer database content without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. +`stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. The principal entry points are: diff --git a/test/e2e-obsidian/scripts/review-harness.ts b/test/e2e-obsidian/scripts/review-harness.ts index 3915f58a..49e22c67 100644 --- a/test/e2e-obsidian/scripts/review-harness.ts +++ b/test/e2e-obsidian/scripts/review-harness.ts @@ -204,6 +204,10 @@ async function runAutomaticScenarios(): Promise { .getByText("Passed:", { exact: false }) .waitFor({ state: "visible", timeout: uiTimeoutMs }); } + await harness + .locator('[data-testid="review-harness-result-file-content-hash-performance"]') + .getByText("Not run:", { exact: false }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); }); } @@ -252,6 +256,52 @@ async function runIdBenchmark(): Promise { }); } +async function runFileContentHashBenchmark(): Promise { + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const snapshotSettings = () => + page.evaluate(() => { + const plugin = (globalThis as ReviewHarnessTestGlobal).app?.plugins?.plugins["obsidian-livesync"] as { + core: { services: { setting: { currentSettings(): unknown } } }; + }; + return JSON.stringify(plugin.core.services.setting.currentSettings()); + }); + const before = await snapshotSettings(); + const harness = page.locator('[data-testid="review-harness"]'); + await harness + .locator('[data-testid="review-harness-run-file-content-hash-performance"]') + .click({ timeout: uiTimeoutMs }); + const result = harness.locator('[data-testid="review-harness-result-file-content-hash-performance"]'); + await result.getByText("Passed:", { exact: false }).waitFor({ state: "visible", timeout: uiTimeoutMs * 10 }); + const observations = await result.locator("li").allTextContents(); + for (const label of ["4 KiB", "64 KiB", "1 MiB", "50 MiB"]) { + for (const algorithm of ["SHA-256", "xxHash64"]) { + if ( + !observations.some( + (line) => line.startsWith(`${algorithm}, ${label}: median=`) && line.includes("; range=") + ) + ) { + throw new Error(`Missing file content hash timing and range: ${algorithm}, ${label}`); + } + } + } + for (const label of [ + "Input construction is excluded from timings.", + "xxHash64 engine initialisation:", + "Warm-up on 4 KiB", + "Calls run serially", + "JavaScript heap:", + ]) { + if (!observations.some((line) => line.includes(label))) { + throw new Error(`Missing file content hash measurement detail: ${label}`); + } + } + if ((await snapshotSettings()) !== before) throw new Error("The benchmark changed the live settings."); + await assertNoHorizontalOverflow(page, harness, { label: "file content hash benchmark results" }); + const timings = observations.filter((line) => line.startsWith("SHA-256,") || line.startsWith("xxHash64,")); + console.log(`File content hash benchmark observations:\n${timings.join("\n")}`); + }); +} + async function runVaultFixture(): Promise { await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { await page.locator('[data-testid="review-harness-run-vault-round-trip"]').click({ timeout: uiTimeoutMs }); @@ -409,6 +459,7 @@ async function verifyMobileHarness(): Promise { "review-harness-run-full", "review-harness-copy-report", "review-harness-run-id-generation-performance", + "review-harness-run-file-content-hash-performance", ]) { await assertLocatorHasMinimumTouchTarget(page, harness.locator(`[data-testid="${testId}"]`), { label: testId, @@ -482,6 +533,7 @@ async function main(): Promise { const resumedScreenshot = await restartAndResumeHarness(); await completeResumedCompatibilityStep(); await runIdBenchmark(); + await runFileContentHashBenchmark(); const report = await copyAndReadReport(); if (!report.includes("## Self-hosted LiveSync Review Harness report")) { throw new Error("The copied Review Harness report was not Markdown evidence."); @@ -491,6 +543,10 @@ async function main(): Promise { "; per ID=", "ID key derivation at save time:", "JavaScript heap:", + "SHA-256, 4 KiB: median=", + "xxHash64, 4 KiB: median=", + "SHA-256, 50 MiB: median=", + "xxHash64, 50 MiB: median=", ]) { if (!report.includes(expected)) throw new Error(`Missing copied benchmark observation: ${expected}`); } diff --git a/test/e2e-obsidian/scripts/stale-file-restart.ts b/test/e2e-obsidian/scripts/stale-file-restart.ts index 7a7f3d33..136f6969 100644 --- a/test/e2e-obsidian/scripts/stale-file-restart.ts +++ b/test/e2e-obsidian/scripts/stale-file-restart.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { evalObsidianJson } from "../runner/cli.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { @@ -11,6 +12,7 @@ import { createTemporaryVault } from "../runner/vault.ts"; const paths = ["stale-known.md", "stale-unknown.md"]; const recoveredPath = "stale-recovered.md"; +const legacyPath = "stale-legacy.md"; const oldContent = "# Note\nKeep\n\nTail\n\nFooter\n"; const newContent = oldContent.replace( "Footer\n", @@ -18,7 +20,15 @@ const newContent = oldContent.replace( ); type Branch = { rev: string; content: string; history: string[] }; -type FileState = { path: string; content: string; rev: string; branches: Branch[]; provenance: string | null }; +type FileState = { + path: string; + content: string; + rev: string; + branches: Branch[]; + provenance: string | null; + contentHash: string | null; + contentSize: number | null; +}; async function readState(cliBinary: string, env: NodeJS.ProcessEnv): Promise { return await evalObsidianJson( @@ -27,7 +37,7 @@ async function readState(cliBinary: string, env: NodeJS.ProcessEnv): Promise { await evalObsidianJson( cliBinary, `(async()=>{ - for(const path of ${JSON.stringify([...paths, recoveredPath])}) await app.vault.create(path,${JSON.stringify(oldContent)}); + for(const path of ${JSON.stringify([...paths, recoveredPath, legacyPath])}) await app.vault.create(path,${JSON.stringify(oldContent)}); return JSON.stringify(true); })()`, session.cliEnv ); - for (const path of [...paths, recoveredPath]) await waitForLocalDatabaseEntry(cliBinary, session.cliEnv, path); + for (const path of [...paths, recoveredPath, legacyPath]) + await waitForLocalDatabaseEntry(cliBinary, session.cliEnv, path); // Drain real Vault events before creating a persisted pending-event fixture. // The DB advances without reflecting it in the Vault, as on an offline device. - const fixture = await evalObsidianJson<{ current: string[]; original: string[]; recovered: string }>( + const fixture = await evalObsidianJson<{ + current: string[]; + original: string[]; + recovered: string; + legacy: string; + }>( cliBinary, `(async()=>{ const core=app.plugins.plugins['obsidian-livesync'].core; @@ -106,16 +123,32 @@ async function main(): Promise { const result=await core.localDatabase.putDBEntry({...meta,data,mtime:file.stat.mtime+60000, size:data.size,children:[]},false,meta._rev); if(!result?.ok) throw new Error('Could not advance '+path); - current.push(result.rev); original.push(meta._rev); - if(index===0) await store.set(path,{revision:meta._rev,observedStorageMtime:file.stat.mtime}); - else await store.delete(path); + let currentRev=result.rev; + if(index===0){ + const record=await store.get(path); + if(record?.revision!==meta._rev || !record.contentHash) throw new Error('Initial save did not record content evidence'); + // Advance beyond the local retention limit without reflecting the incoming bytes. + let raw=await core.localDatabase.getRaw(meta._id,{rev:currentRev}); + for(let update=0;update<100;update++){ + const next=await core.localDatabase.putRaw({...raw,mtime:raw.mtime+1}); + currentRev=next.rev; raw={...raw,_rev:currentRev,mtime:raw.mtime+1}; + } + let pruned=false; + try { await core.localDatabase.getRaw(meta._id,{rev:meta._rev}); } + catch(error) { if(error.status!==404) throw error; pruned=true; } + if(!pruned) throw new Error('The recorded revision was not pruned'); + } else await store.delete(path); + current.push(currentRev); original.push(meta._rev); snapshot.push({type:'CHANGED',key:'CHANGED-'+path,args:{file}}); } const recovered=${JSON.stringify(recoveredPath)}; const recoveredMeta=await core.localDatabase.getDBEntryMeta(recovered,{},true); await store.delete(recovered); + const legacy=${JSON.stringify(legacyPath)}; + const legacyMeta=await core.localDatabase.getDBEntryMeta(legacy,{},true); + await store.set(legacy,{revision:legacyMeta._rev}); await core.kvDB.set('storage-event-manager-snapshot',snapshot); - return JSON.stringify({current,original,recovered:recoveredMeta._rev}); + return JSON.stringify({current,original,recovered:recoveredMeta._rev,legacy:legacyMeta._rev}); })()`, session.cliEnv ); @@ -124,7 +157,7 @@ async function main(): Promise { session = await startObsidianLiveSyncSession({ binary, cliBinary, vault }); await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); - const [known, unknown, recovered] = await readState(cliBinary, session.cliEnv); + const [known, unknown, recovered, legacy] = await readState(cliBinary, session.cliEnv); assertEqual(known.rev, fixture.current[0], "An unchanged stale file created a revision during restart."); assertEqual(known.branches.length, 1, "An unchanged stale file created a conflict."); assertEqual(known.content, newContent, "The newer DB content was not reflected after suppressing the save."); @@ -143,6 +176,15 @@ async function main(): Promise { assertEqual(recovered.content, oldContent, "Unchanged recovery modified the Vault file."); assertEqual(recovered.provenance, fixture.recovered, "Start-up did not recover missing file provenance."); + const oldHash = `sha256:${createHash("sha256").update(oldContent).digest("hex")}`; + const newHash = `sha256:${createHash("sha256").update(newContent).digest("hex")}`; + assertEqual(known.contentHash, newHash, "Reflection did not record the incoming bytes."); + assertEqual(recovered.contentHash, oldHash, "Recovered provenance lacks verified content evidence."); + assertEqual(legacy.rev, fixture.legacy, "Legacy upgrade created a database revision."); + assertEqual(legacy.content, oldContent, "Legacy upgrade changed storage."); + assertEqual(legacy.contentHash, oldHash, "Legacy provenance was not upgraded after exact verification."); + assertEqual(legacy.contentSize, Buffer.byteLength(oldContent), "Legacy provenance has the wrong byte length."); + const receivedRevision = await evalObsidianJson( cliBinary, `(async()=>{ @@ -204,7 +246,7 @@ async function main(): Promise { assertEqual(resolved.branches.length, 1, "The explicit newer-file option did not resolve the conflict."); assertEqual(resolved.content, newContent, "The newer-file option did not reflect the newer DB version."); console.log( - "Stale-file restart: missing unchanged provenance recovered; later update reflected; unknown content preserved; explicit newer-file resolution retained." + "Stale-file restart: pruned revision recognised from persisted content evidence; legacy provenance upgraded; missing unchanged provenance recovered; later update reflected; unknown content preserved; explicit newer-file resolution retained." ); } finally { if (session) await session.app.stop(); diff --git a/updates.md b/updates.md index 31d529ec..c8dc71d6 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,15 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Fixed + +- Unchanged files can receive synchronised updates even after the local database has discarded their recorded revision, avoiding unnecessary conflicts. (#1207) + - Saved content evidence survives revision-history pruning. Genuine local edits remain protected, and older records are upgraded only after their content is verified. + +### Maintenance + +- The Review Harness can compare SHA-256 and xxHash64 file-content hashing on the current device, with fixed inputs from 4 KiB to 50 MiB and separate initialisation and warm-up measurements. + ## 1.0.37 16th October, 2026 From ddb4d52931515e4652b43a99d7e36d968845f5d5 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 13:31:19 +0000 Subject: [PATCH 06/68] Add opt-in Unicode NFC path identity for CouchDB --- docs/adr/2026_10_unicode_path_identity.md | 135 ++++++ docs/design_docs/unicode_path_identity.md | 263 ++++++++++++ docs/settings.md | 12 + package-lock.json | 8 +- package.json | 2 +- src/apps/cli/README.md | 48 ++- .../cli/adapters/NodeFileSystemAdapter.ts | 71 +++- .../NodeFileSystemAdapter.unit.spec.ts | 388 +++++++++++++++++- src/apps/cli/adapters/NodeVaultAdapter.ts | 13 +- .../UnicodeResolvingNodeStorageAdapter.ts | 205 +++++++++ src/apps/cli/commands/runCommand.ts | 45 +- src/apps/cli/commands/runCommand.unit.spec.ts | 71 ++++ src/apps/cli/commands/types.ts | 4 + src/apps/cli/commands/utils.ts | 12 + src/apps/cli/commands/utils.unit.spec.ts | 16 +- src/apps/cli/main.bootstrap.unit.spec.ts | 60 ++- src/apps/cli/main.ts | 31 +- src/apps/cli/main.unit.spec.ts | 26 +- .../cli/serviceModules/CLIServiceModules.ts | 1 + src/apps/cli/serviceModules/FileAccessCLI.ts | 29 +- .../serviceModules/FSAPIServiceModules.ts | 1 + src/common/messagesJson/en.json | 4 + src/common/messagesYAML/en.yaml | 5 + src/common/replicatorConfigurationIdentity.ts | 18 +- ...plicatorConfigurationIdentity.unit.spec.ts | 24 ++ src/main.ts | 7 +- .../SettingDialogue/AdvancedSettingSpecs.ts | 13 + .../SettingDialogue/LiveSyncSetting.ts | 23 ++ .../ObsidianLiveSyncSettingTab.ts | 83 +++- .../ObsidianLiveSyncSettingTab.unit.spec.ts | 254 +++++++++++- .../SettingDialogue/PaneAdvanced.unit.spec.ts | 4 +- .../features/SettingDialogue/SettingSpec.ts | 69 +++- .../SettingDialogue/SettingSpec.unit.spec.ts | 47 ++- .../SettingsPageCatalogue.unit.spec.ts | 1 + .../replication/ReplicateResultProcessor.ts | 26 +- .../ReplicateResultProcessor.unit.spec.ts | 63 +++ test/e2e-obsidian/README.md | 4 + .../scripts/cli-to-obsidian-sync.ts | 250 ++++++++++- updates.md | 6 + 39 files changed, 2245 insertions(+), 97 deletions(-) create mode 100644 docs/adr/2026_10_unicode_path_identity.md create mode 100644 docs/design_docs/unicode_path_identity.md create mode 100644 src/apps/cli/adapters/UnicodeResolvingNodeStorageAdapter.ts diff --git a/docs/adr/2026_10_unicode_path_identity.md b/docs/adr/2026_10_unicode_path_identity.md new file mode 100644 index 00000000..eb503b50 --- /dev/null +++ b/docs/adr/2026_10_unicode_path_identity.md @@ -0,0 +1,135 @@ +--- +date: 2026-10-17 +commonlib-version: "0.1.42" +self-hosted-livesync-version: "1.0.37" +status: unreleased +--- + +# Architectural Decision Record: Unicode Path Identity for CouchDB + +## Status + +Accepted on 2026-10-17. The feature remains unreleased. Local validation +evidence and platform limits are recorded in the design document. + +## Context + +Self-hosted LiveSync derives normal-file Metadata document IDs from Vault paths. +Unicode permits visually identical text to use different code-point sequences. +For example, `Ê` can be represented as one precomposed character or as `e` +followed by a combining acute accent. Filesystems and applications can expose +these forms differently. Without a shared canonical form, two clients can +derive different document IDs for the same logical path, or fail to resolve an +existing document when another client reports an equivalent spelling. + +Changing only a lookup performed when receiving a document is insufficient. +New documents, edits, deletions, local provenance, and resumable Fetch +checkpoints must all use the same path identity. Applying the change to a +populated database in place would +also leave old document IDs and revision histories behind. Distinct files or +Metadata documents can collapse to one identity under NFC, so automatic +selection or merging could lose user data. + +The Metadata path and file content have separate roles: Metadata stores file +properties and references to Chunks, while Chunks store the content. A path +identity change must not require physical files to be renamed or content to be +rewritten in Metadata. + +## Decision + +Add the opt-in `pathNormalizationVersion` setting. An absent value and `0` +retain the existing path-to-ID behaviour. Version `1` applies Unicode +Normalization Form C (NFC) to the logical path used for Metadata IDs and path +comparison keys. Existing case handling remains an independent rule; CLI path +resolution uses NFC equivalence regardless of the `handleFilenameCaseSensitive` +setting. NFC changes logical identity only: it does not rewrite the stored +`Metadata.path`, rename a physical file, or change Chunk IDs or content. It is +independent of `idDerivationVersion` and does not require encryption. + +When the CLI resolves a path, it uses the actual spelling of each existing +file and directory. A missing component is created under the actual resolved +parent using the spelling received for that component. Obsidian resolves paths +through its `TFile` and Vault APIs. Its collision checks are limited to the +paths those APIs expose and do not promise to find raw physical aliases hidden +by the Obsidian index. + +The initial support boundary is CLI and Obsidian clients using CouchDB. +Journal and P2P reject version `1`; Direct file access refuses a remote which +declares the NFC feature. Other host adapters are not supported for version `1` +until they provide equivalent path resolution and ambiguity handling within +their supported storage listing. + +Version `1` is represented by the required `nfc-path-identity-v1` feature in +the existing generation-13 database version document. Every client must agree +on the path identity before it can write Metadata. A mode `0` client cannot +join a database requiring this feature. An existing database without the +feature cannot acquire it through an ordinary Metadata write while it already +contains Metadata. These admission checks remain active when ordinary +configuration-mismatch warnings are disabled. + +`pathNormalizationVersion` is a required Tweak even when encryption is +disabled. It does not replace or modify `idDerivationVersion`. + +The supported migration is reconstruction, not an in-place ID rewrite: + +1. Update every participating client and stop synchronisation. +2. Select a complete, authoritative Vault and resolve NFC-equivalent files or + Metadata documents which would produce the same ID. +3. Set version `1` on the source client and Rebuild both databases from that + Vault. +4. Set version `1` on each remaining client and Fetch the rebuilt remote. +5. Resume synchronisation only after every client has the same path identity. + +In Obsidian, the setting remains pending through ordinary settings saves, the +settings-only Apply fallback, and reload. It becomes active only when the +existing Rebuild or Fetch operation is successfully reserved. A `remoteOnly` +operation cannot apply a pending identity change, and the settings-only +fallback is not offered. The Setup URI shares +`pathNormalizationVersion` so a new client receives the intended value; +importing that value does not replace the required Rebuild or Fetch operation. + +The CLI exposes explicit `rebuild` and `fetch` commands. Both require +`--force` before application start-up because they replace local or remote +database state. `--force` confirms the requested operation; it does not bypass +identity checks, collision preflight, compatibility checks, or required human +remediation. The commands use the existing Commonlib Rebuilder; this change +does not introduce a second migration engine. + +The migration procedure also uses `--write-settings` to retain the selected +local database and settings across subsequent CLI invocations. Collision +preflight uses the supported host listing and local Metadata. It refuses +ambiguous file or parent-directory candidates before database reconstruction; +it does not claim to detect physical aliases which a host does not expose. + +## Alternatives considered + +- **Use NFC by default:** rejected because existing clients and databases must + retain their current IDs until their owners deliberately migrate them. +- **Normalise only lookups:** rejected because new Metadata writes and deletes + could continue using legacy IDs and create duplicate documents. +- **Change `idDerivationVersion`:** rejected because it controls cryptographic + ID derivation, not Unicode canonicalisation, and must remain meaningful when + encryption is disabled. +- **Rename files to NFC:** rejected because the decision concerns logical + identity, while physical spelling belongs to the Vault and its filesystem. +- **Rewrite remote IDs automatically or merge collisions:** rejected because + changing document IDs cannot preserve CouchDB revision ancestry, and the + system cannot choose safely between distinct file histories. + +## Consequences + +Legacy mode remains unchanged, and NFC identity requires a coordinated, +destructive Rebuild and Fetch. An ambiguity found in the supported storage +listing or local Metadata stops reconstruction before either database is +reset; the user must resolve the names or synchronisation scope. The check is +bounded by each host's listing contract: Obsidian uses its `TFile` and Vault +APIs and does not scan the physical Vault for aliases hidden by that index. +No claim is made that raw physical aliases outside a host's supported listing +are detected. + +## References + +- [Unicode path identity design](../design_docs/unicode_path_identity.md) +- [Setting details](../settings.md#unicode-path-identity) +- [Commonlib Unicode path identity contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md) +- [Project glossary](../glossary.md) diff --git a/docs/design_docs/unicode_path_identity.md b/docs/design_docs/unicode_path_identity.md new file mode 100644 index 00000000..7c3fd9f8 --- /dev/null +++ b/docs/design_docs/unicode_path_identity.md @@ -0,0 +1,263 @@ +--- +date: 2026-10-17 +commonlib-version: "0.1.42" +self-hosted-livesync-version: "1.0.37" +status: unreleased +--- + +# Unicode path identity + +## Purpose and baseline + +This design defines how Self-hosted LiveSync opts into Unicode NFC for normal +file Metadata document IDs, how hosts preserve the Vault's physical names, and +how existing databases move to the new identity. The accepted rationale is in +the [Unicode path identity ADR](../adr/2026_10_unicode_path_identity.md). +Commonlib's [consumer contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md) +defines the shared package boundary. + +This records an unreleased change to Self-hosted LiveSync `1.0.37`, using the +published Commonlib `0.1.42` package as an exact dependency. + +## Scope + +| Area | Contract | +| --- | --- | +| Logical identity | Version `1` uses NFC for Metadata ID and path comparison keys. Existing case handling remains a separate rule. Missing and `0` retain legacy behaviour. | +| Stored paths and content | Keep the spelling in `Metadata.path` and existing physical Vault names. NFC does not change Chunk IDs or content. | +| Initial hosts and remote | Support CLI and Obsidian with CouchDB. | +| Other transports | Journal and P2P reject version `1`; Direct file access refuses a remote declaring the NFC feature. Other host adapters remain outside the support matrix. | +| Migration | Rebuild both databases from one complete authoritative Vault, then Fetch each remaining client. Do not rewrite IDs or merge histories in place. | + +## Path identity contract + +`pathNormalizationVersion` selects the logical path used by Commonlib's +`PathService` for normal-file Metadata IDs and path comparison keys: + +| Value | Path identity | +| --- | --- | +| Missing or `0` | Preserve the legacy path and ID behaviour exactly. | +| `1` | Apply Unicode NFC to the logical path, then apply the existing case handling and ID derivation rules. | +| Any other value | Reject it; do not fall back to legacy mode. | + +NFC changes only logical identity and comparison keys. It is not a physical +filename conversion, a rewrite of `Metadata.path`, or a Unicode compatibility +transformation such as NFKC. Existing case handling remains governed by +`handleFilenameCaseSensitive`; it is independent of NFC equivalence resolution +in the CLI. `idDerivationVersion` continues to select its separate +cryptographic ID construction. NFC path identity works with or without +encryption and does not change Chunk IDs or content. + +The version is a required Tweak, even when encryption is disabled, represented +by `nfc-path-identity-v1` in the existing generation-13 database version +document's `used_features` list. The Setup URI includes +`pathNormalizationVersion`, decodes an absent value as `0`, and rejects +unsupported values. Sharing the setting communicates the target identity; +setup still requires an explicit Rebuild or Fetch choice. + +Fast Fetch checkpoints include the path identity version. A checkpoint created +under legacy identity cannot be resumed under version `1`, or vice versa. This +reuses the existing checkpoint identity and does not add a separate recovery +store. + +## Admission and database compatibility + +Clients validate local and remote database identity before accepting ordinary +replication, including when configuration-mismatch warnings are disabled. The +checks cover both directions: + +| Client setting | Database feature | Result | +| --- | --- | --- | +| Missing or `0` | No NFC feature | Continue with legacy identity. | +| `1` | NFC feature present | Continue with NFC identity. | +| `1` | Neither database has the feature or Metadata | A fresh database may declare the feature before its first Metadata mutation. | +| `1` | A database has Metadata without the feature | Reject ordinary writes; require Rebuild. | +| Missing or `0` | A local or remote database has the NFC feature | Reject; the database requires a version-`1` client. | + +The feature is independent of encryption. Receiving an incompatible version +document records the incompatibility and stops the active synchronisation +session. This is a compatibility fence, not a rollback of operations already +admitted elsewhere; every device must be stopped before migration begins. + +## Host path resolution and collision preflight + +Commonlib owns logical ID derivation and the Rebuilder's preflight contract. +Each host maps that logical path to entries in its supported storage listing. + +The CLI resolves existing files and directories using NFC comparison keys, +regardless of `handleFilenameCaseSensitive`. It carries the actual spelling +reported for each existing component through reads, writes, deletes, stat, +directory creation, and rename operations. If a path has missing components, +the CLI uses the actual existing parent and preserves the received spelling for +each new component. A scan enumerates each directory once and stats the raw +paths returned by that enumeration; it does not repeat canonical lookup for +each listed entry. No persistent index or recovery store is added. + +If more than one entry resolves to the same logical comparison key after NFC +and the configured case rule at any component, including a parent directory, +the path is ambiguous. The CLI warns and refuses the operation; it must not +select a candidate based on directory enumeration order, merge the entries, or +rename them. The same rule applies during Rebuild preflight. + +Obsidian resolution remains within its `TFile` and Vault API boundary. Its +collision preflight checks paths exposed through those APIs; it does not walk +the physical Vault independently and cannot promise to detect raw filesystem +aliases hidden by Obsidian's index. Other host adapters remain outside the +version-`1` support matrix until their path resolution and ambiguity handling +are implemented and verified within their own supported listings. + +Before resetting either database, Rebuild checks the supported storage +listing and local Metadata for paths which would resolve to the same identity +under NFC and the configured case rule. When a local database is not open, +preflight checks the storage paths returned by the host listing. If multiple +candidate files or Metadata documents collide, or a parent component is +ambiguous, the operation stops before local or remote mutation. It does not +choose a winner, merge revision histories, or rename a Vault file. The user +must resolve the ambiguous names or exclude one file from the synchronisation +scope, then retry the reconstruction. This check is bounded by the host's +listing contract and does not imply that every raw physical alias is visible. + +## Reconstruction and client operations + +Mode changes are never applied by an ordinary Metadata write. In Obsidian, the +Advanced page offers **Legacy path identity** (`0`) and **Unicode NFC path +identity** (`1`) for CouchDB. A changed `pathNormalizationVersion` remains +pending through ordinary settings saves, the settings-only Apply fallback, and +reload. It becomes active only after the existing Rebuild or Fetch operation +has been successfully reserved; cancellation or failed reservation leaves it +pending. A `remoteOnly` operation cannot apply a pending identity change, and +the settings-only fallback is not offered. + +Migration uses this ordered procedure: + +1. Update every CLI and Obsidian client which will use the database. Stop + synchronisation and editing on all devices, and keep a complete backup. +2. Select the Vault whose complete file set is authoritative. Resolve any + collision reported by Rebuild before continuing. +3. Select version `1` on that source. In Obsidian, choose **Overwrite Server + Data with This Device's Files**; the new mode becomes active only when the + Rebuild reservation succeeds. In the CLI, use settings with + `pathNormalizationVersion: 1` and run `rebuild` with `--force + --write-settings`. Both local and remote databases are reconstructed from + the specified physical Vault. +4. Select version `1` on each remaining client. In Obsidian, choose **Reset + Synchronisation on This Device**; the new mode becomes active only when the + Fetch reservation succeeds. In the CLI, use settings with + `pathNormalizationVersion: 1` and run `fetch` with `--force + --write-settings`. Each operation replaces that client's local database + from the rebuilt remote. +5. Confirm that every participating client is using version `1`, then resume + synchronisation. + +The CLI checks `--force` before core start-up so an omitted confirmation cannot +trigger database access or destructive preparation. Neither `--force` nor a +headless default choice overrides an NFC collision, a feature mismatch, or a +preflight failure. If a required setup decision cannot be made in a headless +CLI invocation, the command must fail with a clear remediation message rather +than report success after cancellation. + +`--write-settings` retains the Rebuilder's selected local database name and +lasting settings changes for subsequent invocations. Without this option, the +existing CLI policy leaves the settings file unchanged, so a later command can +reopen the previously selected database. + +## State which follows the selected identity + +The path identity is also used by two existing resume mechanisms: + +- **File-reflection provenance:** the existing device-local provenance storage + interface remains an exact-key store. The private + `ServiceFileHandlerBase.getProvenancePath()` method canonicalises provenance + keys with NFC in mode `1` for reads, writes, deletes, raw snapshot lookup, + and untracked revision recording. Mode `0` retains the old keys. Rebuild and + Fetch use the existing local-state reconstruction and replacement lifecycle; + no new persistent field or in-place key migration is introduced. +- **Fast Fetch checkpoints:** the path version participates in checkpoint + identity so an interrupted transfer cannot resume using a different + path-to-ID rule. Existing checkpoint persistence and retry handling remain + in control. + +These rules keep a physical NFD filename addressable when an NFC-equivalent +Metadata path arrives, while using one canonical identity for subsequent +updates and deletions. + +## Compatibility matrix + +| Client and remote | Version `0` | Version `1` | +| --- | --- | --- | +| CLI with CouchDB | Supported with legacy identity. | Supported with NFC path resolution. | +| Obsidian with CouchDB | Supported with legacy identity. | Supported with NFC path resolution. | +| Journal or P2P | Existing behaviour. | Rejected. | +| Direct file access | Existing behaviour. | Refuses a remote requiring `nfc-path-identity-v1`. | +| Other host adapters, including WebApp | Existing behaviour, subject to their existing support. | Outside the initial support matrix until physical-path handling is verified. | +| Older client without this feature | Can use a legacy database. | Cannot join a database declaring `nfc-path-identity-v1`. | + +## Verification contract and limits + +The implementation acceptance matrix covers: + +- unchanged legacy IDs for missing and explicit version `0`, plus matching + ASCII IDs in both modes; +- identical normal-file Metadata IDs for NFC and NFD spellings in version `1`, + with and without path obfuscation, and with the existing case rule applied; +- Metadata creation, update, deletion, and file-reflection provenance using the + same canonical identity without renaming the physical file; +- CLI reads, writes, deletes, stat, directory creation, and rename through the + actual spelling of existing files and parents; preservation of the received + spelling for new components; and resolution independent of the case setting; +- refusal to write when an existing legacy database has Metadata, refusal of + version-`0` clients against an NFC database, and enforcement when ordinary + mismatch warnings are disabled; +- URI round trips, checkpoint invalidation across path versions, and transport + rejection for Journal, P2P, and Direct file access; +- refusal for ambiguous file or parent-directory candidates before local or + remote reset, preserving both databases on rejection, within the paths + returned by each host's supported listing; and +- CLI `rebuild` and `fetch` command dispatch, pre-start `--force` enforcement, + and end-to-end creation, update, and deletion through CouchDB between CLI and + Obsidian. + +Focused unit and package checks do not substitute for the real CouchDB and +Obsidian round trip using the exact published Commonlib artefact. Obsidian checks +are limited to paths exposed through its `TFile` and Vault APIs; they do not +establish detection of physical aliases hidden by the Obsidian index. No result +for raw physical filesystem aliases outside a host's supported listing is +claimed. + +Local validation on Linux with real Obsidian `1.12.7` and the published +Commonlib `0.1.42` package passed after the CLI path resolution, case-aware +collision scan, and pending-setting changes. It covered: + +- NFC identity with unencrypted Metadata; +- NFC identity with E2EE and Path Obfuscation; +- two legacy CLI databases migrated through `rebuild` and `fetch`, followed by + the encrypted Obsidian round trip; and +- the existing encrypted ASCII scenario in legacy mode. + +In those NFC scenarios, an NFD directory and mixed-form filename were retained +while checking content, Metadata creation time, remote updates, absence of an +NFC duplicate, and remote deletion through the polling CLI daemon. The `mirror` +command retains tombstoned local files by default; its update behaviour was +tested separately from daemon deletion reflection. + +## Non-goals + +This design does not: + +- change Chunk IDs, E2EE, Properties encryption, or the Metadata and Chunk + document formats; +- convert or rename existing physical files; +- migrate remote document IDs or revision histories in place; +- choose among ambiguous NFC-equivalent files or parents, or merge their + content; +- change the default away from legacy path identity; +- enable version `1` for Journal, P2P, Direct file access, WebApp, or an + unverified host adapter; or +- claim native iOS or macOS filesystem compatibility. + +## References + +- [Unicode path identity ADR](../adr/2026_10_unicode_path_identity.md) +- [Setting details](../settings.md#unicode-path-identity) +- [Commonlib Unicode path identity contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md) +- [Metadata and Chunks glossary](../glossary.md#user-facing-and-operational-terms) diff --git a/docs/settings.md b/docs/settings.md index 0abe75d2..519ff707 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -1050,6 +1050,18 @@ When this setting is disabled, changing only the letter case of a file name with New Vaults use case-insensitive handling for cross-platform compatibility. Existing settings with an explicit value preserve that choice. Earlier releases also followed the case-insensitive branch when this value was absent, so 1.0 saves a missing legacy value as `false` without requiring a compatibility review or database rebuild. +#### Unicode path identity + +Setting key: pathNormalizationVersion + +Version `0` preserves legacy file identity and is the default for existing and new settings. Version `1` uses Unicode NFC when deriving document IDs, so equivalent composed and decomposed paths identify the same file across the CLI and Obsidian. Existing physical file names are retained. This mode currently supports CouchDB only. + +Before enabling version `1`, update every participating client, stop synchronisation, and back up a complete authoritative Vault. Rebuild both databases from that Vault, then select version `1` and Fetch on each remaining device. Changing this setting without reconstruction is refused when it would mix database identities. Do not resume an older client against the migrated remote. + +The selected value stays pending through ordinary settings saves and reloads. It becomes active only when the Rebuild or Fetch operation is successfully reserved; cancellation leaves it pending. + +Distinct files or Metadata documents which derive the same identity stop reconstruction before either database is reset. This check uses the host's supported file listing; Obsidian checks the paths exposed by its Vault APIs and cannot detect physical aliases hidden by its index. Resolve the duplicate names or synchronisation scope first; Self-hosted LiveSync does not merge them automatically. Case-insensitive handling also applies the existing letter-case rule. + ### 4. Compatibility (Internal API Usage) #### Scan changes on customisation sync diff --git a/package-lock.json b/package-lock.json index 6aadafc4..e607283a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.40", + "@vrtmrz/livesync-commonlib": "0.1.42", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4389,9 +4389,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.40", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.40.tgz", - "integrity": "sha512-GVskcVFoQIpb45rR+QT9Wjtdqc6oAxYz5IB2WeJwusWZ8lTmqVrba+6YD9FTtsbzqbPOyJ/LdQxnET7SUexVtw==", + "version": "0.1.42", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.42.tgz", + "integrity": "sha512-LyWl0Mkrj9UsQXV0nOUaYJybcN3wLkpAAxCeHMZYOEizaUTQ7ezN+FgzN+OQxDRA8UXWnoatxh719NWzOwBpBw==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 16dc3c0b..fb7edb0b 100644 --- a/package.json +++ b/package.json @@ -191,7 +191,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.40", + "@vrtmrz/livesync-commonlib": "0.1.42", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 8528f494..17674fc3 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -64,9 +64,10 @@ livesync-cli [database-path] [command] [args...] - `database-path`: Path to the directory where `.livesync` folder and `settings.json` are (or will be) located. - Note: In previous versions, this was referred to as the "vault" path. Now it is clearly distinguished from the actual vault (the directory containing your `.md` files). -- `--vault ` / `-V `: (daemon/mirror only) Path to the vault directory containing `.md` files. +- `--vault ` / `-V `: (daemon/mirror/rebuild/fetch) Path to the Vault directory containing `.md` files. - Allows the PouchDB database directory and the actual vault directory to be different locations. - For `mirror` command, the positional `[vault-path]` argument takes precedence over `--vault`. +- `--force` / `-f`: Required by `rebuild` and `fetch` because they replace a database. It also retains its existing `init-settings` overwrite behaviour. - `--write-settings`: Write setting migrations and other lasting changes after the command succeeds. - `init-settings` writes its target file. `setup`, `remote-add`, `remote-rm`, `remote-set`, and `remote-activate` write their settings changes without this option. - All remaining commands leave the settings file unchanged by default. @@ -82,6 +83,9 @@ livesync-cli [database-path] [command] [args...] - If `vault-path` is provided, the CLI will synchronise the database with files in the vault directory. - If `vault-path` is omitted, it defaults to `database-path` (compatibility mode). - Use this command to keep your local `.md` files in sync with the database. +- `rebuild`: Rebuild the local and remote databases from the selected Vault. This replaces existing local and remote database state and requires `--force`. + - This command may select a new local database suffix. Add `--write-settings` if later CLI commands should use the resulting database selection; otherwise the existing settings file remains unchanged. +- `fetch`: Replace the local database with the configured remote state, then reflect fetched files to the selected Vault. Existing local database state is discarded, so this command requires `--force`. - `ls [prefix]`: List files currently stored in the local database. - `push `: Push a local file `` into the database at path ``. - `pull `: Pull a file `` from the database into local file ``. @@ -110,6 +114,12 @@ livesync-cli ./my-db sync # Mirroring to your actual Obsidian vault livesync-cli ./my-db mirror /path/to/obsidian-vault +# Rebuild local and remote databases from an authoritative Vault +livesync-cli ./my-db --vault /path/to/obsidian-vault rebuild --force --write-settings + +# Replace the local database and Vault files from the configured remote +livesync-cli ./my-db --vault /path/to/obsidian-vault fetch --force --write-settings + # Manual file operations livesync-cli ./my-db push ./note.md folder/note.md livesync-cli ./my-db pull folder/note.md ./note.md @@ -369,6 +379,36 @@ The CLI uses the same settings format as the Obsidian plug-in. Create a `.livesy - `couchDB_DBNAME`: Database name - `isConfigured`: Set to `true` after configuration +#### Unicode path identity + +The `pathNormalizationVersion` setting controls how file paths identify +Metadata documents when using CouchDB. An absent value or `0` keeps the legacy +identity. Version `1` uses Unicode Normalization Form C (NFC) for Metadata IDs +and path comparison keys. It does not rewrite `Metadata.path`, rename existing +files, or change Chunks or file content. `handleFilenameCaseSensitive` remains +an independent case-handling setting; the CLI resolves NFC-equivalent names +even when case-sensitive handling is enabled. + +When the CLI resolves an existing path, it uses the actual spelling of each +matching file or directory for reads, writes, deletes, `stat` calls, directory +creation, and renames. If a path component is missing, it uses the actual +existing parent and preserves the spelling received for the new component. If +a file or any parent directory has multiple candidates for the same logical +key after NFC and the existing case rule, the CLI warns and refuses the +operation. It does not choose a candidate based on directory order, merge +entries, or rename them. + +Version `1` requires a coordinated Rebuild and Fetch; changing the setting by +itself does not migrate a database. Update every participating client, stop +synchronisation, and back up a complete authoritative Vault. Set +`pathNormalizationVersion` to `1` on the source client, then run `rebuild` +with `--force --write-settings`. Set the same value on each remaining CLI +client, then run `fetch` with `--force --write-settings`. Use **Overwrite +Server Data with This Device's Files** and **Reset Synchronisation on This +Device** for the corresponding Obsidian Rebuild and Fetch operations. Resolve +any ambiguous names before reconstruction. Version `1` is supported with +CouchDB only; do not resume a legacy client against the rebuilt database. + ### Command-line Reference ``` @@ -381,11 +421,11 @@ Arguments: Options: --settings, -s Path to settings file (default: .livesync/settings.json in local database directory) - --force, -f Overwrite existing file on init-settings + --force, -f Required by rebuild/fetch; overwrite existing file on init-settings --verbose, -v Enable verbose logging --debug, -d Enable debug logging (includes verbose) --interval , -i (daemon only) Poll CouchDB every N seconds instead of using the _changes feed - --vault , -V (daemon/mirror) Path to vault directory, decoupled from database-path + --vault , -V (daemon/mirror/rebuild/fetch) Path to vault directory, decoupled from database-path --write-settings Write setting changes after a successful command --compat-remote-admin-exit-zero Preserve the former zero exit code when remote-administration verification fails @@ -398,6 +438,8 @@ Commands: p2p-peers Show discovered peers as [peer] p2p-sync Synchronise with specified peer-id or peer-name p2p-host Start P2P host mode and wait until interrupted (Ctrl+C) + rebuild Rebuild local and remote databases from the selected Vault (requires --force) + fetch Replace the local database with remote state and reflect it to the Vault (requires --force) push Push local file into local database path pull Pull file from local database into local file pull-rev Pull specific revision into local file diff --git a/src/apps/cli/adapters/NodeFileSystemAdapter.ts b/src/apps/cli/adapters/NodeFileSystemAdapter.ts index d386b7cf..6cb8f458 100644 --- a/src/apps/cli/adapters/NodeFileSystemAdapter.ts +++ b/src/apps/cli/adapters/NodeFileSystemAdapter.ts @@ -3,7 +3,7 @@ import type { IFileSystemAdapter } from "@vrtmrz/livesync-commonlib/compat/servi import { NodePathAdapter } from "./NodePathAdapter"; import { NodeTypeGuardAdapter } from "./NodeTypeGuardAdapter"; import { NodeConversionAdapter } from "./NodeConversionAdapter"; -import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node"; +import { AmbiguousNormalisedPathError, UnicodeResolvingNodeStorageAdapter } from "./UnicodeResolvingNodeStorageAdapter"; import { NodeVaultAdapter } from "./NodeVaultAdapter"; import type { NodeFile, NodeFolder, NodeStat } from "./NodeTypes"; import { path } from "@vrtmrz/livesync-commonlib/node"; @@ -16,19 +16,21 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter(); constructor( private basePath: string, - private reportDiagnostic: CliDiagnosticReporter = () => undefined + private reportDiagnostic: CliDiagnosticReporter = () => undefined, + getPathNormalizationVersion: () => number = () => 0, + isCaseInsensitive: () => boolean = () => false ) { this.path = new NodePathAdapter(); this.typeGuard = new NodeTypeGuardAdapter(); this.conversion = new NodeConversionAdapter(); - this.storage = new NodeStorageAdapter(basePath); + this.storage = new UnicodeResolvingNodeStorageAdapter(basePath, getPathNormalizationVersion, isCaseInsensitive); this.vault = new NodeVaultAdapter(this.storage); } @@ -58,6 +60,14 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { const pathStr = this.normalisePath(p); + if (this.storage.isUnicodePathNormalizationEnabled) { + const resolvedPath = await this.storage.resolveLogicalPath(pathStr, false, false); + if (resolvedPath === null) { + this.fileCache.delete(pathStr); + return null; + } + return await this.refreshResolvedFile(resolvedPath); + } if (!this.fileCache.has(pathStr) && !(await this.hasExactPathCase(pathStr))) { this.fileCache.delete(pathStr); return null; @@ -68,6 +78,15 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { const pathStr = this.normalisePath(p); + if (this.storage.isUnicodePathNormalizationEnabled) { + const resolvedPath = await this.storage.resolveLogicalPath(pathStr, true, false); + if (resolvedPath === null) { + this.fileCache.delete(pathStr); + return null; + } + return await this.refreshResolvedFile(resolvedPath); + } + const exact = await this.getAbstractFileByPath(pathStr); if (exact) { return exact; @@ -115,17 +134,34 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { + async refreshFile(p: string, caseInsensitive = false): Promise { const pathStr = this.normalisePath(p); try { - const stat = await this.storage.stat(pathStr); - if (stat?.type !== "file") { + const resolvedPath = this.storage.isUnicodePathNormalizationEnabled + ? await this.storage.resolveLogicalPath(pathStr, caseInsensitive, false) + : pathStr; + if (resolvedPath === null) { this.fileCache.delete(pathStr); return null; } + return await this.refreshResolvedFile(resolvedPath); + } catch (error) { + if (error instanceof AmbiguousNormalisedPathError) throw error; + this.fileCache.delete(pathStr); + return null; + } + } + + private async refreshResolvedFile(resolvedPath: string): Promise { + try { + const stat = await this.storage.statExact(resolvedPath); + if (stat?.type !== "file") { + this.fileCache.delete(resolvedPath); + return null; + } const file: NodeFile = { - path: pathStr as FilePath, + path: resolvedPath as FilePath, stat: { size: stat.size, mtime: stat.mtime, @@ -133,11 +169,12 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { - await this.scanDirectoryInto(relativePath, this.fileCache); + const resolvedPath = await this.storage.resolveLogicalPath(relativePath, false, false); + if (resolvedPath !== null) await this.scanDirectoryInto(resolvedPath, this.fileCache); } private async scanDirectoryInto(relativePath: string, files: Map): Promise { const fullPath = this.resolvePath(relativePath); try { - const directoryStat = await this.storage.stat(relativePath); + const directoryStat = this.storage.isUnicodePathNormalizationEnabled + ? await this.storage.statExact(relativePath) + : await this.storage.stat(relativePath, false); if (directoryStat?.type !== "folder") throw new Error(`Directory does not exist: ${fullPath}`); - const entries = await this.storage.list(relativePath); + const entries = await this.storage.listExact(relativePath); for (const entryPath of entries.files) { - const stat = await this.storage.stat(entryPath); + const stat = await this.storage.statExact(entryPath); if (stat?.type !== "file") continue; const file: NodeFile = { path: entryPath as FilePath, @@ -169,6 +209,7 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter { @@ -116,3 +123,382 @@ describe("NodeFileSystemAdapter file enumeration", () => { await expect(adapter.getFiles()).resolves.toEqual([]); }); }); + +describe("NodeFileSystemAdapter Unicode path resolution", () => { + const tempDirs: string[] = []; + + async function createVault() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-unicode-")); + tempDirs.push(directory); + return directory; + } + + function createNormalisingAdapter(directory: string, caseInsensitive = false) { + return new NodeFileSystemAdapter( + directory, + undefined, + () => 1, + () => caseInsensitive + ); + } + + afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true }))); + }); + + it.each([0, 1])("keeps an NFD-only file in the storage listing in identity mode %s", async (version) => { + const directory = await createVault(); + const originalPath = "cafÊ.md".normalize("NFD"); + await fs.writeFile(path.join(directory, originalPath), "original"); + const report = vi.fn(); + const adapter = new NodeFileSystemAdapter(directory, report, () => version); + + expect((await adapter.getFiles()).map((file) => file.path)).toEqual([originalPath]); + expect(report).not.toHaveBeenCalled(); + await expect(fs.readFile(path.join(directory, originalPath), "utf8")).resolves.toBe("original"); + expect(await fs.readdir(directory)).toEqual([originalPath]); + }); + + it.each([false, true])( + "enumerates nested mixed names once per directory with case-insensitive matching %s", + async (caseInsensitive) => { + const directory = await createVault(); + const parent = "cafÊ".normalize("NFD"); + const child = `${parent}/Étage`; + await fs.mkdir(path.join(directory, child), { recursive: true }); + const names = Array.from( + { length: 12 }, + (_, index) => + `${child}/${index}-rÊsumÊ.md` + .normalize(index % 2 ? "NFC" : "NFD") + .split("/") + .pop()! + ); + for (const name of names) await fs.writeFile(path.join(directory, child, name), "content"); + const adapter = createNormalisingAdapter(directory, caseInsensitive); + const list = vi.spyOn(NodeStorageAdapter.prototype, "list"); + try { + const files = await adapter.getFiles(); + expect(files.map((file) => file.path).sort()).toEqual(names.map((name) => `${child}/${name}`).sort()); + expect(list).toHaveBeenCalledTimes(3); + } finally { + list.mockRestore(); + } + } + ); + + it("uses the existing mixed parent during directory preparation and preserves the new leaf spelling", async () => { + const directory = await createVault(); + const parent = "cafÊ".normalize("NFD"); + const child = `${parent}/Étage`; + await fs.mkdir(path.join(directory, child), { recursive: true }); + const adapter = createNormalisingAdapter(directory); + await adapter.storage.mkdir("cafÊ"); + await adapter.storage.mkdir("cafÊ/Étage"); + const newName = "rÊsumÊ.md".normalize("NFD"); + const file = await adapter.vault.create(`cafÊ/Étage/${newName}`, "created"); + + expect(file.path).toBe(`${child}/${newName}`); + expect(await fs.readdir(directory)).toEqual([parent]); + expect(await fs.readdir(path.join(directory, parent))).toEqual(["Étage"]); + expect(await fs.readdir(path.join(directory, child))).toEqual([newName]); + await expect(adapter.storage.stat(`cafÊ/Étage/${newName.normalize("NFC")}`)).resolves.toMatchObject({ + size: 7, + }); + }); + + it("rejects an NFC-equivalent file and directory before scanning or writing a child", async () => { + const directory = await createVault(); + const composed = "cafÊ"; + const decomposed = composed.normalize("NFD"); + await fs.writeFile(path.join(directory, composed), "file"); + await fs.mkdir(path.join(directory, decomposed)); + const adapter = createNormalisingAdapter(directory); + + await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i); + await expect(adapter.storage.write(`${composed}/new.md`, "new")).rejects.toThrow(/ambiguous/i); + await expect(fs.readFile(path.join(directory, composed), "utf8")).resolves.toBe("file"); + expect(await fs.readdir(path.join(directory, decomposed))).toEqual([]); + }); + + it.each([false, true])( + "enumerates each directory once for a flat NFC scan with case-insensitive matching %s", + async (caseInsensitive) => { + const directory = await createVault(); + for (let index = 0; index < 12; index++) { + await fs.writeFile(path.join(directory, `${index}.md`), "content"); + } + const adapter = createNormalisingAdapter(directory, caseInsensitive); + const list = vi.spyOn(NodeStorageAdapter.prototype, "list"); + try { + expect(await adapter.getFiles()).toHaveLength(12); + expect(list).toHaveBeenCalledTimes(1); + } finally { + list.mockRestore(); + } + } + ); + + it("reads, updates, and deletes an NFD file through its NFC path", async () => { + const directory = await createVault(); + const nfcFolder = "cafÊ"; + const nfdFolder = nfcFolder.normalize("NFD"); + const nfcName = "ÅngstrÃļm.md"; + const nfdName = nfcName.normalize("NFD"); + await fs.mkdir(path.join(directory, nfdFolder)); + await fs.writeFile(path.join(directory, nfdFolder, nfdName), "before", "utf8"); + const adapter = createNormalisingAdapter(directory); + + const file = await adapter.getAbstractFileByPath(`${nfcFolder}/${nfcName}`); + expect(file?.path).toBe(`${nfdFolder}/${nfdName}`); + if (!file) throw new Error("Expected the NFC path to resolve to the NFD file"); + await expect(adapter.vault.read(file)).resolves.toBe("before"); + await adapter.vault.modify(file, "after"); + await expect(fs.readFile(path.join(directory, nfdFolder, nfdName), "utf8")).resolves.toBe("after"); + expect(await fs.readdir(directory)).toEqual([nfdFolder]); + expect(await fs.readdir(path.join(directory, nfdFolder))).toEqual([nfdName]); + + await adapter.vault.delete(file); + + await expect(fs.readdir(path.join(directory, nfdFolder))).resolves.toEqual([]); + }); + + it("creates a new file inside an existing NFD directory using that directory spelling", async () => { + const directory = await createVault(); + const nfdFolder = "cafÊ".normalize("NFD"); + await fs.mkdir(path.join(directory, nfdFolder)); + const adapter = createNormalisingAdapter(directory); + + const file = await adapter.vault.create("cafÊ/new.md", "content"); + + expect(file.path).toBe(`${nfdFolder}/new.md`); + await expect(fs.readFile(path.join(directory, nfdFolder, "new.md"), "utf8")).resolves.toBe("content"); + expect(await fs.readdir(directory)).toEqual([nfdFolder]); + }); + + it("renames a file by case inside an NFD directory while preserving that directory spelling", async () => { + const directory = await createVault(); + const nfcFolder = "Caf\u00e9"; + const nfdFolder = nfcFolder.normalize("NFD"); + await fs.mkdir(path.join(directory, nfdFolder)); + await fs.writeFile(path.join(directory, nfdFolder, "Note.md"), "content", "utf8"); + const adapter = createNormalisingAdapter(directory, true); + const file = await adapter.getAbstractFileByPathInsensitive(`${nfcFolder}/Note.md`); + if (!file) throw new Error("Expected the NFC path to resolve to the NFD file"); + + const renamedFile = await adapter.renameFile(file, `${nfcFolder}/note.md`); + + expect(renamedFile.path).toBe(`${nfdFolder}/note.md`); + expect(await fs.readdir(directory)).toEqual([nfdFolder]); + await expect(fs.readdir(path.join(directory, nfdFolder))).resolves.toEqual(["note.md"]); + await expect(fs.readFile(path.join(directory, nfdFolder, "note.md"), "utf8")).resolves.toBe("content"); + }); + + it("rejects an NFC/NFD file collision for lookup and vault mutations", async () => { + const directory = await createVault(); + const nfcName = "cafÊ.md"; + const nfdName = nfcName.normalize("NFD"); + await fs.writeFile(path.join(directory, nfcName), "NFC", "utf8"); + await fs.writeFile(path.join(directory, nfdName), "NFD", "utf8"); + const adapter = createNormalisingAdapter(directory); + const exactFile = { + path: nfcName as FilePath, + stat: { ctime: 1, mtime: 2, size: 3, type: "file" as const }, + }; + + await expect(adapter.getAbstractFileByPath(nfcName)).rejects.toThrow(/ambiguous/i); + await expect(adapter.vault.modify(exactFile, "changed")).rejects.toThrow(/ambiguous/i); + await expect(adapter.vault.delete(exactFile)).rejects.toThrow(/ambiguous/i); + + await expect(fs.readFile(path.join(directory, nfcName), "utf8")).resolves.toBe("NFC"); + await expect(fs.readFile(path.join(directory, nfdName), "utf8")).resolves.toBe("NFD"); + expect((await fs.readdir(directory)).sort()).toEqual([nfcName, nfdName].sort()); + }); + + it("rejects an NFC/NFD directory collision before creating a child", async () => { + const directory = await createVault(); + const nfcFolder = "cafÊ"; + const nfdFolder = nfcFolder.normalize("NFD"); + await fs.mkdir(path.join(directory, nfcFolder)); + await fs.mkdir(path.join(directory, nfdFolder)); + await fs.writeFile(path.join(directory, nfcFolder, "inside.md"), "NFC", "utf8"); + await fs.writeFile(path.join(directory, nfdFolder, "inside.md"), "NFD", "utf8"); + const adapter = createNormalisingAdapter(directory); + + await expect(adapter.getAbstractFileByPath(`${nfcFolder}/inside.md`)).rejects.toThrow(/ambiguous/i); + await expect(adapter.storage.write(`${nfcFolder}/created.md`, "created")).rejects.toThrow(/ambiguous/i); + + await expect(fs.readdir(path.join(directory, nfcFolder))).resolves.toEqual(["inside.md"]); + await expect(fs.readdir(path.join(directory, nfdFolder))).resolves.toEqual(["inside.md"]); + await expect(fs.readFile(path.join(directory, nfcFolder, "inside.md"), "utf8")).resolves.toBe("NFC"); + await expect(fs.readFile(path.join(directory, nfdFolder, "inside.md"), "utf8")).resolves.toBe("NFD"); + }); + + it("rejects enumeration when NFC-equivalent directories contain different files", async () => { + const directory = await createVault(); + const nfcFolder = "cafÊ"; + const nfdFolder = nfcFolder.normalize("NFD"); + await fs.mkdir(path.join(directory, nfcFolder)); + await fs.mkdir(path.join(directory, nfdFolder)); + await fs.writeFile(path.join(directory, nfcFolder, "a.md"), "NFC", "utf8"); + await fs.writeFile(path.join(directory, nfdFolder, "b.md"), "NFD", "utf8"); + const adapter = createNormalisingAdapter(directory); + + await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i); + await expect(adapter.scanDirectory()).rejects.toThrow(/ambiguous/i); + + await expect(fs.readFile(path.join(directory, nfcFolder, "a.md"), "utf8")).resolves.toBe("NFC"); + await expect(fs.readFile(path.join(directory, nfdFolder, "b.md"), "utf8")).resolves.toBe("NFD"); + }); + + it("rechecks the directory after a unique file is cached", async () => { + const directory = await createVault(); + const nfcName = "cafÊ.md"; + const nfdName = nfcName.normalize("NFD"); + await fs.writeFile(path.join(directory, nfdName), "NFD", "utf8"); + const adapter = createNormalisingAdapter(directory); + + await expect(adapter.getAbstractFileByPath(nfcName)).resolves.toMatchObject({ path: nfdName }); + await fs.writeFile(path.join(directory, nfcName), "NFC", "utf8"); + + await expect(adapter.getAbstractFileByPath(nfcName)).rejects.toThrow(/ambiguous/i); + await expect(fs.readFile(path.join(directory, nfcName), "utf8")).resolves.toBe("NFC"); + await expect(fs.readFile(path.join(directory, nfdName), "utf8")).resolves.toBe("NFD"); + }); + + it.each([ + ["Foo", "foo"], + ["CafÊ", "cafÊ".normalize("NFD")], + ])("rejects case-insensitive parent aliases %s and %s containing different files", async (first, second) => { + const directory = await createVault(); + for (const [parent, name] of [ + [first, "one.md"], + [second, "two.md"], + ]) { + await fs.mkdir(path.join(directory, parent)); + await fs.writeFile(path.join(directory, parent, name), name); + } + const adapter = createNormalisingAdapter(directory, true); + + await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i); + await expect(adapter.scanDirectory()).rejects.toThrow(/ambiguous/i); + await expect(fs.readFile(path.join(directory, first, "one.md"), "utf8")).resolves.toBe("one.md"); + await expect(fs.readFile(path.join(directory, second, "two.md"), "utf8")).resolves.toBe("two.md"); + }); + + it.each([ + { version: 0, caseInsensitive: true }, + { version: 1, caseInsensitive: false }, + ])( + "compatibility: retains distinct case paths in mode $version with case-insensitive matching $caseInsensitive", + async ({ version, caseInsensitive }) => { + const directory = await createVault(); + const names = ["CafÊ/one.md", "cafÊ/two.md".normalize("NFD")]; + for (const name of names) { + await fs.mkdir(path.dirname(path.join(directory, name))); + await fs.writeFile(path.join(directory, name), name); + } + const adapter = new NodeFileSystemAdapter( + directory, + undefined, + () => version, + () => caseInsensitive + ); + + expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual([...names].sort()); + for (const name of names) await expect(adapter.storage.read(name)).resolves.toBe(name); + } + ); + + it("refuses reconstruction before starting remote activity when parent aliases are ambiguous", async () => { + const directory = await createVault(); + for (const name of ["CafÊ/one.md", "cafÊ/two.md".normalize("NFD")]) { + await fs.mkdir(path.dirname(path.join(directory, name))); + await fs.writeFile(path.join(directory, name), name); + } + const adapter = createNormalisingAdapter(directory, true); + const settings = { + ...DEFAULT_SETTINGS, + pathNormalizationVersion: 1 as const, + handleFilenameCaseSensitive: false, + }; + const setting = { currentSettings: () => settings, suspendAllSync: { addHandler: vi.fn() } }; + const runBoundedRemoteActivity = vi.fn(async () => undefined); + const rebuilder = new ServiceRebuilder({ + setting, + storageAccess: { getFileNames: async () => (await adapter.getFiles()).map((file) => file.path) }, + database: { + onDatabaseReset: { addHandler: vi.fn() }, + isDatabaseReady: () => true, + localDatabase: { findAllDocs: async function* () {} }, + }, + vault: { isTargetFile: async () => true }, + path: new PathServiceCompat(new ServiceContext(), { settingService: setting as never }), + replicator: { runBoundedRemoteActivity }, + UI: { showMarkdownDialog: vi.fn(async () => undefined) }, + } as unknown as ServiceRebuilderDependencies); + + await expect(rebuilder.$rebuildEverything()).rejects.toThrow(/ambiguous/i); + expect(runBoundedRemoteActivity).not.toHaveBeenCalled(); + }); + + it("uses fresh listings after external additions, renames, deletions, and collisions", async () => { + const directory = await createVault(); + const nfdName = "cafÊ.md".normalize("NFD"); + await fs.writeFile(path.join(directory, nfdName), "original"); + const adapter = createNormalisingAdapter(directory); + expect((await adapter.getFiles()).map((file) => file.path)).toEqual([nfdName]); + + const renamed = "rÊsumÊ.md".normalize("NFD"); + await fs.rename(path.join(directory, nfdName), path.join(directory, renamed)); + await fs.writeFile(path.join(directory, "new.md"), "new"); + expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(["new.md", renamed]); + + await fs.rm(path.join(directory, "new.md")); + expect((await adapter.getFiles()).map((file) => file.path)).toEqual([renamed]); + await fs.writeFile(path.join(directory, renamed.normalize("NFC")), "collision"); + await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i); + }); + + it("retains the storage boundary for exact metadata reads and directory scans", async () => { + const directory = await createVault(); + const outside = await createVault(); + await fs.writeFile(path.join(outside, "note.md"), "outside"); + await fs.symlink(outside, path.join(directory, "linked"), "dir"); + const adapter = createNormalisingAdapter(directory); + + await expect(adapter.getFiles()).resolves.toEqual([]); + await expect(adapter.storage.statExact("linked/note.md")).rejects.toThrow(/symbolic links/i); + await expect(adapter.storage.write("linked/note.md", "changed")).rejects.toThrow(/symbolic links/i); + await expect(fs.readFile(path.join(outside, "note.md"), "utf8")).resolves.toBe("outside"); + }); + + it("detects NFC and case collisions when lookup is case-insensitive", async () => { + const directory = await createVault(); + const upperName = "CafÊ.md"; + const lowerNfdName = "cafÊ.md".normalize("NFD"); + await fs.writeFile(path.join(directory, upperName), "upper", "utf8"); + await fs.writeFile(path.join(directory, lowerNfdName), "lower", "utf8"); + const adapter = createNormalisingAdapter(directory, true); + + await expect(adapter.getAbstractFileByPathInsensitive("CAFÉ.md")).rejects.toThrow(/ambiguous/i); + await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i); + await expect(adapter.scanDirectory()).rejects.toThrow(/ambiguous/i); + await expect(fs.readFile(path.join(directory, upperName), "utf8")).resolves.toBe("upper"); + await expect(fs.readFile(path.join(directory, lowerNfdName), "utf8")).resolves.toBe("lower"); + }); + + it("retains exact-path behaviour when path normalisation is disabled", async () => { + const directory = await createVault(); + const nfcName = "cafÊ.md"; + const nfdName = nfcName.normalize("NFD"); + await fs.writeFile(path.join(directory, nfdName), "NFD", "utf8"); + const adapter = new NodeFileSystemAdapter(directory); + + await expect(adapter.getAbstractFileByPath(nfcName)).resolves.toBeNull(); + await adapter.storage.write(nfcName, "NFC"); + + await expect(fs.readFile(path.join(directory, nfcName), "utf8")).resolves.toBe("NFC"); + await expect(fs.readFile(path.join(directory, nfdName), "utf8")).resolves.toBe("NFD"); + }); +}); diff --git a/src/apps/cli/adapters/NodeVaultAdapter.ts b/src/apps/cli/adapters/NodeVaultAdapter.ts index c605eeb7..00ff734d 100644 --- a/src/apps/cli/adapters/NodeVaultAdapter.ts +++ b/src/apps/cli/adapters/NodeVaultAdapter.ts @@ -2,6 +2,7 @@ import type { FilePath, UXDataWriteOptions } from "@vrtmrz/livesync-commonlib/co import type { IVaultAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters"; import type { NodeFile, NodeFolder } from "./NodeTypes"; import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node"; +import { UnicodeResolvingNodeStorageAdapter } from "./UnicodeResolvingNodeStorageAdapter"; /** * Vault adapter implementation for Node.js @@ -73,8 +74,16 @@ export class NodeVaultAdapter implements IVaultAdapter { } private async toNodeFile(path: string): Promise { - const stat = await this.storage.stat(path); + const resolvedPath = + this.storage instanceof UnicodeResolvingNodeStorageAdapter + ? await this.storage.resolveLogicalPath(path, undefined, false) + : path; + if (resolvedPath === null) throw new Error(`Could not read created file metadata: ${path}`); + const stat = + this.storage instanceof UnicodeResolvingNodeStorageAdapter + ? await this.storage.statExact(resolvedPath) + : await this.storage.stat(resolvedPath); if (stat?.type !== "file") throw new Error(`Could not read created file metadata: ${path}`); - return { path: path as FilePath, stat }; + return { path: resolvedPath as FilePath, stat }; } } diff --git a/src/apps/cli/adapters/UnicodeResolvingNodeStorageAdapter.ts b/src/apps/cli/adapters/UnicodeResolvingNodeStorageAdapter.ts new file mode 100644 index 00000000..7041e132 --- /dev/null +++ b/src/apps/cli/adapters/UnicodeResolvingNodeStorageAdapter.ts @@ -0,0 +1,205 @@ +import { NodeStorageAdapter, validateStoragePath } from "@vrtmrz/livesync-commonlib/node"; + +export class AmbiguousNormalisedPathError extends Error { + constructor(path: string) { + super(`Ambiguous path after Unicode normalisation: ${path}`); + this.name = "AmbiguousNormalisedPathError"; + } +} + +/** Resolves NFC-equivalent logical paths against current Node storage listings. */ +export class UnicodeResolvingNodeStorageAdapter extends NodeStorageAdapter { + constructor( + rootPath: string, + private readonly getPathNormalizationVersion: () => number = () => 0, + private readonly isCaseInsensitive: () => boolean = () => false + ) { + super(rootPath); + } + + get isUnicodePathNormalizationEnabled(): boolean { + return this.getPathNormalizationVersion() === 1; + } + + /** Read metadata for an entry already resolved or obtained from a storage listing. */ + async statExact(path: string) { + return await super.trystat(path); + } + + /** Inspect a physical directory once before descending into its enumerated entries. */ + async listExact(path: string) { + const entries = await super.list(path); + if (this.isUnicodePathNormalizationEnabled) { + const caseInsensitive = this.isCaseInsensitive(); + const names = new Map(); + for (const entry of [...entries.files, ...entries.folders]) { + const key = this.comparisonKey(entry, caseInsensitive); + if (names.has(key)) throw new AmbiguousNormalisedPathError(entry); + names.set(key, entry); + } + } + return entries; + } + + async resolveLogicalPath( + path: string, + caseInsensitive = this.isCaseInsensitive(), + allowMissing = true + ): Promise { + if (!this.isUnicodePathNormalizationEnabled) return path; + + validateStoragePath(path, true); + if (path === "") return ""; + + const segments = path.split("/"); + let resolvedParent = ""; + + for (let index = 0; index < segments.length; index++) { + const segment = segments[index]; + const entries = await super.list(resolvedParent); + const candidates = [...entries.files, ...entries.folders]; + const requestedKey = this.comparisonKey(segment, caseInsensitive); + const matches = candidates.filter((entryPath) => { + const entryName = entryPath.slice(resolvedParent === "" ? 0 : resolvedParent.length + 1); + return this.comparisonKey(entryName, caseInsensitive) === requestedKey; + }); + + if (matches.length > 1) throw new AmbiguousNormalisedPathError(path); + if (matches.length === 1) { + resolvedParent = matches[0]; + continue; + } + + if (!allowMissing) return null; + const remainingPath = segments.slice(index).join("/"); + return resolvedParent === "" ? remainingPath : `${resolvedParent}/${remainingPath}`; + } + + return resolvedParent; + } + + override async exists(path: string, caseInsensitive = this.isCaseInsensitive()): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false); + if (resolvedPath === null) return false; + return await super.exists(resolvedPath); + } + + override async trystat(path: string, caseInsensitive = this.isCaseInsensitive()) { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false); + if (resolvedPath === null) return null; + return await super.trystat(resolvedPath); + } + + override async stat(path: string, caseInsensitive = this.isCaseInsensitive()) { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false); + if (resolvedPath === null) return null; + return await super.trystat(resolvedPath); + } + + override async mkdir(path: string, caseInsensitive = this.isCaseInsensitive()): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true); + if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`); + await super.mkdir(resolvedPath); + } + + override async remove(path: string, caseInsensitive = this.isCaseInsensitive()): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false); + await super.remove(resolvedPath ?? path); + } + + override async read(path: string, caseInsensitive = this.isCaseInsensitive()): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false); + return await super.read(resolvedPath ?? path); + } + + override async readBinary(path: string, caseInsensitive = this.isCaseInsensitive()): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false); + return await super.readBinary(resolvedPath ?? path); + } + + override async write( + path: string, + data: string, + options?: Parameters[2], + caseInsensitive = this.isCaseInsensitive() + ): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true); + if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`); + await super.write(resolvedPath, data, options); + } + + override async writeBinary( + path: string, + data: ArrayBuffer, + options?: Parameters[2], + caseInsensitive = this.isCaseInsensitive() + ): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true); + if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`); + await super.writeBinary(resolvedPath, data, options); + } + + override async append( + path: string, + data: string, + options?: Parameters[2], + caseInsensitive = this.isCaseInsensitive() + ): Promise { + const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true); + if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`); + await super.append(resolvedPath, data, options); + } + + override async rename( + sourcePath: string, + targetPath: string, + caseInsensitive = this.isCaseInsensitive() + ): Promise { + const resolvedSource = await this.resolveLogicalPath(sourcePath, caseInsensitive, false); + if (resolvedSource === null) return await super.rename(sourcePath, targetPath); + + const resolvedTarget = await this.resolveLogicalPath(targetPath, caseInsensitive, true); + if (resolvedTarget === null) throw new Error(`Could not resolve path: ${targetPath}`); + + // Preserve explicit case-only or Unicode-spelling renames when the target resolves to the source. + const sourceKey = this.comparisonKey(resolvedSource, caseInsensitive); + const targetKey = this.comparisonKey(resolvedTarget, caseInsensitive); + const target = + sourceKey === targetKey && resolvedTarget === resolvedSource && targetPath !== resolvedSource + ? await this.resolveRenameTargetSpelling(targetPath, caseInsensitive) + : resolvedTarget; + + await super.rename(resolvedSource, target); + } + + override async list(basePath: string, caseInsensitive = this.isCaseInsensitive()) { + const resolvedPath = await this.resolveLogicalPath(basePath, caseInsensitive, false); + if (resolvedPath === null) return { files: [], folders: [] }; + return await super.list(resolvedPath); + } + + private async resolveRenameTargetSpelling(path: string, caseInsensitive: boolean): Promise { + validateStoragePath(path, false); + const segments = path.split("/"); + const leaf = segments.pop(); + if (leaf === undefined) throw new Error(`Could not resolve path: ${path}`); + const parentPath = segments.join("/"); + const resolvedParent = await this.resolveLogicalPath(parentPath, caseInsensitive, true); + if (resolvedParent === null) throw new Error(`Could not resolve path: ${path}`); + + const entries = await super.list(resolvedParent); + const requestedKey = this.comparisonKey(leaf, caseInsensitive); + const matches = [...entries.files, ...entries.folders].filter((entryPath) => { + const entryName = entryPath.slice(resolvedParent === "" ? 0 : resolvedParent.length + 1); + return this.comparisonKey(entryName, caseInsensitive) === requestedKey; + }); + if (matches.length > 1) throw new AmbiguousNormalisedPathError(path); + + return resolvedParent === "" ? leaf : `${resolvedParent}/${leaf}`; + } + + private comparisonKey(segment: string, caseInsensitive: boolean): string { + const normalisedSegment = segment.normalize("NFC"); + return caseInsensitive ? normalisedSegment.toLowerCase() : normalisedSegment; + } +} diff --git a/src/apps/cli/commands/runCommand.ts b/src/apps/cli/commands/runCommand.ts index 32887a17..2c6d1174 100644 --- a/src/apps/cli/commands/runCommand.ts +++ b/src/apps/cli/commands/runCommand.ts @@ -6,6 +6,7 @@ import { type ObsidianLiveSyncSettings, type EntryDoc, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { isRemediationModeActive } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString"; import { activateRemoteConfiguration, @@ -13,7 +14,7 @@ import { } from "@vrtmrz/livesync-commonlib/remote-configurations"; import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import type { CLICommandContext, CLIOptions } from "./types"; -import { toArrayBuffer, toDatabaseRelativePath } from "./utils"; +import { getForceRequirementError, toArrayBuffer, toDatabaseRelativePath } from "./utils"; import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { fsPromises as fs, getNodeFileCreationTime, path } from "@vrtmrz/livesync-commonlib/node"; @@ -37,8 +38,25 @@ function redactConnectionString(uri: string): string { } export async function runCommand(options: CLIOptions, context: CLICommandContext): Promise { - const { databasePath, core, replicationScheduling, settingsPath } = context; + const { core } = context; const { standardIo } = core.services.context; + const forceRequirementError = getForceRequirementError(options.command, options.force); + if (forceRequirementError) { + writeStderrLine(standardIo, `[Error] ${forceRequirementError}`); + return false; + } + if ( + options.command === "fetch" && + isRemediationModeActive(core.services.setting.currentSettings()) + ) { + writeStderrLine( + standardIo, + `[Error] The fetch command cannot run while remediation mode is active. Clear the modification-time limit before fetching.` + ); + return false; + } + + const { databasePath, replicationScheduling, settingsPath } = context; const vaultPath = context.vaultPath || databasePath; await core.services.control.activated; @@ -519,6 +537,29 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext return true; } + if (options.command === "rebuild") { + writeStderrLine(standardIo, "[Command] rebuild"); + try { + await core.serviceModules.rebuilder.$rebuildEverything(); + return true; + } catch (error) { + writeStderrLine(standardIo, "[Error] Rebuild failed:", error); + return false; + } + } + + if (options.command === "fetch") { + writeStderrLine(standardIo, "[Command] fetch"); + try { + // Fetch replaces the local database from remote; do not persist legacy local files first. + await core.serviceModules.rebuilder.$fetchLocal(false, true); + return true; + } catch (error) { + writeStderrLine(standardIo, "[Error] Fetch failed:", error); + return false; + } + } + if (options.command === "remote-add") { if (options.commandArgs.length < 2) { throw new Error("remote-add requires two arguments: "); diff --git a/src/apps/cli/commands/runCommand.unit.spec.ts b/src/apps/cli/commands/runCommand.unit.spec.ts index 21a1de6a..34de0716 100644 --- a/src/apps/cli/commands/runCommand.unit.spec.ts +++ b/src/apps/cli/commands/runCommand.unit.spec.ts @@ -118,6 +118,10 @@ function createCoreMock() { fetch: vi.fn(async () => undefined), storeContent: vi.fn(async () => true), }, + rebuilder: { + $rebuildEverything: vi.fn(async () => undefined), + $fetchLocal: vi.fn(async () => undefined), + }, }, } as any; } @@ -268,6 +272,73 @@ describe("runCommand abnormal cases", () => { vi.restoreAllMocks(); }); + it.each(["rebuild", "fetch"] as const)("requires --force before activating for %s", async (command) => { + const core = createCoreMock(); + const activated = vi.fn(() => Promise.resolve()); + Object.defineProperty(core.services.control, "activated", { get: activated }); + + await expect(runCommand(makeOptions(command, []), { ...context, core })).resolves.toBe(false); + + expect(activated).not.toHaveBeenCalled(); + expect(core.serviceModules.rebuilder.$rebuildEverything).not.toHaveBeenCalled(); + expect(core.serviceModules.rebuilder.$fetchLocal).not.toHaveBeenCalled(); + expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith( + expect.stringContaining("requires --force") + ); + }); + + it("dispatches a forced rebuild to the existing Rebuilder service", async () => { + const core = createCoreMock(); + const options = makeOptions("rebuild", []); + options.force = true; + + await expect(runCommand(options, { ...context, core })).resolves.toBe(true); + + expect(core.serviceModules.rebuilder.$rebuildEverything).toHaveBeenCalledOnce(); + expect(core.serviceModules.rebuilder.$fetchLocal).not.toHaveBeenCalled(); + }); + + it("dispatches a forced fetch without saving legacy Vault files first", async () => { + const core = createCoreMock(); + const options = makeOptions("fetch", []); + options.force = true; + + await expect(runCommand(options, { ...context, core })).resolves.toBe(true); + + expect(core.serviceModules.rebuilder.$fetchLocal).toHaveBeenCalledOnce(); + expect(core.serviceModules.rebuilder.$fetchLocal).toHaveBeenCalledWith(false, true); + expect(core.serviceModules.rebuilder.$rebuildEverything).not.toHaveBeenCalled(); + }); + + it("rejects fetch in remediation mode even when forced", async () => { + const core = createCoreMock(); + core.services.setting.currentSettings().maxMTimeForReflectEvents = 1; + const options = makeOptions("fetch", []); + options.force = true; + + await expect(runCommand(options, { ...context, core })).resolves.toBe(false); + + expect(core.serviceModules.rebuilder.$fetchLocal).not.toHaveBeenCalled(); + expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith( + expect.stringContaining("remediation mode") + ); + }); + + it("surfaces an internal rebuild refusal as a failed command", async () => { + const core = createCoreMock(); + core.serviceModules.rebuilder.$rebuildEverything.mockRejectedValue( + new Error("Rebuilding is not available while remediation mode is active.") + ); + const options = makeOptions("rebuild", []); + options.force = true; + + await expect(runCommand(options, { ...context, core })).resolves.toBe(false); + + expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith( + expect.stringContaining("Rebuilding is not available while remediation mode is active") + ); + }); + it("retains visible progress for the interactive sync command", async () => { const core = createCoreMock(); diff --git a/src/apps/cli/commands/types.ts b/src/apps/cli/commands/types.ts index a60eeaa5..c5348a44 100644 --- a/src/apps/cli/commands/types.ts +++ b/src/apps/cli/commands/types.ts @@ -22,6 +22,8 @@ export type CLICommand = | "rm" | "resolve" | "mirror" + | "rebuild" + | "fetch" | "remote-add" | "remote-rm" | "remote-ls" @@ -100,6 +102,8 @@ export const VALID_COMMANDS = new Set([ "rm", "resolve", "mirror", + "rebuild", + "fetch", "remote-add", "remote-rm", "remote-ls", diff --git a/src/apps/cli/commands/utils.ts b/src/apps/cli/commands/utils.ts index 4163d3f0..1a3410ae 100644 --- a/src/apps/cli/commands/utils.ts +++ b/src/apps/cli/commands/utils.ts @@ -1,4 +1,16 @@ import { path } from "@vrtmrz/livesync-commonlib/node"; +import type { CLICommand } from "./types"; + +export function getForceRequirementError(command: CLICommand, force = false): string | undefined { + if (force) return undefined; + if (command === "rebuild") { + return "Command 'rebuild' requires --force because it replaces the local and remote databases."; + } + if (command === "fetch") { + return "Command 'fetch' requires --force because it replaces the local database with remote state."; + } + return undefined; +} export function toArrayBuffer(data: Buffer): ArrayBuffer { return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer; diff --git a/src/apps/cli/commands/utils.unit.spec.ts b/src/apps/cli/commands/utils.unit.spec.ts index 26292f58..1014e910 100644 --- a/src/apps/cli/commands/utils.unit.spec.ts +++ b/src/apps/cli/commands/utils.unit.spec.ts @@ -1,6 +1,20 @@ import { path } from "@vrtmrz/livesync-commonlib/node"; import { describe, expect, it } from "vitest"; -import { toDatabaseRelativePath } from "./utils"; +import { getForceRequirementError, toDatabaseRelativePath } from "./utils"; + +describe("getForceRequirementError", () => { + it.each([ + ["rebuild", "Command 'rebuild' requires --force because it replaces the local and remote databases."], + ["fetch", "Command 'fetch' requires --force because it replaces the local database with remote state."], + ] as const)("requires force for %s", (command, expected) => { + expect(getForceRequirementError(command, false)).toBe(expected); + expect(getForceRequirementError(command, true)).toBeUndefined(); + }); + + it("does not require force for ordinary commands", () => { + expect(getForceRequirementError("sync", false)).toBeUndefined(); + }); +}); describe("toDatabaseRelativePath", () => { const databasePath = path.resolve("/tmp/livesync-vault"); diff --git a/src/apps/cli/main.bootstrap.unit.spec.ts b/src/apps/cli/main.bootstrap.unit.spec.ts index 9e262838..3fcc3913 100644 --- a/src/apps/cli/main.bootstrap.unit.spec.ts +++ b/src/apps/cli/main.bootstrap.unit.spec.ts @@ -3,6 +3,7 @@ import { appendFile, mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node: import { tmpdir } from "node:os"; import { join } from "node:path"; import * as chokidar from "chokidar"; +import * as offlineScanner from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import { AutoAccepting, @@ -20,6 +21,7 @@ import { cloneSettings } from "./settingsPersistence"; import type { CLICommand } from "./commands/types"; vi.mock("chokidar", { spy: true }); +vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", { spy: true }); function createStandardIoMock() { return { @@ -41,6 +43,7 @@ describe("CLI database preparation", () => { beforeEach(async () => { vi.mocked(chokidar.watch).mockClear(); + vi.mocked(offlineScanner.useOfflineScanner).mockClear(); directory = await mkdtemp(join(tmpdir(), "livesync-cli-bootstrap-")); vaultPath = join(directory, "vault"); settingsPath = join(directory, "settings.json"); @@ -69,8 +72,18 @@ describe("CLI database preparation", () => { await rm(directory, { recursive: true, force: true }); }); - async function start(command: CLICommand, runner: CliCommandRunner, exitCode = 1) { - process.argv = ["node", "livesync-cli", directory, "--vault", vaultPath, "--settings", settingsPath, command]; + async function start(command: CLICommand, runner: CliCommandRunner, exitCode = 1, force = false) { + process.argv = [ + "node", + "livesync-cli", + directory, + ...(force ? ["--force"] : []), + "--vault", + vaultPath, + "--settings", + settingsPath, + command, + ]; // Daemon probes return false so the real core unloads without keeping a daemon alive. await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`); } @@ -255,6 +268,49 @@ describe("CLI database preparation", () => { expect(watch).not.toHaveBeenCalled(); }); + it.each(["rebuild", "fetch"] as const)("does not scan or mirror legacy Vault files before %s", async (command) => { + const storedPaths: string[] = []; + const runner = vi.fn(async (_options, { core }) => { + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + return false; + }); + + await start(command, runner, 1, true); + + expect(runner).toHaveBeenCalledOnce(); + expect(storedPaths).toEqual([]); + expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content"); + expect(vi.mocked(chokidar.watch)).not.toHaveBeenCalled(); + }); + + it("registers the Rebuild scanner after readiness and makes its Vault scan available", async () => { + const enumerate = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles"); + const useScanner = vi.mocked(offlineScanner.useOfflineScanner); + + let scansBeforeCommand = -1; + let scannerRegistrationsBeforeCommand = -1; + let storedPaths: string[] = []; + const runner = vi.fn(async (_options, { core }) => { + scansBeforeCommand = enumerate.mock.calls.length; + scannerRegistrationsBeforeCommand = useScanner.mock.calls.length; + await core.services.vault.scanVault(true, true); + for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) { + storedPaths.push(doc.path); + } + return false; + }); + + await start("rebuild", runner, 1, true); + + expect(useScanner).toHaveBeenCalledOnce(); + expect(scannerRegistrationsBeforeCommand).toBe(1); + expect(scansBeforeCommand).toBe(0); + expect(enumerate).toHaveBeenCalledOnce(); + expect(storedPaths).toEqual(["notes/local.md"]); + }); + it.each([ { command: "daemon" as const, commandRuns: true }, { command: "mirror" as const, commandRuns: false }, diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 820d189a..028bd465 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -21,6 +21,7 @@ import { import { runCommand } from "./commands/runCommand"; import { isCLICommand } from "./commands/types"; import type { CLICommand, CLICommandContext, CLIOptions } from "./commands/types"; +import { getForceRequirementError } from "./commands/utils"; import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils"; import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { IgnoreRules } from "./serviceModules/IgnoreRules"; @@ -47,6 +48,7 @@ const SETTINGS_FILE = ".livesync/settings.json"; interface CLIVaultSyncMode { readonly watchFiles: boolean; readonly reflectReplicationResults: boolean; + readonly scanOnStartup?: boolean; readonly startupDatabaseOptions: StartupDatabaseOptions; } @@ -62,6 +64,18 @@ const VAULT_SYNC_MODES: Readonly>> reflectReplicationResults: false, startupDatabaseOptions: { ignoreSuspending: true, continueOnFileFailure: false }, }, + rebuild: { + watchFiles: false, + reflectReplicationResults: false, + scanOnStartup: false, + startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: false }, + }, + fetch: { + watchFiles: false, + reflectReplicationResults: true, + scanOnStartup: false, + startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: false }, + }, }; ensureGlobalNodeLocalStorage(); @@ -90,6 +104,8 @@ Commands: p2p-sync Sync with the specified peer-id or peer-name p2p-host Start P2P host mode and wait until interrupted + rebuild Rebuild local and remote databases from the selected Vault (requires --force) + fetch Replace the local database with remote state and reflect it to the Vault (requires --force) push Push local file into local database path pull Pull file from local database into local file pull-rev Pull file at specific revision into local file @@ -122,8 +138,9 @@ Commands: Show remote database status Options: - --vault , -V (daemon/mirror) Path to the vault directory containing .md files + --vault , -V (daemon/mirror/rebuild/fetch) Path to the vault directory containing .md files (defaults to database-path; allows separate PouchDB and vault dirs) + --force, -f Required for rebuild and fetch because they replace databases --interval , -i (daemon only) Poll CouchDB every N seconds instead of using the _changes feed --write-settings Write setting changes after a successful command --compat-remote-admin-exit-zero @@ -319,6 +336,12 @@ export async function main( commandRunner: CliCommandRunner = runCommand ) { const options = parseArgs(standardIo); + const forceRequirementError = getForceRequirementError(options.command, options.force); + if (forceRequirementError) { + writeStderrLine(standardIo, `[Error] ${forceRequirementError}`); + process.exitCode = 1; + return; + } const vaultSyncMode = VAULT_SYNC_MODES[options.command]; if (options.interval && options.command !== "daemon") { writeStderrLine( @@ -521,7 +544,7 @@ export async function main( () => [], // No add-ons (core, coreFeatureViews) => { replicationScheduling = coreFeatureViews.replicationScheduling; - if (vaultSyncMode) { + if (vaultSyncMode && vaultSyncMode.scanOnStartup !== false) { useOfflineScanner(core); } // Register P2P replicator feature. @@ -641,6 +664,10 @@ export async function main( if (!readyResult) { throw new Error("Failed to initialise LiveSync."); } + if (vaultSyncMode?.scanOnStartup === false) { + // Keep the startup reset clean, then enable scans for the explicit Rebuild or Fetch operation. + useOfflineScanner(core); + } const settingsBeforeCommand = cloneSettings(core.services.setting.currentSettings()); const transientSettingKeys = changedSettingKeys(settingsBeforeSuspend, settingsAfterSuspend); for (const key of CLI_RUNTIME_ONLY_SETTING_KEYS) { diff --git a/src/apps/cli/main.unit.spec.ts b/src/apps/cli/main.unit.spec.ts index 49712bd9..7d730f5d 100644 --- a/src/apps/cli/main.unit.spec.ts +++ b/src/apps/cli/main.unit.spec.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { parseArgs as parseCliArgs } from "./main"; +import { main as runCliMain, parseArgs as parseCliArgs } from "./main"; function createStandardIoMock() { return { @@ -19,6 +19,7 @@ function mockProcessExit() { describe("CLI parseArgs", () => { const originalArgv = process.argv.slice(); + const originalExitCode = process.exitCode; let standardIo: ReturnType; beforeEach(() => { @@ -29,6 +30,7 @@ describe("CLI parseArgs", () => { afterEach(() => { process.argv = originalArgv.slice(); + process.exitCode = originalExitCode; vi.restoreAllMocks(); }); @@ -99,6 +101,28 @@ describe("CLI parseArgs", () => { expect(parsed.commandArgs).toEqual([]); }); + it.each(["rebuild", "fetch"] as const)("parses %s and its force flag", (command) => { + process.argv = ["node", "livesync-cli", "./databasePath", "--force", command]; + const parsed = parseArgs(); + + expect(parsed.command).toBe(command); + expect(parsed.force).toBe(true); + expect(parsed.commandArgs).toEqual([]); + }); + + it.each(["rebuild", "fetch"] as const)("rejects %s without force before database access", async (command) => { + process.argv = ["node", "livesync-cli", "./missing-database", command]; + process.exitCode = undefined; + const runner = vi.fn(async () => true); + + await expect(runCliMain(standardIo, runner as any)).resolves.toBeUndefined(); + + expect(runner).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + expect(standardIo.writeStderr).toHaveBeenCalledWith(expect.stringContaining("requires --force")); + expect(standardIo.writeStderr).not.toHaveBeenCalledWith(expect.stringContaining("Database directory")); + }); + it("parses remote-add command", () => { process.argv = [ "node", diff --git a/src/apps/cli/serviceModules/CLIServiceModules.ts b/src/apps/cli/serviceModules/CLIServiceModules.ts index 2430cfaa..29b9df90 100644 --- a/src/apps/cli/serviceModules/CLIServiceModules.ts +++ b/src/apps/cli/serviceModules/CLIServiceModules.ts @@ -99,6 +99,7 @@ export function initialiseServiceModulesCLI( // Rebuilder (platform-independent) const rebuilder = new ServiceRebuilder({ + path: services.path, events: services.context.events, API: services.API, database: services.database, diff --git a/src/apps/cli/serviceModules/FileAccessCLI.ts b/src/apps/cli/serviceModules/FileAccessCLI.ts index 7a258c30..b7251791 100644 --- a/src/apps/cli/serviceModules/FileAccessCLI.ts +++ b/src/apps/cli/serviceModules/FileAccessCLI.ts @@ -1,6 +1,10 @@ -import { FileAccessBase, type FileAccessBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/FileAccessBase"; +import { + FileAccessBase, + type FileAccessBaseDependencies, +} from "@vrtmrz/livesync-commonlib/compat/serviceModules/FileAccessBase"; import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter"; import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; +import { getPathNormalizationVersion } from "@vrtmrz/livesync-commonlib/settings"; /** * CLI-specific implementation of FileAccessBase @@ -8,12 +12,23 @@ import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; */ export class FileAccessCLI extends FileAccessBase { constructor(basePath: string, dependencies: FileAccessBaseDependencies) { - const adapter = new NodeFileSystemAdapter(basePath, (message, detail) => { - dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE); - if (detail !== undefined) { - dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE); - } - }); + const pathNormalizationVersion = () => + getPathNormalizationVersion(dependencies.settingService.currentSettings()); + const isCaseInsensitive = () => { + const settings = dependencies.settingService.currentSettings(); + return !settings.handleFilenameCaseSensitive || dependencies.vaultService.isStorageInsensitive(); + }; + const adapter = new NodeFileSystemAdapter( + basePath, + (message, detail) => { + dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE); + if (detail !== undefined) { + dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE); + } + }, + pathNormalizationVersion, + isCaseInsensitive + ); super(adapter, dependencies); } diff --git a/src/apps/webapp/serviceModules/FSAPIServiceModules.ts b/src/apps/webapp/serviceModules/FSAPIServiceModules.ts index b9054cec..bda8059d 100644 --- a/src/apps/webapp/serviceModules/FSAPIServiceModules.ts +++ b/src/apps/webapp/serviceModules/FSAPIServiceModules.ts @@ -90,6 +90,7 @@ export function initialiseServiceModulesFSAPI( // Rebuilder (platform-independent) const rebuilder = new ServiceRebuilder({ + path: services.path, events: services.context.events, API: services.API, database: services.database, diff --git a/src/common/messagesJson/en.json b/src/common/messagesJson/en.json index 58a944b0..6304bbf1 100644 --- a/src/common/messagesJson/en.json +++ b/src/common/messagesJson/en.json @@ -139,6 +139,7 @@ "Copy Report to clipboard": "Copy Report to clipboard", "CouchDB Configuration": "CouchDB Configuration", "CouchDB Connection Tweak": "CouchDB Connection Tweak", + "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.": "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.", "CouchDB validates the database name when you connect. The name must not be empty.": "CouchDB validates the database name when you connect. The name must not be empty.", "Create or connect to database and continue": "Create or connect to database and continue", "Create P2P remote": "Create P2P remote", @@ -404,6 +405,7 @@ "lang-zh": "įŽ€äŊ“中文", "lang-zh-tw": "įšéĢ”中文", "Later": "Later", + "Legacy path identity": "Legacy path identity", "Limit: {datetime} ({timestamp})": "Limit: {datetime} ({timestamp})", "live": "live", "LiveSync could not handle multiple vaults which have same name without different prefix, This should be automatically configured.": "LiveSync could not handle multiple vaults which have same name without different prefix, This should be automatically configured.", @@ -792,6 +794,7 @@ "Paste a connection string": "Paste a connection string", "Paste the Setup URI generated from one of your active devices.": "Paste the Setup URI generated from one of your active devices.", "Path": "Path", + "Path identity mode": "Path identity mode", "Path Obfuscation": "Path Obfuscation", "Patterns to match files for overwriting instead of merging": "Patterns to match files for overwriting instead of merging", "Patterns to match files for syncing": "Patterns to match files for syncing", @@ -1449,6 +1452,7 @@ "Ui.SetupWizard.SetupRemote.ProceedCouchDb": "Continue to CouchDB setup", "Ui.SetupWizard.SetupRemote.ProceedP2P": "Continue to P2P setup", "Ui.SetupWizard.SetupRemote.Title": "Choose a synchronisation remote", + "Unicode NFC path identity": "Unicode NFC path identity", "Unique name between all synchronized devices. To edit this setting, please disable customization sync once.": "Unique name between all synchronized devices. To edit this setting, please disable customization sync once.", "Unless you are certain, selecting this options is bit dangerous. It assumes that the server configuration is compatible with this device. If this is not the case, data loss may occur. Please ensure you know what you are doing.": "Unless you are certain, selecting this options is bit dangerous. It assumes that the server configuration is compatible with this device. If this is not the case, data loss may occur. Please ensure you know what you are doing.", "Updating list...": "Updating list...", diff --git a/src/common/messagesYAML/en.yaml b/src/common/messagesYAML/en.yaml index 134517bb..40b46387 100644 --- a/src/common/messagesYAML/en.yaml +++ b/src/common/messagesYAML/en.yaml @@ -169,6 +169,8 @@ Copy: Copy Copy Report to clipboard: Copy Report to clipboard CouchDB Configuration: CouchDB Configuration CouchDB Connection Tweak: CouchDB Connection Tweak +"CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.": + "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1." CouchDB validates the database name when you connect. The name must not be empty.: CouchDB validates the database name when you connect. The name must not be empty. @@ -556,6 +558,7 @@ lang-ru: Đ ŅƒŅŅĐēиК lang-zh: įŽ€äŊ“中文 lang-zh-tw: įšéĢ”中文 Later: Later +Legacy path identity: Legacy path identity "Limit: {datetime} ({timestamp})": "Limit: {datetime} ({timestamp})" live: live LiveSync could not handle multiple vaults which have same name without different prefix, This should be automatically configured.: @@ -1251,6 +1254,7 @@ password: password Password: Password Paste a connection string: Paste a connection string Path: Path +Path identity mode: Path identity mode Path Obfuscation: Path Obfuscation Patterns to match files for overwriting instead of merging: Patterns to match files for overwriting instead of merging Patterns to match files for syncing: Patterns to match files for syncing @@ -2042,6 +2046,7 @@ TweakMismatchResolve: TweakResolving: Configuration Mismatch Detected UseRemoteConfig: Use Remote Configuration AutoAcceptCompatible: Auto-Accept Available +Unicode NFC path identity: Unicode NFC path identity Unique name between all synchronized devices. To edit this setting, please disable customization sync once.: Unique name between all synchronized devices. To edit this setting, please disable customization sync once. diff --git a/src/common/replicatorConfigurationIdentity.ts b/src/common/replicatorConfigurationIdentity.ts index 03f10306..3cd8cb69 100644 --- a/src/common/replicatorConfigurationIdentity.ts +++ b/src/common/replicatorConfigurationIdentity.ts @@ -1,5 +1,6 @@ import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { usesEncryptedInternalMetadata } from "@vrtmrz/livesync-commonlib/replication"; +import { getPathNormalizationVersion } from "@vrtmrz/livesync-commonlib/settings"; type EndpointProjection = readonly [kind: "url" | "invalid-url", value: string]; @@ -44,17 +45,18 @@ function projectHeaders(value: string): readonly (readonly [name: string, value: function projectRemoteSecurity(settings: RemoteDBSettings) { return [ + getPathNormalizationVersion(settings), settings.idDerivationVersion, settings.idDerivationKey, settings.encrypt - ? ([ - "encrypted", - settings.passphrase, - settings.useDynamicIterationCount, - settings.E2EEAlgorithm, - settings.permitEmptyPassphrase, - ] as const) - : (["plain"] as const), + ? ([ + "encrypted", + settings.passphrase, + settings.useDynamicIterationCount, + settings.E2EEAlgorithm, + settings.permitEmptyPassphrase, + ] as const) + : (["plain"] as const), ] as const; } diff --git a/src/common/replicatorConfigurationIdentity.unit.spec.ts b/src/common/replicatorConfigurationIdentity.unit.spec.ts index f5a540b4..014188e0 100644 --- a/src/common/replicatorConfigurationIdentity.unit.spec.ts +++ b/src/common/replicatorConfigurationIdentity.unit.spec.ts @@ -7,6 +7,30 @@ import { } from "./replicatorConfigurationIdentity"; describe("active Replicator configuration identity", () => { + it("replaces a connection when the path identity mode changes", () => { + const previous = configuredSettings({ pathNormalizationVersion: 0 }); + const next = { ...previous, pathNormalizationVersion: 1 as const }; + expect(getCouchDBReplicatorConfigurationIdentity(next)).not.toBe( + getCouchDBReplicatorConfigurationIdentity(previous) + ); + expect(getObjectStorageReplicatorConfigurationIdentity(next)).not.toBe( + getObjectStorageReplicatorConfigurationIdentity(previous) + ); + }); + + it("treats a missing path identity mode as version 0", () => { + const missingVersion = configuredSettings(); + Reflect.deleteProperty(missingVersion, "pathNormalizationVersion"); + const explicitVersion0 = { ...missingVersion, pathNormalizationVersion: 0 as const }; + + expect(getCouchDBReplicatorConfigurationIdentity(explicitVersion0)).toBe( + getCouchDBReplicatorConfigurationIdentity(missingVersion) + ); + expect(getObjectStorageReplicatorConfigurationIdentity(explicitVersion0)).toBe( + getObjectStorageReplicatorConfigurationIdentity(missingVersion) + ); + }); + function configuredSettings(overrides: Partial = {}): ObsidianLiveSyncSettings { return Object.assign(createNewVaultSettings(), { activeConfigurationId: "profile-a", diff --git a/src/main.ts b/src/main.ts index 7ea2f730..dd58dcc1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -112,6 +112,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { fileReflectionProvenance: createFileReflectionProvenance(services.keyValueDB), }); const rebuilder = new ServiceRebuilder({ + path: services.path, events: services.context.events, API: services.API, database: services.database, @@ -184,7 +185,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin { core, (_compatibilityReplicator, p2p) => createInteractiveP2PReplication(p2p), createOpenRebuildUI(this.app), - { prepareP2PSettings: useP2PSettingsPreparation(core.services.API.webCompatFetch.bind(core.services.API)) } + { + prepareP2PSettings: useP2PSettingsPreparation( + core.services.API.webCompatFetch.bind(core.services.API) + ), + } ); setupManager.registerP2PSetupConnectionProbe(replicator.connectionProbe); useP2PReplicatorCommands(core, replicator); diff --git a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts index 3dbbe96d..1ba23bb5 100644 --- a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts +++ b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts @@ -1,4 +1,5 @@ import { ChunkAlgorithmNames } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { $msg } from "@/common/translation"; import type { SettingSpecGroup } from "./SettingSpec.ts"; export type AdvancedSettingSpecContext = { @@ -22,6 +23,18 @@ export function createAdvancedSettingSpecGroups({ control: { type: "dropdown", options: () => ChunkAlgorithmNames }, }, { key: "customChunkSize", control: { type: "number", min: 0, allowZero: true } }, + { + key: "pathNormalizationVersion", + holdValue: true, + control: { + type: "numeric-dropdown", + options: () => ({ + "0": $msg("Legacy path identity"), + "1": $msg("Unicode NFC path identity"), + }), + }, + visible: isCouchDB, + }, ], }, { diff --git a/src/modules/features/SettingDialogue/LiveSyncSetting.ts b/src/modules/features/SettingDialogue/LiveSyncSetting.ts index bed51bd0..19ff983e 100644 --- a/src/modules/features/SettingDialogue/LiveSyncSetting.ts +++ b/src/modules/features/SettingDialogue/LiveSyncSetting.ts @@ -243,6 +243,29 @@ export class LiveSyncSetting extends Setting { }); return this; } + + autoWireNumericDropDown(key: AllNumericItemKey, opt: AutoWireOption & { options: Record }) { + const conf = this.autoWireSetting(key, opt); + this.addDropdown((dropdown) => { + this.autoWiredComponent = dropdown; + const setValue = wrapMemo((value: string) => { + dropdown.setValue(value); + }); + + dropdown.addOptions(opt.options); + this.invalidateValue = () => setValue(`${LiveSyncSetting.env.editingSettings[key] ?? ""}`); + this.invalidateValue(); + dropdown.onChange(async (value) => { + const numericValue = Number(value); + if (!Number.isInteger(numericValue) || !Object.prototype.hasOwnProperty.call(opt.options, value)) { + return false; + } + await this.commitValue(numericValue); + }); + this.autoWireComponent(this.autoWiredComponent, conf, opt); + }); + return this; + } addApplyButton(keys: AllSettingItemKey[], text?: string) { this.addButton((button) => { this.applyButtonComponent = button; diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 02caffa8..424213f8 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -133,7 +133,11 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { applySetting(keys: AllSettingItemKey[]) { for (const k of keys) { if (!this.isDirty(k)) continue; - if (this.isConfigurationEncryptionSetting(k) || k in OnDialogSettingsDefault) { + if ( + this.isReconstructionRequiredSetting(k) || + this.isConfigurationEncryptionSetting(k) || + k in OnDialogSettingsDefault + ) { // //@ts-ignore // this.initialSettings[k] = this.editingSettings[k]; continue; @@ -147,8 +151,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { }); } async applyAllSettings() { - const changedKeys = (Object.keys(this.editingSettings ?? {}) as AllSettingItemKey[]).filter((e) => - this.isDirty(e) + const changedKeys = (Object.keys(this.editingSettings ?? {}) as AllSettingItemKey[]).filter( + (e) => this.isDirty(e) && !this.isReconstructionRequiredSetting(e) ); await this.saveConfigurationEncryptionIfDirty(changedKeys); this.applySetting(changedKeys); @@ -168,7 +172,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { */ async saveSettings(keys: AllSettingItemKey[]) { let hasChanged = false; - const appliedKeys = keys.filter((k) => this.isDirty(k)); + const appliedKeys = keys.filter((k) => this.isDirty(k) && !this.isReconstructionRequiredSetting(k)); await this.saveConfigurationEncryptionIfDirty(appliedKeys); for (const k of appliedKeys) { if (this.isConfigurationEncryptionSetting(k)) continue; @@ -193,11 +197,16 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { await Promise.all(handlers); // } keys.forEach((e) => { + if (this.isReconstructionRequiredSetting(e) && this.isDirty(e)) return; if (this.isConfigurationEncryptionSetting(e) && this.isDirty(e)) return; this.refreshSetting(e); }); } + private isReconstructionRequiredSetting(key: AllSettingItemKey): boolean { + return key === "pathNormalizationVersion"; + } + private isConfigurationEncryptionSetting(key: AllSettingItemKey): boolean { return key === "configPassphraseStore" || key === "configPassphrase"; } @@ -225,8 +234,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { * @param keys setting keys for applying */ async saveAllDirtySettings() { - const changedKeys = (Object.keys(this.editingSettings ?? {}) as AllSettingItemKey[]).filter((e) => - this.isDirty(e) + const changedKeys = (Object.keys(this.editingSettings ?? {}) as AllSettingItemKey[]).filter( + (e) => this.isDirty(e) && !this.isReconstructionRequiredSetting(e) ); await this.saveSettings(changedKeys); this.reloadAllSettings(); @@ -299,6 +308,13 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { reloadAllSettings(skipUpdate: boolean = false) { const previousSettings = this._editingSettings; const previousSnapshot = this.initialSettings; + const pendingPathNormalizationVersion = + previousSettings && previousSnapshot && this.isDirty("pathNormalizationVersion") + ? { + value: previousSettings.pathNormalizationVersion, + appliedValue: previousSnapshot.pathNormalizationVersion, + } + : undefined; const pendingConfigurationEncryption = previousSettings && previousSnapshot ? { @@ -326,6 +342,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { if (pendingConfigurationEncryption?.configPassphrase) { this._editingSettings.configPassphrase = pendingConfigurationEncryption.configPassphrase.value; } + if (pendingPathNormalizationVersion) { + this._editingSettings.pathNormalizationVersion = pendingPathNormalizationVersion.value; + this.initialSettings.pathNormalizationVersion = pendingPathNormalizationVersion.appliedValue; + } if (!skipUpdate) this.requestUpdate(); } @@ -333,6 +353,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { * Reread each setting and request invalidate */ refreshSetting(key: AllSettingItemKey) { + if (this.isReconstructionRequiredSetting(key) && this.isDirty(key)) { + this.requestUpdate(); + return; + } const localSetting = this.reloadAllLocalSettings(); if (key in this.core.settings) { if (key in localSetting) { @@ -627,6 +651,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { "passphrase", "useDynamicIterationCount", "usePathObfuscation", + "pathNormalizationVersion", "encrypt", // "remoteType", ]); @@ -637,6 +662,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { "passphrase", "useDynamicIterationCount", "usePathObfuscation", + "pathNormalizationVersion", "encrypt", ]); } @@ -682,16 +708,23 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { override getControlValue(key: string): unknown { const spec = this.getDeclarativeSettingSpec(key); - return this.editingSettings[spec.key]; + const value = this.editingSettings[spec.key]; + return spec.control.type === "numeric-dropdown" ? `${value}` : value; } override async setControlValue(key: string, value: unknown): Promise { const spec = this.getDeclarativeSettingSpec(key); - if (!isValidSettingSpecValue(spec, value)) { + const persistedValue = + spec.control.type === "numeric-dropdown" && typeof value === "string" ? Number(value) : value; + if (!isValidSettingSpecValue(spec, persistedValue)) { throw new TypeError(`Invalid value for declarative setting ${key}`); } - Reflect.set(this.editingSettings, spec.key, value); - await this.saveSettings([spec.key]); + Reflect.set(this.editingSettings, spec.key, persistedValue); + if (spec.holdValue) { + this.requestUpdate(); + } else { + await this.saveSettings([spec.key]); + } } private createRebuildRequiredAction(): SettingDefinitionAction { @@ -1059,6 +1092,15 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { }; rebuildDB = async (method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks") => { + if (method === "remoteOnly" && this.isDirty("pathNormalizationVersion")) { + Logger( + $msg( + "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1." + ), + LOG_LEVEL_NOTICE + ); + return; + } if (this.editingSettings.encrypt && this.editingSettings.passphrase == "") { Logger($msg("obsidianLiveSyncSettingTab.logEncryptionNoPassphrase"), LOG_LEVEL_NOTICE); return; @@ -1109,6 +1151,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { this.editingSettings.passphrase = ""; } await this.saveAllDirtySettings(); + await this.applyReconstructionRequiredSettings(); }, }); if (result.result === "scheduled") { @@ -1118,6 +1161,9 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { if (result.result === "failed") { return; } + if (this.isDirty("pathNormalizationVersion")) { + return; + } const applyWithoutInitialisation = $msg( "Ui.SetupWizard.ApplySettingsInitialisation.ApplyWithoutInitialisation" @@ -1136,6 +1182,23 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { } } + private async applyReconstructionRequiredSettings(): Promise { + const key = "pathNormalizationVersion"; + if (!this.isDirty(key)) return; + + const previousValue = this.core.settings.pathNormalizationVersion; + this.copySettingValue(this.core.settings, this.editingSettings, key); + try { + await this.services.setting.saveSettingData(); + } catch (error) { + this.core.settings.pathNormalizationVersion = previousValue; + throw error; + } + + this.copySettingValue(this.initialSettings, this.core.settings, key); + this.refreshSetting(key); + } + // The imperative renderer remains required by the declared Obsidian 1.7.2 minimum version. override display(): void { this.displayImperative(); diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts index 04332958..1349edfc 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts @@ -316,6 +316,117 @@ describe("ObsidianLiveSyncSettingTab pending-setting initialisation", () => { }; } + it("requires local and remote rebuilds when path identity changes", () => { + const { tab } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + initialSettings.handleFilenameCaseSensitive = false; + tab.editingSettings.handleFilenameCaseSensitive = false; + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + + expect(tab.isNeedRebuildLocal()).toBe(true); + expect(tab.isNeedRebuildRemote()).toBe(true); + }); + + it("does not offer settings-only fallback when path identity changes", async () => { + const { tab, saveSettingData, confirmWithMessage } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockResolvedValueOnce({ + result: "cancelled", + }); + + await tab.confirmRebuild(); + + expect(confirmWithMessage).not.toHaveBeenCalled(); + expect(saveSettingData).not.toHaveBeenCalled(); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(Reflect.get(initialSettings, "pathNormalizationVersion")).toBe(0); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + }); + + it.each([ + ["regular-expression Apply", "syncOnlyRegEx", "notes/.*"], + ["preset Apply", "preset", "LIVESYNC"], + ] as const)("keeps path identity pending during %s", async (_label, setting, value) => { + const { tab } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + Reflect.set(tab.editingSettings, setting, value); + if (setting === "preset") { + tab.addOnSaved("preset", async () => { + tab.editingSettings.liveSync = true; + await tab.saveAllDirtySettings(); + }); + } + + await tab.saveAllDirtySettings(); + + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + if (setting === "syncOnlyRegEx") { + expect(tab.core.settings.syncOnlyRegEx).toBe(value); + } else { + expect(tab.core.settings.liveSync).toBe(true); + } + }); + + it("excludes path identity from direct save operations", async () => { + const { tab, saveSettingData } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + + await tab.saveSettings(["pathNormalizationVersion"]); + + expect(saveSettingData).not.toHaveBeenCalled(); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + }); + + it("excludes path identity from ordinary Apply operations", async () => { + const { tab, saveSettingData } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + + await tab.applyAllSettings(); + + expect(saveSettingData).not.toHaveBeenCalled(); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + }); + + it("retains a pending path identity edit and its applied baseline after reload", () => { + const { tab } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + + tab.reloadAllSettings(); + + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + expect(tab.isNeedRebuildLocal()).toBe(true); + expect(tab.isNeedRebuildRemote()).toBe(true); + }); + it("keeps pending settings in the editing buffer when initialisation and the fallback are cancelled", async () => { const { tab, saveSettingData, confirmWithMessage, closeSetting } = createSettingsTab(); settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockResolvedValueOnce({ @@ -353,8 +464,11 @@ describe("ObsidianLiveSyncSettingTab pending-setting initialisation", () => { expect(closeSetting).not.toHaveBeenCalled(); }); - it("closes settings only after initialisation has been scheduled", async () => { + it("applies path identity only from a successful initialisation reservation callback", async () => { const { tab, saveSettingData, confirmWithMessage, closeSetting } = createSettingsTab(); + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(tab.initialSettings!, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockImplementationOnce( async ({ applySettings }: { applySettings: () => Promise }) => { await applySettings(); @@ -364,9 +478,118 @@ describe("ObsidianLiveSyncSettingTab pending-setting initialisation", () => { await tab.confirmRebuild(); - expect(saveSettingData).toHaveBeenCalledOnce(); + expect(saveSettingData).toHaveBeenCalledTimes(2); expect(confirmWithMessage).not.toHaveBeenCalled(); expect(closeSetting).toHaveBeenCalledOnce(); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(1); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(1); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(false); + }); + + it("rolls path identity back and keeps it pending when saving after reservation fails", async () => { + const { tab, saveSettingData, closeSetting } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + tab.editingSettings.handleFilenameCaseSensitive = false; + const failure = new Error("settings could not be saved"); + saveSettingData.mockRejectedValueOnce(failure); + settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockImplementationOnce( + async ({ applySettings }: { applySettings: () => Promise }) => { + await applySettings(); + return { result: "scheduled", mode: "rebuild" }; + } + ); + + await expect(tab.confirmRebuild()).rejects.toBe(failure); + + expect(saveSettingData).toHaveBeenCalledOnce(); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + expect(closeSetting).not.toHaveBeenCalled(); + }); + + it("retains pending path identity when fetch validation fails", async () => { + const { tab } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + vi.spyOn(tab, "checkWorkingPassphrase").mockResolvedValue(false); + settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockImplementationOnce( + async ({ validateChoice }: { validateChoice: (mode: "fetch" | "rebuild") => Promise }) => { + expect(await validateChoice("fetch")).toBe(false); + return { result: "failed", mode: "fetch" }; + } + ); + + await tab.confirmRebuild(); + + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + expect(tab.closeSetting).not.toHaveBeenCalled(); + }); + + it("retains pending path identity when initialisation reservation fails", async () => { + const { tab, saveSettingData } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockResolvedValueOnce({ + result: "failed", + mode: "rebuild", + }); + + await tab.confirmRebuild(); + + expect(saveSettingData).not.toHaveBeenCalled(); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + expect(tab.closeSetting).not.toHaveBeenCalled(); + }); + + it("refuses remote-only overwrite with pending path identity before side effects", async () => { + const { tab, saveSettingData, suspendAllSync, suspendExtraSync, performRebuildDB, closeSetting } = + createSettingsTab(); + const initialSettings = tab.initialSettings!; + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 1); + tab.editingSettings.encrypt = false; + const applyAllSettings = vi.spyOn(tab, "applyAllSettings"); + const reloadAllSettings = vi.spyOn(tab, "reloadAllSettings"); + + await tab.rebuildDB("remoteOnly"); + + expect(applyAllSettings).not.toHaveBeenCalled(); + expect(suspendAllSync).not.toHaveBeenCalled(); + expect(suspendExtraSync).not.toHaveBeenCalled(); + expect(reloadAllSettings).not.toHaveBeenCalled(); + expect(saveSettingData).not.toHaveBeenCalled(); + expect(performRebuildDB).not.toHaveBeenCalled(); + expect(closeSetting).not.toHaveBeenCalled(); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.initialSettings!, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.isDirty("pathNormalizationVersion")).toBe(true); + expect(vi.mocked($msg)).toHaveBeenCalledWith( + "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1." + ); + expect(loggerMocks.Logger).toHaveBeenCalledWith( + expect.stringContaining( + "Rebuild both databases from a complete authoritative Vault and Fetch on the other devices." + ), + LOG_LEVEL_NOTICE + ); }); it("does not offer the settings-only fallback after an initialisation failure", async () => { @@ -489,6 +712,33 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { expect(saveSettings).toHaveBeenCalledWith(["hashCacheMaxCount"]); }); + it("keeps a path identity dropdown edit pending until the rebuild flow applies it", async () => { + const { tab, saveSettingData } = createSettingsTab(); + const initialSettings = tab.initialSettings!; + const saveSettings = vi.spyOn(tab, "saveSettings"); + vi.spyOn(tab, "requestUpdate").mockImplementation(() => undefined); + Reflect.set(tab.core.settings, "pathNormalizationVersion", 0); + Reflect.set(tab.editingSettings, "pathNormalizationVersion", 0); + Reflect.set(initialSettings, "pathNormalizationVersion", 0); + + expect(tab.getControlValue("pathNormalizationVersion")).toBe("0"); + + await tab.setControlValue("pathNormalizationVersion", "1"); + + expect(Reflect.get(tab.editingSettings, "pathNormalizationVersion")).toBe(1); + expect(tab.getControlValue("pathNormalizationVersion")).toBe("1"); + expect(Reflect.get(tab.core.settings, "pathNormalizationVersion")).toBe(0); + expect(Reflect.get(initialSettings, "pathNormalizationVersion")).toBe(0); + expect(saveSettings).not.toHaveBeenCalled(); + expect(tab.requestUpdate).toHaveBeenCalledOnce(); + expect(saveSettingData).not.toHaveBeenCalled(); + expect(tab.isNeedRebuildLocal()).toBe(true); + expect(tab.isNeedRebuildRemote()).toBe(true); + await expect(tab.setControlValue("pathNormalizationVersion", "0.5")).rejects.toThrow( + /Invalid value for declarative setting/u + ); + }); + it("rejects unregistered declarative control keys", async () => { const { tab } = createSettingsTab(); diff --git a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts index e33b3429..859727cd 100644 --- a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts +++ b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts @@ -43,7 +43,7 @@ describe("paneAdvanced", () => { }; paneAdvanced.call(host as never, {} as HTMLElement, { addPanel } as never); - await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(9)); + await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(10)); expect(addPanel.mock.calls.map(([, heading]) => heading)).toEqual([ "Memory cache", @@ -55,6 +55,7 @@ describe("paneAdvanced", () => { panelElements.get("Memory cache"), panelElements.get("Local Database Tweak"), panelElements.get("Local Database Tweak"), + panelElements.get("Local Database Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), @@ -66,6 +67,7 @@ describe("paneAdvanced", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", diff --git a/src/modules/features/SettingDialogue/SettingSpec.ts b/src/modules/features/SettingDialogue/SettingSpec.ts index a14c6adb..502fcc9e 100644 --- a/src/modules/features/SettingDialogue/SettingSpec.ts +++ b/src/modules/features/SettingDialogue/SettingSpec.ts @@ -11,6 +11,7 @@ export type PersistedSettingKey = PersistedBooleanSettingKey | PersistedStringSe type SettingSpecBase = { key: K; control: C; + holdValue?: boolean; visible?: () => boolean; disabled?: () => boolean; aliases?: string[]; @@ -34,6 +35,14 @@ type NumberSettingSpec = SettingSpecBase< } >; +type NumericDropdownSettingSpec = SettingSpecBase< + PersistedNumericSettingKey, + { + type: "numeric-dropdown"; + options: () => Record; + } +>; + type DropdownSettingSpec = SettingSpecBase< PersistedStringSettingKey, { @@ -42,7 +51,7 @@ type DropdownSettingSpec = SettingSpecBase< } >; -export type SettingSpec = ToggleSettingSpec | NumberSettingSpec | DropdownSettingSpec; +export type SettingSpec = ToggleSettingSpec | NumberSettingSpec | NumericDropdownSettingSpec | DropdownSettingSpec; /** An existing Advanced-page panel and the standard controls rendered within it. */ export type SettingSpecGroup = { @@ -54,7 +63,7 @@ export type SettingSpecMessages = { valueShouldBeInRange: (range: { min?: number; max?: number }) => string; }; -type LegacySettingOptions = Pick; +type LegacySettingOptions = Pick; export type LegacySettingBinding = | { @@ -71,6 +80,11 @@ export type LegacySettingBinding = acceptZero?: boolean; }; } + | { + type: "numeric-dropdown"; + key: PersistedNumericSettingKey; + options: LegacySettingOptions & { options: Record }; + } | { type: "dropdown"; key: PersistedStringSettingKey; @@ -83,6 +97,10 @@ export interface LegacySettingSpecRenderer { key: AllNumericItemKey, options: AutoWireOption & { clampMin?: number; clampMax?: number; acceptZero?: boolean } ): unknown; + autoWireNumericDropDown( + key: AllNumericItemKey, + options: AutoWireOption & { options: Record } + ): unknown; autoWireDropDown(key: AllStringItemKey, options: AutoWireOption & { options: Record }): unknown; } @@ -94,15 +112,21 @@ function isNumberSettingSpec(spec: SettingSpec): spec is NumberSettingSpec { return spec.control.type === "number"; } +function isNumericDropdownSettingSpec(spec: SettingSpec): spec is NumericDropdownSettingSpec { + return spec.control.type === "numeric-dropdown"; +} + function toLegacyUpdateOptions(spec: SettingSpec): LegacySettingOptions { - if (!spec.visible && !spec.disabled) { - return {}; - } return { - onUpdate: (): OnUpdateResult => ({ - ...(spec.visible ? { visibility: spec.visible() } : {}), - ...(spec.disabled ? { disabled: spec.disabled() } : {}), - }), + ...(spec.holdValue ? { holdValue: spec.holdValue } : {}), + ...(!spec.visible && !spec.disabled + ? {} + : { + onUpdate: (): OnUpdateResult => ({ + ...(spec.visible ? { visibility: spec.visible() } : {}), + ...(spec.disabled ? { disabled: spec.disabled() } : {}), + }), + }), }; } @@ -131,6 +155,16 @@ export function toLegacySettingBinding(spec: SettingSpec): LegacySettingBinding }, }; } + if (isNumericDropdownSettingSpec(spec)) { + return { + type: "numeric-dropdown", + key: spec.key, + options: { + ...updateOptions, + options: spec.control.options(), + }, + }; + } return { type: "dropdown", key: spec.key, @@ -151,6 +185,9 @@ export function renderLegacySettingSpec(renderer: LegacySettingSpecRenderer, spe case "number": renderer.autoWireNumeric(binding.key, binding.options); return; + case "numeric-dropdown": + renderer.autoWireNumericDropDown(binding.key, binding.options); + return; case "dropdown": renderer.autoWireDropDown(binding.key, binding.options); } @@ -176,6 +213,12 @@ export function isValidSettingSpecValue(spec: SettingSpec, value: unknown): bool return typeof value === "boolean"; case "number": return typeof value === "number" && !numberIsOutOfRange(value, spec.control); + case "numeric-dropdown": + return ( + typeof value === "number" && + Number.isInteger(value) && + Object.prototype.hasOwnProperty.call(spec.control.options(), `${value}`) + ); case "dropdown": return typeof value === "string" && Object.prototype.hasOwnProperty.call(spec.control.options(), value); } @@ -229,6 +272,14 @@ export function toObsidianSettingDefinition( }; break; } + case "numeric-dropdown": + control = { + type: "dropdown", + key: spec.key, + options: spec.control.options(), + ...(spec.disabled ? { disabled: spec.disabled } : {}), + }; + break; case "dropdown": control = { type: "dropdown", diff --git a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts index 5c1c029f..62958bc4 100644 --- a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, expectTypeOf, it, vi } from "vitest"; import { statusDisplay, type ConfigurationItem } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { getConfig, type OnDialogSettings } from "./settingConstants.ts"; import { + isValidSettingSpecValue, renderLegacySettingSpec, toLegacySettingBinding, toObsidianSettingDefinition, @@ -16,7 +17,7 @@ const rangeMessages = { }; describe("Advanced setting specifications", () => { - it("lists only the nine controls currently exposed by the Advanced page", () => { + it("lists only the ten controls currently exposed by the Advanced page", () => { const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => true }); expect(groups.map(({ heading }) => heading)).toEqual([ @@ -29,6 +30,7 @@ describe("Advanced setting specifications", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", @@ -38,12 +40,13 @@ describe("Advanced setting specifications", () => { ]); }); - it("keeps CouchDB visibility on the four remote chunk controls", () => { + it("keeps path identity and the four remote chunk controls CouchDB-only", () => { let couchDB = false; const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => couchDB }); const conditional = groups.flatMap(({ items }) => items).filter(({ visible }) => visible !== undefined); expect(conditional.map(({ key }) => key)).toEqual([ + "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", @@ -55,6 +58,44 @@ describe("Advanced setting specifications", () => { expect(conditional.every(({ visible }) => visible?.() === true)).toBe(true); }); + it("exposes CouchDB path identity as a two-choice numeric dropdown", () => { + let couchDB = false; + const spec = createAdvancedSettingSpecGroups({ isCouchDB: () => couchDB }) + .flatMap(({ items }) => items) + .find(({ key }) => key === "pathNormalizationVersion"); + if (!spec) throw new Error("Path identity mode is not exposed on the Advanced page"); + if (spec.control.type !== "numeric-dropdown") throw new Error("Expected a numeric dropdown"); + + expect(spec.control).toMatchObject({ type: "numeric-dropdown", options: expect.any(Function) }); + expect(spec.visible?.()).toBe(false); + couchDB = true; + expect(spec.visible?.()).toBe(true); + expect(spec.control.options()).toEqual({ + "0": "Legacy path identity", + "1": "Unicode NFC path identity", + }); + expect(isValidSettingSpecValue(spec, 0)).toBe(true); + expect(isValidSettingSpecValue(spec, 1)).toBe(true); + expect(isValidSettingSpecValue(spec, 0.5)).toBe(false); + expect(isValidSettingSpecValue(spec, 2)).toBe(false); + + const definition = toObsidianSettingDefinition(spec, { name: "Path identity mode" }, rangeMessages); + expect(definition.control).toEqual({ + type: "dropdown", + key: "pathNormalizationVersion", + options: { "0": "Legacy path identity", "1": "Unicode NFC path identity" }, + }); + expect(toLegacySettingBinding(spec)).toEqual({ + type: "numeric-dropdown", + key: "pathNormalizationVersion", + options: { + holdValue: true, + onUpdate: expect.any(Function), + options: { "0": "Legacy path identity", "1": "Unicode NFC path identity" }, + }, + }); + }); + it("excludes transient settings-dialogue keys from the persisted key type", () => { type OnDialogOverlap = Extract; expectTypeOf().toEqualTypeOf(); @@ -225,6 +266,7 @@ describe("SettingSpec conversion", () => { const renderer = { autoWireToggle: vi.fn(), autoWireNumeric: vi.fn(), + autoWireNumericDropDown: vi.fn(), autoWireDropDown: vi.fn(), }; const visible = vi.fn(() => false); @@ -241,6 +283,7 @@ describe("SettingSpec conversion", () => { expect(options.onUpdate()).toEqual({ visibility: false }); expect(visible).toHaveBeenCalledOnce(); expect(renderer.autoWireNumeric).not.toHaveBeenCalled(); + expect(renderer.autoWireNumericDropDown).not.toHaveBeenCalled(); expect(renderer.autoWireDropDown).not.toHaveBeenCalled(); }); }); diff --git a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts index 9de8ff93..32bf0e82 100644 --- a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts @@ -86,6 +86,7 @@ describe("settings page catalogue", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", "concurrencyOfReadChunksOnline", diff --git a/src/serviceFeatures/replication/ReplicateResultProcessor.ts b/src/serviceFeatures/replication/ReplicateResultProcessor.ts index 9474e58e..523043d9 100644 --- a/src/serviceFeatures/replication/ReplicateResultProcessor.ts +++ b/src/serviceFeatures/replication/ReplicateResultProcessor.ts @@ -1,4 +1,9 @@ -import { assessRemoteFeatureDocument, describeRemoteFeatureRejection } from "@vrtmrz/livesync-commonlib/replication"; +import { + assessRemoteFeatureDocument, + describeRemoteFeatureRejection, + NFC_PATH_IDENTITY_FEATURE, +} from "@vrtmrz/livesync-commonlib/replication"; +import { getPathNormalizationVersion } from "@vrtmrz/livesync-commonlib/settings"; import { SYNCINFO_ID, VERSIONING_DOCID, @@ -33,7 +38,7 @@ const REPROCESS_BATCH_SIZE = 100; type ReplicateResultProcessorSettings = Pick< ObsidianLiveSyncSettings, "maxMTimeForReflectEvents" | "suspendParseReplicationResult" ->; +> & { pathNormalizationVersion?: 0 | 1 }; type ReplicateResultProcessorServices = Pick< LiveSyncBaseCore["services"], "appLifecycle" | "database" | "path" | "replication" | "vault" @@ -285,13 +290,20 @@ export class ReplicateResultProcessor { if (change._id === VERSIONING_DOCID || change.type === "versioninfo") { this.log(`Version info document received: ${change._id}`, LOG_LEVEL_VERBOSE); const assessment = assessRemoteFeatureDocument(change); - if (assessment.status !== "supported" && assessment.status !== "older-generation") { + const pathIdentityDoesNotMatch = + assessment.status === "supported" && + assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE) !== + (getPathNormalizationVersion(this.context.currentSettings()) === 1); + if ( + (assessment.status !== "supported" && assessment.status !== "older-generation") || + pathIdentityDoesNotMatch + ) { // Fence and retire the active publication through its owner. this.context.requestActiveReplicatorRetirement(); - this.log( - `${describeRemoteFeatureRejection(assessment)} Update Self-hosted LiveSync before synchronising.`, - LOG_LEVEL_NOTICE - ); + const rejection = pathIdentityDoesNotMatch + ? "Local and remote path identity settings do not match. Select the matching mode and Fetch the migrated database before synchronising." + : `${describeRemoteFeatureRejection(assessment)} Update Self-hosted LiveSync before synchronising.`; + this.log(rejection, LOG_LEVEL_NOTICE); } return true; } diff --git a/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts b/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts index 4e992692..5ed7bf31 100644 --- a/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts +++ b/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts @@ -6,6 +6,7 @@ import { type EntryDoc, type FilePathWithPrefix, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import * as replicationCompatibility from "@vrtmrz/livesync-commonlib/replication"; import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication"; import { isValidFilenameInAndroid, @@ -46,6 +47,7 @@ type SetupOptions = { localVersionInfo?: unknown; isTargetFile?: (path: string) => Promise; databaseId?: string; + pathNormalizationVersion?: number; }; function setup(options: SetupOptions = {}) { @@ -96,6 +98,7 @@ function setup(options: SetupOptions = {}) { currentSettings: () => ({ maxMTimeForReflectEvents: options.maxMTimeForReflectEvents ?? 0, suspendParseReplicationResult: false, + pathNormalizationVersion: options.pathNormalizationVersion ?? 0, }), getKeyValueDB: () => core.kvDB, getLocalDatabase: () => core.localDatabase, @@ -322,6 +325,66 @@ describe("ReplicateResultProcessor", () => { expect(onCloseActiveReplication).not.toHaveBeenCalled(); }); + it.each([ + ["local version 0 and remote version 1", 0, ["nfc-path-identity-v1"], true], + ["local version 1 and remote version 0", 1, [], true], + ["both use version 1", 1, ["nfc-path-identity-v1"], false], + ["both use version 0", 0, [], false], + ] as const)( + "checks path identity compatibility for %s", + (_description, localVersion, remoteFeatures, shouldRetire) => { + const originalAssessment = replicationCompatibility.assessRemoteFeatureDocument; + const assessmentSpy = vi + .spyOn(replicationCompatibility, "assessRemoteFeatureDocument") + .mockImplementation((document) => { + const assessment = originalAssessment(document); + if ( + assessment.status !== "unknown-features" || + assessment.identifiers.length !== 1 || + assessment.identifiers[0] !== "nfc-path-identity-v1" + ) { + return assessment; + } + return { status: "supported", usedFeatures: ["nfc-path-identity-v1"] }; + }); + + try { + const { onCloseActiveReplication, processor } = setup({ pathNormalizationVersion: localVersion }); + processor.enqueueAll([ + { + _id: VERSIONING_DOCID, + _rev: "2-path-identity", + type: "versioninfo", + version: REMOTE_FEATURE_GENERATION, + used_features: [...remoteFeatures], + } as PouchDB.Core.ExistingDocument, + ]); + + if (shouldRetire) { + expect(onCloseActiveReplication).toHaveBeenCalledOnce(); + } else { + expect(onCloseActiveReplication).not.toHaveBeenCalled(); + } + } finally { + assessmentSpy.mockRestore(); + } + } + ); + + it("keeps the existing older-generation gate for local path identity version 1", () => { + const { onCloseActiveReplication, processor } = setup({ pathNormalizationVersion: 1 }); + processor.enqueueAll([ + { + _id: VERSIONING_DOCID, + _rev: "1-legacy-path-identity", + type: "versioninfo", + version: 11, + } as PouchDB.Core.ExistingDocument, + ]); + + expect(onCloseActiveReplication).not.toHaveBeenCalled(); + }); + it("reports unknown feature identifiers and requests Replicator retirement", () => { const logger = vi.fn(); setGlobalLogFunction(logger); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 4030b4f8..2ee71938 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -170,6 +170,10 @@ Set `E2E_OBSIDIAN_ONLY_DIFFERENT_CHUNK_ID_KEYS=true` to run the focused case whe `test:e2e:obsidian:cli-to-obsidian-sync` is the cross-runtime compatibility check for the official LiveSync CLI and the real Obsidian plug-in. Build the plug-in first, and build the local CLI too when no external CLI command is selected. The script uses E2EE, Path Obfuscation, and the current preferred chunk settings to create and synchronise a note through the CLI, starts real Obsidian with an isolated Vault and profile, synchronises the same CouchDB database, and verifies that the plug-in materialises identical note content. This covers the boundary that CLI-only and plug-in-only round trips do not exercise. +Set `E2E_OBSIDIAN_UNICODE_PATHS=true` to exercise path identity version 1 on Linux. This variant uses an NFD directory and a mixed NFC/NFD file name, checks reflection at Obsidian's NFC path, and returns an update and deletion to the original CLI physical path. Updates use `sync` and `mirror`; deletions use the polling daemon, because `mirror` retains tombstoned local files by default. It uses unencrypted Metadata by default to match issue #1259; add `E2E_OBSIDIAN_UNICODE_ENCRYPTION=true` for the encrypted variant. These checks do not establish native macOS or iOS filesystem behaviour. + +Set `E2E_OBSIDIAN_UNICODE_MIGRATION=true` to begin with two CLI databases using legacy path identity. The source runs `rebuild --force --write-settings` after selecting version 1, and the second client runs `fetch --force --write-settings`. The scenario checks that the old document ID is replaced, the NFC feature is declared, and both physical NFD filenames are preserved before continuing the real Obsidian update and deletion checks. + The isolated Obsidian session starts with its CouchDB settings and device-local compatibility acknowledgement already in place. This keeps the scenario focused on cross-runtime data compatibility; unconfigured start-up and visible CouchDB onboarding are covered by their dedicated workflows. By default, the compatibility check runs `node src/apps/cli/dist/index.cjs`. Set `LIVESYNC_CLI_COMMAND` to test another CLI build or distribution. The value may be a quoted command line or a JSON array of executable and prefix arguments; the scenario arguments are appended without going through a shell. diff --git a/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts b/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts index 6daebc30..96267663 100644 --- a/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts +++ b/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts @@ -28,7 +28,11 @@ process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "30000"; process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ??= "30000"; const liveSyncCli = resolve("src/apps/cli/dist/index.cjs"); -const notePath = "E2E/cli-to-obsidian.md"; +const migrateUnicodePaths = process.env.E2E_OBSIDIAN_UNICODE_MIGRATION === "true"; +const unicodePaths = migrateUnicodePaths || process.env.E2E_OBSIDIAN_UNICODE_PATHS === "true"; +const useEncryption = !unicodePaths || process.env.E2E_OBSIDIAN_UNICODE_ENCRYPTION === "true"; +const notePath = unicodePaths ? "Cafe\u0301/r\u00e9sume\u0301.md" : "E2E/cli-to-obsidian.md"; +const receivedNotePath = unicodePaths ? notePath.normalize("NFC") : notePath; const noteContent = [ "# CLI to real Obsidian", "", @@ -183,7 +187,8 @@ async function configureLiveSyncCli( command: LiveSyncCliCommand, settingsPath: string, couchDb: Awaited>, - dbName: string + dbName: string, + pathNormalizationVersion = unicodePaths ? 1 : 0 ): Promise { await runLiveSyncCli(command, ["init-settings", "--force", settingsPath]); const settings = JSON.parse(await readFile(settingsPath, "utf-8")) as Record; @@ -210,16 +215,61 @@ async function configureLiveSyncCli( handleFilenameCaseSensitive: false, doNotUseFixedRevisionForChunks: true, E2EEAlgorithm: "v2", - encrypt: true, + encrypt: useEncryption, passphrase: e2eePassphrase, - usePathObfuscation: true, - encryptInternalMetadata: true, + usePathObfuscation: useEncryption, + encryptInternalMetadata: useEncryption, + pathNormalizationVersion, doctorProcessedVersion: "0.25.27", isConfigured: true, }); await writeFile(settingsPath, `${JSON.stringify(settings, null, 2)}\n`, "utf-8"); } +async function startLiveSyncCliDaemon(command: LiveSyncCliCommand, databasePath: string, settingsPath: string) { + const child = spawn( + command.executable, + [...command.prefixArgs, databasePath, "--settings", settingsPath, "--interval", "1", "daemon"], + { cwd: process.cwd(), env: process.env, stdio: ["ignore", "pipe", "pipe"] } + ); + let output = ""; + let spawnError: Error | undefined; + child.stdout.on("data", (chunk: Buffer) => { + output += chunk.toString("utf-8"); + }); + child.stderr.on("data", (chunk: Buffer) => { + output += chunk.toString("utf-8"); + }); + child.on("error", (error) => { + spawnError = error; + }); + const stop = async () => { + if (child.exitCode !== null || child.signalCode !== null || spawnError) return; + await new Promise((resolvePromise) => { + const timeout = setTimeout(() => child.kill("SIGKILL"), 10000); + child.once("exit", () => { + clearTimeout(timeout); + resolvePromise(); + }); + child.kill("SIGTERM"); + }); + }; + const deadline = Date.now() + Number(process.env.E2E_LIVESYNC_CLI_TIMEOUT_MS ?? 60000); + try { + while (!output.includes("[Daemon] Polling mode:")) { + if (spawnError) throw spawnError; + if (child.exitCode !== null || child.signalCode !== null || Date.now() >= deadline) { + throw new Error(`The CLI daemon did not become ready.\n${output}`); + } + await new Promise((resolvePromise) => setTimeout(resolvePromise, 100)); + } + return { stop, getOutput: () => output }; + } catch (error) { + await stop(); + throw error; + } +} + async function writeCliNote( command: LiveSyncCliCommand, databasePath: string, @@ -280,6 +330,86 @@ async function waitForVaultContent( throw new Error(`Timed out waiting for CLI-created note at ${fullPath}. Last content:\n${lastContent}`); } +async function migrateCliDatabases( + command: LiveSyncCliCommand, + cliState: string, + databasePath: string, + settingsPath: string, + couchDb: Awaited>, + dbName: string, + legacyFileInfo: CliFileInfo +): Promise { + const secondaryDatabasePath = join(cliState, "secondary-database"); + const secondarySettingsPath = join(cliState, "secondary-settings.json"); + await mkdir(secondaryDatabasePath, { recursive: true }); + await configureLiveSyncCli(command, secondarySettingsPath, couchDb, dbName, 0); + await runLiveSyncCli(command, [secondaryDatabasePath, "--settings", secondarySettingsPath, "sync"]); + await runLiveSyncCli(command, [secondaryDatabasePath, "--settings", secondarySettingsPath, "mirror"]); + assertEqual( + await readFile(join(secondaryDatabasePath, notePath), "utf-8"), + editedNoteContent, + "The secondary CLI did not materialise the legacy NFD note before migration." + ); + + for (const path of [settingsPath, secondarySettingsPath]) { + const settings = JSON.parse(await readFile(path, "utf-8")); + settings.pathNormalizationVersion = 1; + await writeFile(path, `${JSON.stringify(settings, null, 2)}\n`, "utf-8"); + } + await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "rebuild", "--force", "--write-settings"]); + const info = await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "info", notePath]); + const fileInfo = JSON.parse(info.stdout) as CliFileInfo; + if (fileInfo.id === legacyFileInfo.id) { + throw new Error("Rebuild retained the legacy NFD document ID."); + } + await waitForCouchDbDocs(couchDb, dbName, (docs) => { + const ids = new Set(docs.map((doc) => doc._id)); + return ( + !ids.has(legacyFileInfo.id) && + ids.has(fileInfo.id) && + fileInfo.children.every((childId) => ids.has(childId)) && + docs.some((doc) => Array.isArray(doc.used_features) && doc.used_features.includes("nfc-path-identity-v1")) + ); + }); + await runLiveSyncCli(command, [ + secondaryDatabasePath, + "--settings", + secondarySettingsPath, + "fetch", + "--force", + "--write-settings", + ]); + const secondaryInfo = await runLiveSyncCli(command, [ + secondaryDatabasePath, + "--settings", + secondarySettingsPath, + "info", + notePath, + ]); + assertEqual( + (JSON.parse(secondaryInfo.stdout) as CliFileInfo).id, + fileInfo.id, + "Fetch did not replace the secondary CLI's legacy document identity." + ); + for (const path of [databasePath, secondaryDatabasePath]) { + assertEqual( + await readFile(join(path, notePath), "utf-8"), + editedNoteContent, + "Migration changed the content of the original NFD physical file." + ); + await access(join(path, receivedNotePath)).then( + () => { + throw new Error("Migration created a second NFC physical file."); + }, + (error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + } + ); + } + console.log("CLI Rebuild and Fetch migrated legacy document IDs while preserving both NFD physical files."); + return { ...fileInfo, expectedCtime: legacyFileInfo.expectedCtime }; +} + async function main(): Promise { const liveSyncCliCommand = resolveLiveSyncCliCommand(); if (process.env.LIVESYNC_CLI_COMMAND === undefined) { @@ -304,12 +434,19 @@ async function main(): Promise { const cliSourcePath = join(cliState, "source", "cli-to-obsidian.md"); const vault = await createTemporaryVault(); let session: ObsidianLiveSyncSession | undefined; + let daemon: Awaited> | undefined; try { await assertCouchDbReachable(couchDb); await createCouchDbDatabase(couchDb, dbName); await mkdir(cliDatabasePath, { recursive: true }); - await configureLiveSyncCli(liveSyncCliCommand, cliSettingsPath, couchDb, dbName); + await configureLiveSyncCli( + liveSyncCliCommand, + cliSettingsPath, + couchDb, + dbName, + migrateUnicodePaths ? 0 : undefined + ); if (process.env.LIVESYNC_CLI_COMMAND === undefined) { console.log(`Using locally built LiveSync CLI: ${liveSyncCli}`); @@ -323,7 +460,21 @@ async function main(): Promise { console.log(`Temporary Obsidian vault: ${vault.path}`); console.log(`Temporary CouchDB database: ${dbName}`); - const cliFileInfo = await writeCliNote(liveSyncCliCommand, cliDatabasePath, cliSettingsPath, cliSourcePath); + let cliFileInfo = await writeCliNote(liveSyncCliCommand, cliDatabasePath, cliSettingsPath, cliSourcePath); + if (migrateUnicodePaths) { + cliFileInfo = await migrateCliDatabases( + liveSyncCliCommand, + cliState, + cliDatabasePath, + cliSettingsPath, + couchDb, + dbName, + cliFileInfo + ); + } + if (unicodePaths && !useEncryption) { + assertEqual(cliFileInfo.id, receivedNotePath.toLowerCase(), "The CLI did not derive an NFC document ID."); + } await waitForCouchDbDocs(couchDb, dbName, (docs) => { const ids = new Set(docs.map((doc) => doc._id)); return ids.has(cliFileInfo.id) && cliFileInfo.children.every((childId) => ids.has(childId)); @@ -342,10 +493,11 @@ async function main(): Promise { dbName, }, { - encrypt: true, + encrypt: useEncryption, passphrase: e2eePassphrase, - usePathObfuscation: true, - encryptInternalMetadata: true, + usePathObfuscation: useEncryption, + encryptInternalMetadata: useEncryption, + pathNormalizationVersion: unicodePaths ? 1 : 0, E2EEAlgorithm: "v2", } ), @@ -355,7 +507,7 @@ async function main(): Promise { await prepareRemote(obsidianCli.binary, session.cliEnv); await pushLocalChanges(obsidianCli.binary, session.cliEnv); - const received = await waitForVaultContent(vault.path, notePath); + const received = await waitForVaultContent(vault.path, receivedNotePath); assertEqual(received, editedNoteContent, "The real Obsidian plug-in did not materialise the CLI-edited note."); if (cliFileInfo.expectedCtime !== undefined) { const receivedCtime = await evalObsidianJson( @@ -363,7 +515,7 @@ async function main(): Promise { [ "(async()=>{", "const core=app.plugins.plugins['obsidian-livesync'].core;", - `const entry=await core.localDatabase.getDBEntry(${JSON.stringify(notePath)},undefined,false,true);`, + `const entry=await core.localDatabase.getDBEntry(${JSON.stringify(receivedNotePath)},undefined,false,true);`, "return JSON.stringify(entry.ctime);", "})()", ].join(""), @@ -376,12 +528,78 @@ async function main(): Promise { ); console.log("The CLI edit retained the filesystem creation time in the real Obsidian client's Metadata."); } else { - console.log("Creation-time verification skipped: native stats did not expose distinct creation and change times."); + console.log( + "Creation-time verification skipped: native stats did not expose distinct creation and change times." + ); } - console.log( - "CLI-edited encrypted note was retrieved by the real Obsidian plug-in with identical content." - ); + if (unicodePaths) { + const returnedContent = `${editedNoteContent}\nUpdated in Obsidian.\n`; + await evalObsidianJson( + obsidianCli.binary, + [ + "(async()=>{", + `const path=${JSON.stringify(receivedNotePath)};`, + "const file=app.vault.getAbstractFileByPath(path);", + "await new Promise(resolve=>setTimeout(resolve,Math.max(0,file.stat.mtime+2500-Date.now())));", + `await app.vault.modify(file,${JSON.stringify(returnedContent)});`, + "return JSON.stringify(true);})()", + ].join(""), + session.cliEnv + ); + await pushLocalChanges(obsidianCli.binary, session.cliEnv); + await runLiveSyncCli(liveSyncCliCommand, [cliDatabasePath, "--settings", cliSettingsPath, "sync"]); + await runLiveSyncCli(liveSyncCliCommand, [cliDatabasePath, "--settings", cliSettingsPath, "mirror"]); + assertEqual( + await readFile(join(cliDatabasePath, notePath), "utf-8"), + returnedContent, + "The Obsidian update did not reach the original NFD physical file." + ); + await access(join(cliDatabasePath, receivedNotePath)).then( + () => { + throw new Error("The CLI created a second NFC physical file."); + }, + (error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + } + ); + // The mirror command intentionally keeps tombstoned local files. The daemon applies remote deletions. + daemon = await startLiveSyncCliDaemon(liveSyncCliCommand, cliDatabasePath, cliSettingsPath); + await evalObsidianJson( + obsidianCli.binary, + [ + "(async()=>{", + `const file=app.vault.getAbstractFileByPath(${JSON.stringify(receivedNotePath)});`, + "await new Promise(resolve=>setTimeout(resolve,Math.max(0,file.stat.mtime+2500-Date.now())));", + "await app.vault.delete(file);", + "return JSON.stringify(true);})()", + ].join(""), + session.cliEnv + ); + await pushLocalChanges(obsidianCli.binary, session.cliEnv); + const deletionDeadline = Date.now() + Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS); + for (;;) { + try { + await access(join(cliDatabasePath, notePath)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + break; + } + if (Date.now() >= deletionDeadline) { + throw new Error( + `The Obsidian deletion did not remove the original NFD physical file.\n${daemon.getOutput()}` + ); + } + await new Promise((resolvePromise) => setTimeout(resolvePromise, 100)); + } + await daemon.stop(); + daemon = undefined; + console.log( + "NFC path identity preserved the NFD physical name through creation, remote update, and deletion." + ); + } + console.log("CLI-edited note was retrieved by the real Obsidian plug-in with identical content."); } finally { + await daemon?.stop(); if (session) { await session.app.stop(); } diff --git a/updates.md b/updates.md index 31d529ec..131b01a7 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,12 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Fixed + +- We can opt into NFC path identity for CouchDB synchronisation between the CLI and Obsidian. This prevents decomposed or mixed Unicode file names from becoming inaccessible through a receiving device's normalised path. (#1259) + - Existing Vaults retain their current identity rules. Migration requires updated clients, a Rebuild from a complete authoritative Vault, and a Fetch on the other devices. + - Reconstruction stops when the supported host file listing or local Metadata exposes ambiguous files or parent folders under NFC and the configured case handling. The CLI retains existing physical file names. P2P and Object Storage do not support the new mode. + ## 1.0.37 16th October, 2026 From 081d3f3339b633e25dc6882eb41911d365420eda Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 13:39:22 +0000 Subject: [PATCH 07/68] Align multi-device tests with retained content evidence --- ...ileHandler.multidevice.integration.spec.ts | 41 +++++++++++++++++-- test/README.md | 5 ++- updates.md | 3 ++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/serviceModules/FileHandler.multidevice.integration.spec.ts b/src/serviceModules/FileHandler.multidevice.integration.spec.ts index af82d242..77abe779 100644 --- a/src/serviceModules/FileHandler.multidevice.integration.spec.ts +++ b/src/serviceModules/FileHandler.multidevice.integration.spec.ts @@ -13,6 +13,7 @@ import { type ServiceDatabaseFileAccessDependencies, } from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceDatabaseFileAccessBase"; import type { ServiceFileHandlerDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceFileHandlerBase"; +import type { FileReflectionProvenanceRecord } from "@vrtmrz/livesync-commonlib/compat/interfaces/FileReflectionProvenance"; import { ServiceFileHandler } from "./FileHandler"; import { createConflictResolutionOperations, @@ -38,7 +39,7 @@ function makeFile(body: string, mtime = oldTime): UXFileInfo { async function makeDevice(name: string) { const db = new PouchDB(name, { adapter: "memory" }); - const reflection = new Map(); + const reflection = new Map(); let storage = makeFile(old); const settings = { ...DEFAULT_SETTINGS, useOnlyLocalChunk: true, writeDocumentsIfConflicted: false }; const setting = { currentSettings: () => settings }; @@ -94,7 +95,7 @@ async function makeDevice(name: string) { conflict, fileReflectionProvenance: { get: (value: FilePathWithPrefix) => Promise.resolve(reflection.get(value)), - set: (value: FilePathWithPrefix, record: { revision: string }) => { + set: (value: FilePathWithPrefix, record: FileReflectionProvenanceRecord) => { reflection.set(value, record); return Promise.resolve(); }, @@ -287,7 +288,38 @@ describe("file provenance across multiple devices and real CouchDB", () => { } }, 60_000); - it.each(["missing record", "compacted base"] as const)( + it("reflects the selected revision after compaction when saved content evidence proves the file unchanged", async () => { + const f = await conflictedDevices(4); + const { keep, content } = await resolveAndReplicate(f); + const index = f.revisions.findIndex((rev, i) => i > 0 && rev !== keep); + const device = f.devices[index]; + const storage = device.getStorage(); + expect(await storage.body.text()).not.toBe(content); + expect(device.reflection.get(path)).toMatchObject({ + revision: f.revisions[index], + contentHash: expect.stringMatching(/^sha256:[0-9a-f]{64}$/u), + contentSize: storage.body.size, + }); + await device.db.compact(); + await expect(device.db.get(path, { rev: f.revisions[index] })).rejects.toMatchObject({ status: 404 }); + + const before = (await device.db.info()).update_seq; + await expect(device.handler.storeFileToDB(path)).resolves.toBe(true); + expect((await device.db.info()).update_seq).toBe(before); + expect(await device.getStorage().body.text()).toBe(content); + expect(device.reflection.get(path)?.revision).toBe(keep); + await expect(device.handler.storeFileToDB(path)).resolves.toBe(true); + expect((await device.db.info()).update_seq).toBe(before); + await device.db.replicate.to(f.remote); + for (const peer of f.devices) { + await peer.db.replicate.from(f.remote); + expect((await leaves(peer.db)).map((doc) => doc._rev)).toEqual([keep]); + expect(await revisionContent(peer, keep)).toBe(content); + } + expect((await leaves(f.remote)).map((doc) => doc._rev)).toEqual([keep]); + }, 60_000); + + it.each(["missing record", "compacted base without content evidence"] as const)( "preserves uncertain storage as one independent conflict with four devices: %s", async (reason) => { const f = await conflictedDevices(4); @@ -299,6 +331,9 @@ describe("file provenance across multiple devices and real CouchDB", () => { // Matching an old ancestor must not be mistaken for an unchanged current branch. device.setStorage(makeFile(old)); } else { + // Legacy records retain the revision but have no saved content hash or byte length. + const { revision, observedStorageMtime } = device.reflection.get(path)!; + device.reflection.set(path, { revision, observedStorageMtime }); await device.db.compact(); await expect(device.db.get(path, { rev: f.revisions[index] })).rejects.toMatchObject({ status: 404 }); } diff --git a/test/README.md b/test/README.md index b1c0cfac..31b87e96 100644 --- a/test/README.md +++ b/test/README.md @@ -98,7 +98,7 @@ npm run test:docker-couchdb:start NODE_OPTIONS=--max-old-space-size=3072 npm run test:integration -- src/serviceModules/FileHandler.multidevice.integration.spec.ts --maxWorkers=1 ``` -The expected result is five passing tests. Each case creates a uniquely named remote database and removes it during teardown. Stop the fixture after the run, including when the test command fails: +The expected result is six passing tests. Each case creates a uniquely named remote database and removes it during teardown. Stop the fixture after the run, including when the test command fails: ```bash npm run test:docker-couchdb:stop @@ -116,7 +116,8 @@ Each simulated device owns a separate real PouchDB database, `LiveSyncLocalDB` m | Four editing devices | Repeat the same sequence with four independently edited branches. | All four contents are initially preserved, and the same unchanged-save and convergence guarantees hold. | | A genuine edit on a losing device | After three-way resolution reaches its DB, a device adds content to its still-unreflected losing file, then saves and replicates it. | The new revision extends that device's recorded branch. The selected result and the new edit remain readable on every replica. | | Missing provenance | After four-way resolution, remove a losing device's provenance record and set its file to the original historical ancestor's content. Save, remove provenance again, and repeat the save. | The file becomes one fresh independent root, distinct from the historical root. Both contents remain readable after replication, and repeated saves create no duplicates. CLI `resolve` can select the independent root and propagate its resolution. | -| Compacted base | After four-way resolution, compact a losing device's local DB and confirm that its recorded revision body is unavailable. Save the remaining file, then repeat after removing provenance. | The file is preserved as one independent conflict rather than discarded. Both contents remain readable on every replica, repeated saves create no duplicates, and CLI `resolve` can select the independent root and propagate its resolution. | +| Compacted base with saved content evidence | After four-way resolution, retain the saved content hash and byte length, compact a losing device's local DB, and confirm that its recorded revision body is unavailable. Process its unchanged file twice. | The selected revision is reflected without a database write or a new conflict. Provenance advances to that revision, and every replica retains only the selected non-deleted branch. | +| Compacted base without content evidence | After four-way resolution, retain a legacy provenance record with only its revision and modification time, compact a losing device's local DB, and confirm that its recorded revision body is unavailable. Save the remaining file, then repeat after removing provenance. | The file is preserved as one independent conflict rather than discarded. Both contents remain readable on every replica, repeated saves create no duplicates, and CLI `resolve` can select the independent root and propagate its resolution. | ### Coverage boundaries diff --git a/updates.md b/updates.md index 131b01a7..495e99aa 100644 --- a/updates.md +++ b/updates.md @@ -17,6 +17,9 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - We can opt into NFC path identity for CouchDB synchronisation between the CLI and Obsidian. This prevents decomposed or mixed Unicode file names from becoming inaccessible through a receiving device's normalised path. (#1259) - Existing Vaults retain their current identity rules. Migration requires updated clients, a Rebuild from a complete authoritative Vault, and a Fetch on the other devices. - Reconstruction stops when the supported host file listing or local Metadata exposes ambiguous files or parent folders under NFC and the configured case handling. The CLI retains existing physical file names. P2P and Object Storage do not support the new mode. +- We can receive incoming updates to unchanged files after their recorded database revision has been pruned, without creating false conflicts. (#1207) + - Saved content evidence establishes that the local file is unchanged. Genuine edits and older records without this evidence retain conservative handling. +- We can receive deletions and renames when the source removes the file's Metadata. ## 1.0.37 From bf8e899bf797fe21f14f8f0d11e9226385148d02 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 13:49:11 +0000 Subject: [PATCH 08/68] Cover content evidence after conflict-base compaction --- ...ileHandler.multidevice.integration.spec.ts | 43 +++++++++++++++++-- test/README.md | 6 ++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/serviceModules/FileHandler.multidevice.integration.spec.ts b/src/serviceModules/FileHandler.multidevice.integration.spec.ts index af82d242..09fd9367 100644 --- a/src/serviceModules/FileHandler.multidevice.integration.spec.ts +++ b/src/serviceModules/FileHandler.multidevice.integration.spec.ts @@ -13,6 +13,7 @@ import { type ServiceDatabaseFileAccessDependencies, } from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceDatabaseFileAccessBase"; import type { ServiceFileHandlerDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceFileHandlerBase"; +import type { FileReflectionProvenanceRecord } from "@vrtmrz/livesync-commonlib/compat/interfaces/FileReflectionProvenance"; import { ServiceFileHandler } from "./FileHandler"; import { createConflictResolutionOperations, @@ -38,7 +39,7 @@ function makeFile(body: string, mtime = oldTime): UXFileInfo { async function makeDevice(name: string) { const db = new PouchDB(name, { adapter: "memory" }); - const reflection = new Map(); + const reflection = new Map(); let storage = makeFile(old); const settings = { ...DEFAULT_SETTINGS, useOnlyLocalChunk: true, writeDocumentsIfConflicted: false }; const setting = { currentSettings: () => settings }; @@ -94,7 +95,7 @@ async function makeDevice(name: string) { conflict, fileReflectionProvenance: { get: (value: FilePathWithPrefix) => Promise.resolve(reflection.get(value)), - set: (value: FilePathWithPrefix, record: { revision: string }) => { + set: (value: FilePathWithPrefix, record: FileReflectionProvenanceRecord) => { reflection.set(value, record); return Promise.resolve(); }, @@ -287,7 +288,32 @@ describe("file provenance across multiple devices and real CouchDB", () => { } }, 60_000); - it.each(["missing record", "compacted base"] as const)( + it("reflects the selected content after compacting an unchanged losing branch", async () => { + const f = await conflictedDevices(4); + const { keep, content } = await resolveAndReplicate(f); + const index = f.revisions.findIndex((rev, i) => i > 0 && rev !== keep); + const device = f.devices[index]; + const record = device.reflection.get(path); + expect(record?.revision).toBe(f.revisions[index]); + expect(record?.contentHash).toMatch(/^sha256:[0-9a-f]{64}$/u); + expect(record?.contentSize).toBe(device.getStorage().body.size); + await device.db.compact(); + await expect(device.db.get(path, { rev: f.revisions[index] })).rejects.toMatchObject({ status: 404 }); + + const before = (await device.db.info()).update_seq; + await expect(device.handler.storeFileToDB(path)).resolves.toBe(true); + expect((await device.db.info()).update_seq).toBe(before); + expect(await device.getStorage().body.text()).toBe(content); + expect(device.reflection.get(path)?.revision).toBe(keep); + await device.db.replicate.to(f.remote); + for (const peer of f.devices) { + await peer.db.replicate.from(f.remote); + expect((await leaves(peer.db)).map((doc) => doc._rev)).toEqual([keep]); + expect(await revisionContent(peer, keep)).toBe(content); + } + }, 60_000); + + it.each(["missing record", "compacted legacy base", "compacted hashed base with local edits"] as const)( "preserves uncertain storage as one independent conflict with four devices: %s", async (reason) => { const f = await conflictedDevices(4); @@ -301,6 +327,17 @@ describe("file provenance across multiple devices and real CouchDB", () => { } else { await device.db.compact(); await expect(device.db.get(path, { rev: f.revisions[index] })).rejects.toMatchObject({ status: 404 }); + if (reason === "compacted legacy base") { + // Revision-only records still require protection when their recorded body is unavailable. + const { revision, observedStorageMtime } = device.reflection.get(path)!; + device.reflection.set(path, { revision, observedStorageMtime }); + } else { + expect(device.reflection.get(path)?.contentHash).toMatch(/^sha256:/u); + const original = device.getStorage(); + const edited = (await original.body.text()).replace("Edited", "EDITED"); + device.setStorage(makeFile(edited, original.stat.mtime)); + expect(device.getStorage().body.size).toBe(original.body.size); + } } const uncertainContent = await device.getStorage().body.text(); await expect(device.handler.storeFileToDB(path)).resolves.toBe(true); diff --git a/test/README.md b/test/README.md index b1c0cfac..7baaaf99 100644 --- a/test/README.md +++ b/test/README.md @@ -98,7 +98,7 @@ npm run test:docker-couchdb:start NODE_OPTIONS=--max-old-space-size=3072 npm run test:integration -- src/serviceModules/FileHandler.multidevice.integration.spec.ts --maxWorkers=1 ``` -The expected result is five passing tests. Each case creates a uniquely named remote database and removes it during teardown. Stop the fixture after the run, including when the test command fails: +The expected result is seven passing tests. Each case creates a uniquely named remote database and removes it during teardown. Stop the fixture after the run, including when the test command fails: ```bash npm run test:docker-couchdb:stop @@ -116,7 +116,9 @@ Each simulated device owns a separate real PouchDB database, `LiveSyncLocalDB` m | Four editing devices | Repeat the same sequence with four independently edited branches. | All four contents are initially preserved, and the same unchanged-save and convergence guarantees hold. | | A genuine edit on a losing device | After three-way resolution reaches its DB, a device adds content to its still-unreflected losing file, then saves and replicates it. | The new revision extends that device's recorded branch. The selected result and the new edit remain readable on every replica. | | Missing provenance | After four-way resolution, remove a losing device's provenance record and set its file to the original historical ancestor's content. Save, remove provenance again, and repeat the save. | The file becomes one fresh independent root, distinct from the historical root. Both contents remain readable after replication, and repeated saves create no duplicates. CLI `resolve` can select the independent root and propagate its resolution. | -| Compacted base | After four-way resolution, compact a losing device's local DB and confirm that its recorded revision body is unavailable. Save the remaining file, then repeat after removing provenance. | The file is preserved as one independent conflict rather than discarded. Both contents remain readable on every replica, repeated saves create no duplicates, and CLI `resolve` can select the independent root and propagate its resolution. | +| Compacted base with verified unchanged content | After four-way resolution, retain a losing device's content hash, compact its local DB, and confirm that its recorded revision body is unavailable. Save the unchanged file. | The selected content is reflected without adding a database revision or conflict, and all replicas retain only the selected branch. | +| Compacted base with legacy provenance | After four-way resolution, retain only the losing device's revision and observed modification time, compact its local DB, and confirm that the recorded revision body is unavailable. Save the remaining file, then repeat after removing provenance. | The file is preserved as one independent conflict. Both contents remain readable on every replica, repeated saves create no duplicates, and CLI `resolve` can select the independent root and propagate its resolution. | +| Compacted base with a genuine edit | After four-way resolution and local compaction, retain the losing device's content hash but edit its file without changing its byte length or modification time. | The changed content is preserved as one independent conflict, remains readable on every replica, and can be selected through CLI `resolve`. Repeated saves after provenance loss create no duplicates. | ### Coverage boundaries From 0fd24cd79e971f685125f89c5858b9fcc306d296 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 14:21:38 +0000 Subject: [PATCH 09/68] Align recovery and path identity documentation with Commonlib 0.1.42 --- docs/adr/2026_10_unicode_path_identity.md | 2 ++ docs/design_docs/tweak_compatibility.md | 28 +++++++++++++----- docs/design_docs/unicode_path_identity.md | 35 +++++++++-------------- docs/recovery.md | 16 +++++++++++ docs/settings.md | 4 ++- docs/specs_conflict_resolution.md | 33 ++++++++++++++------- docs/troubleshooting.md | 8 ++++-- 7 files changed, 84 insertions(+), 42 deletions(-) diff --git a/docs/adr/2026_10_unicode_path_identity.md b/docs/adr/2026_10_unicode_path_identity.md index eb503b50..839ea986 100644 --- a/docs/adr/2026_10_unicode_path_identity.md +++ b/docs/adr/2026_10_unicode_path_identity.md @@ -131,5 +131,7 @@ are detected. - [Unicode path identity design](../design_docs/unicode_path_identity.md) - [Setting details](../settings.md#unicode-path-identity) +- [User migration procedure](../recovery.md#migrate-to-unicode-nfc-path-identity) +- [Device-local file provenance](../specs_conflict_resolution.md#device-local-file-provenance) - [Commonlib Unicode path identity contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md) - [Project glossary](../glossary.md) diff --git a/docs/design_docs/tweak_compatibility.md b/docs/design_docs/tweak_compatibility.md index 965e5f3b..be7dda44 100644 --- a/docs/design_docs/tweak_compatibility.md +++ b/docs/design_docs/tweak_compatibility.md @@ -1,15 +1,16 @@ --- -date: 2026-09-08 -commonlib-version: "0.1.24" -self-hosted-livesync-version: "1.0.27" +date: 2026-10-17 +commonlib-version: "0.1.42" +self-hosted-livesync-version: "1.0.37" status: unreleased --- # Tweak compatibility and recovery -This document describes the integration of Commonlib 0.1.24 with -Self-hosted LiveSync 1.0.27. Commonlib owns the interpretation of synchronisation -settings; LiveSync owns the dialogues and operations which consume that result. +This document describes the compatibility and recovery contract, including +unreleased changes after Self-hosted LiveSync 1.0.37 using Commonlib 0.1.42. +Commonlib owns the interpretation of synchronisation settings; LiveSync owns +the dialogues and operations which consume that result. ## Shared assessment @@ -25,11 +26,16 @@ document IDs. An explicitly enabled value is therefore different from either an explicitly disabled value or a missing value. Settings whose historical missing value has not been established remain unadvertised; the evaluator does not invent defaults or turn every falsy value into an absent value. +`pathNormalizationVersion` is another explicit compatibility rule: missing and +`0` select legacy identity, while `1` requires coordinated reconstruction. +The [Unicode path identity design](unicode_path_identity.md) defines its +admission checks and supported transports. The central replication gate, mismatch dialogues, and RedFlag Fetch preparation consume the same effective differences. The P2P transport retains its separate policy: ordinary representation differences warn without rejecting transfer, -while its existing passphrase and peer checks still apply. Sharing assessment +while its existing passphrase and peer checks still apply. NFC path identity +is outside that warning-only policy: P2P refuses version `1`. Sharing assessment does not make the central replication policy appropriate for every transport. ## Host responsibilities @@ -100,3 +106,11 @@ replication, explicit Fetch, and replication after restart. A unit test which mocks Fetch does not establish those behaviours, and a corrected mismatch dialogue alone does not establish the cause of a reported persistent automatic synchronisation failure. + +The `tweak-compatibility` real-Obsidian scenario covers automatic and manual +adoption, stale target rejection, and adoption which activates either E2EE or +Property Encryption. Those prerequisite changes must retain the adopted +effective Metadata encryption value through Fetch, restart, and two-way file +transfer. See the [test guide](../../test/e2e-obsidian/README.md) for the scenario +and explicit artefact selection. The cause of an earlier saved chunk-size +change is separate from these recovery guarantees. diff --git a/docs/design_docs/unicode_path_identity.md b/docs/design_docs/unicode_path_identity.md index 7c3fd9f8..4a9f4210 100644 --- a/docs/design_docs/unicode_path_identity.md +++ b/docs/design_docs/unicode_path_identity.md @@ -16,7 +16,7 @@ the [Unicode path identity ADR](../adr/2026_10_unicode_path_identity.md). Commonlib's [consumer contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md) defines the shared package boundary. -This records an unreleased change to Self-hosted LiveSync `1.0.37`, using the +This records an unreleased change after Self-hosted LiveSync `1.0.37`, using the published Commonlib `0.1.42` package as an exact dependency. ## Scope @@ -128,26 +128,12 @@ has been successfully reserved; cancellation or failed reservation leaves it pending. A `remoteOnly` operation cannot apply a pending identity change, and the settings-only fallback is not offered. -Migration uses this ordered procedure: - -1. Update every CLI and Obsidian client which will use the database. Stop - synchronisation and editing on all devices, and keep a complete backup. -2. Select the Vault whose complete file set is authoritative. Resolve any - collision reported by Rebuild before continuing. -3. Select version `1` on that source. In Obsidian, choose **Overwrite Server - Data with This Device's Files**; the new mode becomes active only when the - Rebuild reservation succeeds. In the CLI, use settings with - `pathNormalizationVersion: 1` and run `rebuild` with `--force - --write-settings`. Both local and remote databases are reconstructed from - the specified physical Vault. -4. Select version `1` on each remaining client. In Obsidian, choose **Reset - Synchronisation on This Device**; the new mode becomes active only when the - Fetch reservation succeeds. In the CLI, use settings with - `pathNormalizationVersion: 1` and run `fetch` with `--force - --write-settings`. Each operation replaces that client's local database - from the rebuilt remote. -5. Confirm that every participating client is using version `1`, then resume - synchronisation. +The [user migration procedure](../recovery.md#migrate-to-unicode-nfc-path-identity) +owns the operational sequence: updated and stopped clients, a complete backed-up +authoritative Vault, source Rebuild, then Fetch on each remaining client. The +source operation reconstructs both databases; a receiving client's Fetch +replaces only its local database. The [CLI guide](../../src/apps/cli/README.md) +provides the equivalent `rebuild` and `fetch` commands. The CLI checks `--force` before core start-up so an omitted confirmation cannot trigger database access or destructive preparation. Neither `--force` nor a @@ -171,7 +157,11 @@ The path identity is also used by two existing resume mechanisms: keys with NFC in mode `1` for reads, writes, deletes, raw snapshot lookup, and untracked revision recording. Mode `0` retains the old keys. Rebuild and Fetch use the existing local-state reconstruction and replacement lifecycle; - no new persistent field or in-place key migration is introduced. + no NFC-specific persistent field or in-place key migration is introduced. + Commonlib 0.1.42 separately adds content hashes to the same records. The + [file-provenance contract](../specs_conflict_resolution.md#device-local-file-provenance) + defines that content evidence; changing a path key neither changes file + bytes nor supplies missing revision ancestry. - **Fast Fetch checkpoints:** the path version participates in checkpoint identity so an interrupted transfer cannot resume using a different path-to-ID rule. Existing checkpoint persistence and retry handling remain @@ -259,5 +249,6 @@ This design does not: - [Unicode path identity ADR](../adr/2026_10_unicode_path_identity.md) - [Setting details](../settings.md#unicode-path-identity) +- [User migration procedure](../recovery.md#migrate-to-unicode-nfc-path-identity) - [Commonlib Unicode path identity contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md) - [Metadata and Chunks glossary](../glossary.md#user-facing-and-operational-terms) diff --git a/docs/recovery.md b/docs/recovery.md index d9083b82..de9c5c17 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -18,6 +18,22 @@ Use the least destructive operation which matches the evidence: Do not switch transport, enable P2P, or run Garbage Collection as a substitute for diagnosing a stopped CouchDB or Object Storage setup. +## Migrate to Unicode NFC path identity + +Use this coordinated migration when CLI and Obsidian clients share a CouchDB database and composed and decomposed Unicode spellings should identify the same path. Version `0` remains the default and preserves legacy identity. Version `1` is supported with CouchDB only; it is unsupported with Object Storage and P2P. + +This migration reconstructs the databases from one authoritative Vault. It does not rewrite document IDs in place or rename files. + +1. Update every participating CLI and Obsidian client to a version which supports Unicode NFC path identity. Back up a complete authoritative Vault and the CouchDB database. Stop synchronisation and editing on every device until the migration is complete. +2. Choose the complete Vault which contains the intended files. Review any composed and decomposed names which would map to the same path identity, and resolve them before reconstruction. If Rebuild or Fetch reports an ambiguous path or Metadata ID, stop and resolve that collision before retrying; LiveSync does not choose or merge the files for you. +3. On the source client, select **Unicode NFC path identity** (`pathNormalizationVersion: 1`). In Obsidian, choose **Overwrite Server Data with This Device's Files**. In the CLI, set the same value in its settings and run `rebuild` with `--force --write-settings`. +4. On each remaining client, select **Unicode NFC path identity** (`pathNormalizationVersion: 1`) before fetching. In Obsidian, choose **Reset Synchronisation on This Device**. In the CLI, set the same value in its settings and run `fetch` with `--force --write-settings`. +5. Confirm that every client uses version `1` and that the expected files are present before resuming synchronisation. Do not reconnect a version `0` client to the migrated database. + +The Inspector's **Repair this Metadata document ID** action repairs one local entry; it is not a bulk migration tool. Use the Rebuild and Fetch procedure above for path identity changes. + +This procedure does not establish native macOS or iOS filesystem behaviour. Obsidian's collision checks use the paths exposed by its Vault APIs; see [Unicode path identity](settings.md#unicode-path-identity) for that visibility limit. + ## Suspend before diagnosis Close Obsidian completely, then create an empty file or directory named `redflag.md` at the root of the Vault. On the next start, LiveSync enters its emergency suspension state before ordinary database, file-watching, and synchronisation work continues. diff --git a/docs/settings.md b/docs/settings.md index 519ff707..ab3fc26e 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -1012,6 +1012,8 @@ Requires restart of Obsidian Setting key: deleteMetadataOfDeletedFiles +For an unconflicted file, disabling this setting records a logical deletion in the Metadata document; enabling it removes that document through a PouchDB tombstone instead. Both behaviours propagate file deletions and the source side of a rename. Before removing a Vault file, LiveSync revalidates the current winning revision; if this final check observes a concurrent restoration, it keeps the restored file. This setting changes how the deletion is represented; file content remains stored separately in Chunks. Deletion during an unresolved conflict follows the separate [displayed-branch rules](specs_conflict_resolution.md#operations-while-a-conflict-exists). + #### Delete old metadata of deleted files on start-up Setting key: automaticallyDeleteMetadataOfDeletedFiles @@ -1056,7 +1058,7 @@ Setting key: pathNormalizationVersion Version `0` preserves legacy file identity and is the default for existing and new settings. Version `1` uses Unicode NFC when deriving document IDs, so equivalent composed and decomposed paths identify the same file across the CLI and Obsidian. Existing physical file names are retained. This mode currently supports CouchDB only. -Before enabling version `1`, update every participating client, stop synchronisation, and back up a complete authoritative Vault. Rebuild both databases from that Vault, then select version `1` and Fetch on each remaining device. Changing this setting without reconstruction is refused when it would mix database identities. Do not resume an older client against the migrated remote. +Version `1` requires a coordinated migration: set it on the source client before Rebuild, then set it on each remaining client before Fetch. Changing the setting alone does not migrate existing Metadata. Follow [Migrate to Unicode NFC path identity](recovery.md#migrate-to-unicode-nfc-path-identity) for the complete procedure. Do not resume a version `0` client against the migrated remote. The selected value stays pending through ordinary settings saves and reloads. It becomes active only when the Rebuild or Fetch operation is successfully reserved; cancellation leaves it pending. diff --git a/docs/specs_conflict_resolution.md b/docs/specs_conflict_resolution.md index 8f62b992..aabf5807 100644 --- a/docs/specs_conflict_resolution.md +++ b/docs/specs_conflict_resolution.md @@ -38,15 +38,17 @@ At most one revision is the winner, more than one revision can be Vault-matching - Automatic text and structured-data merge uses the nearest `available` revision ID which is present in both leaf histories. - Missing or compacted history stops conservative automatic merge instead of guessing a base. -- A Vault file which still matches its exact recorded revision is unchanged. An ordinary save does not append those stale bytes to a newer database revision; a newer, unconflicted database result is reflected through the existing file-reflection path. +- A Vault file which still matches its exact recorded content is unchanged. The comparison uses the readable recorded revision or its saved SHA-256 hash and byte length, including after that revision has been pruned. An ordinary save does not append those stale bytes to a newer database revision; a newer, unconflicted database result is reflected through the existing file-reflection path. - A file which differs from its readable recorded revision is an edit of that revision, even if its bytes match another historical revision. Saving and incoming overwrite protection use the same rule. -- Without a readable recorded revision, current non-deleted leaves are checked for duplicate content. If none matches, the file is preserved as a fresh independent root under the same document ID. Its unknown ancestry cannot supply a three-way merge base. +- Without a readable recorded revision or matching saved content evidence, current non-deleted leaves are checked for duplicate content. If none matches, the file is preserved as a fresh independent root under the same document ID. Its unknown ancestry cannot supply a three-way merge base. - File bytes, rather than path, size, modification time, or revision generation, determine whether content is known. - Three or more current versions are reviewed one pair at a time in a deterministic order, with each completed pair committed before the next pair is read. - Each device records the exact revision most recently reflected in each Vault file. An ordinary edit extends that displayed branch even before a conflict exists. Conflict-time deletion and case-only rename retain their separate displayed-branch contracts. - A cross-path rename stores the target before logically deleting only the displayed source branch. -The recorded revision can belong to a deleted losing branch. If its readable body still matches the Vault, the propagated resolution can be reflected without recreating the conflict. A historical byte match without that record does not establish that the file is unchanged: it may be an intentional revert. Existing Vaults can lack records, so an upgrade, reset, or unavailable old body can expose additional conflicts requiring review. +The recorded revision can belong to a deleted losing branch. If its readable body or saved content evidence still matches the Vault, the propagated resolution can be reflected without recreating the conflict. A historical byte match without that record does not establish that the file is unchanged: it may be an intentional revert. Existing Vaults can lack records or contain legacy records without content evidence, so an upgrade, reset, or unavailable old body can still expose conflicts requiring review. Content evidence neither reconstructs a missing merge base nor resolves conflicts which have already been created. + +Ordinary deletion reflection accepts both logical deletion Metadata and PouchDB tombstones produced when `deleteMetadataOfDeletedFiles` is enabled. Commonlib revalidates the winning deletion before removing the Vault file. A restoration observed during that final check prevents deletion and allows the restored revision to be reflected. This does not change the separate conflict-time deletion rules below. The explicit **Always overwrite with a newer file** option retains its existing modification-time policy. An independent branch prevents an inferred three-way merge; it does not disable the user's selected conflict-resolution option. Metadata and Chunks retain their existing format, and matching chunks can be shared between branches. @@ -59,10 +61,11 @@ The explicit **Always overwrite with a newer file** option retains its existing | One side deletes content which the other leaves unchanged | Preserve the deletion. | | One side deletes content which the other modifies | Ask the user. | | A receiving file matches its exact readable recorded revision | Apply the propagated database result under the existing conflict policy. | +| A receiving file matches its recorded SHA-256 hash and byte length | Apply the propagated result under the same policy even if the recorded revision has been pruned. | | A receiving file differs from its readable recorded revision | Preserve the edit as a child of that exact revision. | | Provenance is unknown and no current non-deleted leaf matches the file | Preserve a fresh independent branch for conflict resolution. | | Provenance is unknown and current non-deleted leaves already hold the file bytes | Avoid duplicate storage; infer provenance only for a unique match. | -| A required body or shared ancestor is missing or compacted | Ask the user. | +| A body or shared ancestor required for merging is missing or compacted | Ask the user. | | Binary contents differ | Prefer an explicit user selection; semantic merge is unavailable. | The compatibility implementation currently selects the newer modification time for differing binary conflicts even when the general **Always overwrite with a newer file** option is disabled. This is existing behaviour, not a new 1.0 guarantee. Changing it to explicit selection only is a separate compatibility decision. @@ -140,10 +143,16 @@ Consequently, choosing **Not now** or closing Obsidian cannot undo a completed p LiveSync composes Commonlib's injected `FileReflectionProvenance` with its local key-value database. Each device stores: ```text -path -> { revision, observedStorageMtime? } +path -> { revision, observedStorageMtime?, contentHash?, contentSize? } ``` -`revision` identifies the exact database revision most recently saved from or reflected in this device's Vault. It is the base for subsequent local edits, rather than a certificate that the current file still contains those bytes. `observedStorageMtime` is the raw local modification time of the saved snapshot or the file observed after reflection. It is not rounded, combined with another device's value, or used as proof of branch identity. No content hash is persisted. +`revision` identifies the exact database revision most recently saved from or reflected in this device's Vault. It is the base for subsequent local edits, rather than a certificate that the current file still contains those bytes. `observedStorageMtime` is the raw local modification time of the saved snapshot or the file observed after reflection. It is not rounded, combined with another device's value, or used as proof of branch identity. + +Commonlib 0.1.42 adds optional `contentHash` and `contentSize` fields for the exact immutable content successfully saved or reflected. The hash uses `sha256:`; the size is the byte length. LiveSync persists these fields in its existing `file-reflection-provenance-v1` store. They are device-local evidence, not replicated Metadata, a replacement revision ID, or a way to reconstruct missing ancestry. Matching evidence can establish unchanged content, but cannot attach a genuine edit to an unavailable base. + +Legacy records remain readable. An equal-time scan can add evidence when the recorded revision is still the conflict-free current revision and its bytes match storage. An ordinary unchanged save can also upgrade its verified current record. Hashing is followed by the required storage and provenance revalidation; an unreadable or unrelated revision cannot be certified from the current file. Unsupported evidence or a hashing failure retains the existing revision-based comparison and conservative fallback. + +Commonlib owns the [content-hash decision and compatibility contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/file-reflection-content-hash.md). It serialises hashing before reading the Blob to bound simultaneous whole-file buffers, and reuses results only for the same immutable Blob. Whole-file allocation remains necessary. LiveSync's [Review Harness](../devs.md#file-content-hash-performance-measurements) compares that production helper with xxHash64; it does not select the production algorithm. The record changes after a successful database-to-Vault reflection or Vault-to-database write. An equal-time scan may also fill a missing record while the current database revision uniquely matches the file's bytes and no conflict leaf exists. It rereads storage and the current database revision before recording; it does not change file bytes or create a database revision. Ordinary file reads do not change the record. An existing recorded revision remains authoritative even if the user edits the file to bytes which equal another branch; otherwise content equality could silently move the edit to a branch which was not displayed. @@ -153,7 +162,9 @@ The common lock does not stop Obsidian edits, external filesystem writes, or rep LiveSync creates the namespaced store handle during service composition, before the key-value database is open. The sequential `onSettingLoaded` lifecycle opens that database before Vault scanning, watching, or replication starts. Store operations do not wait for implicit readiness: a lifecycle violation fails promptly, avoiding an indefinite or self-referential initialisation wait. Local database reset is a transient unavailable boundary, after which scanning reconstructs derived state. -For ordinary saves and incoming reflection, a missing or unreadable recorded base permits reconstruction only from exactly one matching current non-deleted leaf. Matching several current leaves avoids duplicate storage but does not identify a displayed branch. No current match creates an independent branch, even when an older ancestor has the same bytes. Deletion and rename retain their existing provenance-recovery contracts. +For ordinary saves and incoming reflection, a missing or unreadable recorded base without matching saved content evidence permits reconstruction only from exactly one matching current non-deleted leaf. Matching several current leaves avoids duplicate storage but does not identify a displayed branch. No current match creates an independent branch, even when an older ancestor has the same bytes. Deletion and rename retain their existing provenance-recovery contracts. + +Provenance keys follow the selected [Unicode path identity](design_docs/unicode_path_identity.md#state-which-follows-the-selected-identity). In NFC mode, equivalent path spellings address the same record; the content evidence continues to describe file bytes, independently of that path spelling. ## Operations while a conflict exists @@ -240,7 +251,7 @@ If the user renames `draft.md` to `published.md`, LiveSync stores `published.md` ### A remote resolution reaches a device which still shows the losing content -Android may resolve a conflict and continue editing while Mac still shows the losing revision. When Mac receives the resolved tree, LiveSync compares Mac's bytes with the exact revision recorded for its Vault. If that body remains readable and matches, it can apply Android's resolution without asking Mac to resolve the same unchanged conflict again. +Android may resolve a conflict and continue editing while Mac still shows the losing revision. When Mac receives the resolved tree, LiveSync compares Mac's bytes with the exact revision recorded for its Vault or its saved content evidence. If they match, it can apply Android's resolution without asking Mac to resolve the same unchanged conflict again, including when the recorded revision has been pruned. If the user edited the file on Mac before the resolution arrived, the bytes no longer match that historical revision. LiveSync preserves the Mac edit as an unsynchronised conflict instead of overwriting it. @@ -277,9 +288,11 @@ Do not: ## Verification -LiveSync also exercises three and four independently editing devices through real CouchDB, using the installed Commonlib package and the CLI conflict-resolution command dispatcher. These tests check unchanged losing files before and after resolution, genuine edits on a losing branch, missing provenance, compacted bases, independent-root deduplication, and propagation of the selected result. See the [multiple-device regression procedure and coverage boundaries](../test/README.md#multiple-device-conflict-regression-tests). +LiveSync also exercises three and four independently editing devices through real CouchDB, using the installed Commonlib package and the CLI conflict-resolution command dispatcher. The compaction cases distinguish unchanged files with saved content evidence, legacy records without it, and genuine edits which retain the same byte length and modification time. Repeated saves must create no duplicate revisions, and the selected result must propagate to every replica. See the [multiple-device regression procedure and coverage boundaries](../test/README.md#multiple-device-conflict-regression-tests). -Commonlib owns the real-PouchDB and injected-boundary tests for revision ancestry, content preservation, provenance, independent branches, and repeated file events. LiveSync owns persistent host composition and actual Obsidian restart coverage. The focused `test:e2e:obsidian:stale-file-restart` scenario advances the local DB while old Vault bytes remain, persists pending file events, and restarts the same isolated profile. It requires an unchanged recorded file to reflect the DB without a new revision, an unknown file to remain on an independent branch alongside the DB content, and repeated processing after provenance loss to leave those branches unchanged. It also removes the record for a file whose bytes still match the current database revision, checks that start-up restores the record without a new database revision, and then checks that a later incoming revision reflects without a conflict. It uses real local storage and startup processing; transport replication and mobile lifecycle coverage are separate. +Commonlib owns the real-PouchDB and injected-boundary tests for revision ancestry, content preservation, provenance, independent branches, and repeated file events. LiveSync owns persistent host composition and actual Obsidian restart coverage. The focused `test:e2e:obsidian:stale-file-restart` scenario prunes a recorded revision while its old Vault bytes remain, persists the content evidence and pending file event, and restarts the same isolated profile. The unchanged file must reflect the newer content without a database write. Other fixtures check independent preservation of unknown content, deduplication after provenance loss, reconstruction of a missing record from the current revision, and verified upgrade of a legacy record. It uses real local storage and start-up processing; transport replication and mobile lifecycle coverage are separate. + +`E2E_OBSIDIAN_ONLY_METADATA_DELETION=true` selects two-Vault creation, update, deletion, and rename checks with `deleteMetadataOfDeletedFiles` disabled and enabled. It also delivers a real CouchDB restoration during the final deletion check, requiring the receiving file bytes and restored winning revision to remain, with no deletion event. LiveSync's optional real-Obsidian two-Vault checks have two scopes. `E2E_OBSIDIAN_INCLUDE_MARKDOWN_CONFLICT=true` resolves and edits a Markdown conflict, propagates it to a Vault which still displays the deleted losing content, and requires one current result to remain. `E2E_OBSIDIAN_INCLUDE_CONFLICT_OPERATIONS=true` edits, deletes, case-renames, and cross-path-renames files while conflicts remain active; it verifies the parent revision of each resulting branch, replicates those exact trees, and confirms that the other conflict branches remain intact. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 388c55a3..b4a2decc 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -51,7 +51,9 @@ If the remote is healthy but one device's local database is not, use [Reset Sync If the log says `Preserved unsynchronised local changes as a conflict`, keep both versions available until you have checked their contents. The message means that LiveSync could not prove that the Vault file was unchanged; it does not establish that you edited the file. Use **Inspect conflicts and file/database differences** in Hatch to review the current branches before selecting a version. -At start-up, LiveSync can restore a missing device-local revision record when the file still exactly matches the current local database revision. If a newer revision has already arrived, that match may no longer exist. LiveSync then preserves the old file for review because it cannot distinguish an unchanged copy from an intentional edit back to older content. A new release cannot resolve a conflict which was already created merely by recognising a historical content match. If the problem recurs, include the first relevant log entries, plug-in versions on both devices, and a redacted full report with the issue steps. +At start-up, LiveSync can restore a missing device-local revision record when the file exactly matches the conflict-free current local database revision. For a saved record, it compares the Vault file with that exact revision when the revision body is readable. It also saves a device-local content hash and byte length. Evidence saved before pruning lets LiveSync recognise an unchanged Vault file even after its recorded revision has been pruned. + +If the recorded body is unavailable and no matching content evidence exists, LiveSync cannot prove that the file is unchanged and preserves it for review. Genuine edits and files with unknown origins remain protected; content evidence does not recreate missing revision history. A conflict which was already recorded is not automatically removed or resolved. Use **Inspect conflicts and file/database differences** to review its current branches. If the problem recurs, include the first relevant log entries, plug-in versions on both devices, and a redacted full report with the issue steps. ## Synchronisation is paused for compatibility review @@ -96,7 +98,9 @@ Current releases automatically align compatible settings which control how new c A missing legacy file-name case setting means case-insensitive handling. It matches an explicit disabled setting and does not require a rebuild for that difference. An explicitly enabled setting can use different document IDs and still requires a compatibility decision against either value. Other configuration differences shown in the dialogue must still be resolved. -If the mismatch names **Encrypt internal file Properties**, update every device before accepting that preference. It affects subsequent Metadata writes for Hidden File Sync and Customisation Sync; it does not automatically protect existing Metadata. A manual remote Rebuild is strongly recommended if you need to protect existing paths, times, sizes, and Chunk references. +The saved **Encrypt internal file Properties** preference is effective only for CouchDB when E2EE V2 and **Property Encryption** are also enabled. An inactive preference difference alone does not change Metadata behaviour or require a Rebuild. Applying settings retains that preference only when doing so leaves the adopted effective encryption settings unchanged. + +When active, this option affects subsequent Metadata writes for Hidden File Sync and Customisation Sync; it does not automatically protect existing Metadata. To protect existing paths, times, sizes, and Chunk references, update every device and manually Rebuild the remote database. The `Sync now` command keeps routine replication progress quiet so that it is convenient to assign to a keyboard shortcut; assign one in Obsidian if that suits your workflow. A quiet command may still open this dialogue when a mismatch or another decision requires your attention. From c0275f8f4b6dd2ab7a547bd0c8eba8e3ad2bd420 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:45:16 +0000 Subject: [PATCH 10/68] Releasing 1.0.38 --- manifest.json | 2 +- package-lock.json | 10 +++++----- package.json | 2 +- src/apps/cli/package.json | 2 +- src/apps/webapp/package.json | 2 +- src/apps/webpeer/package.json | 2 +- updates.md | 4 ++++ versions.json | 3 ++- 8 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest.json b/manifest.json index 7c23c321..f1633e1a 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.37", + "version": "1.0.38", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index e607283a..583f5928 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.37", + "version": "1.0.38", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.37", + "version": "1.0.38", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -11862,7 +11862,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.37-cli", + "version": "1.0.38-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11887,7 +11887,7 @@ }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.37-webapp", + "version": "1.0.38-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11899,7 +11899,7 @@ } }, "src/apps/webpeer": { - "version": "1.0.37-webpeer", + "version": "1.0.38-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, diff --git a/package.json b/package.json index fb7edb0b..4f33396b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.37", + "version": "1.0.38", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index bef046f8..67daece6 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.37-cli", + "version": "1.0.38-cli", "main": "dist/index.cjs", "type": "module", "scripts": { diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index f9e24148..e08ebf91 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.37-webapp", + "version": "1.0.38-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index e6b8dcba..de5b3f32 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.37-webpeer", + "version": "1.0.38-webpeer", "type": "module", "scripts": { "dev": "vite", diff --git a/updates.md b/updates.md index 84b05644..3b4efc24 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,10 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +## 1.0.38 + +17th October, 2026 + ### Synchronisation #### Added diff --git a/versions.json b/versions.json index 57ffb18a..34ccf31b 100644 --- a/versions.json +++ b/versions.json @@ -48,5 +48,6 @@ "1.0.34": "1.7.2", "1.0.35": "1.7.2", "1.0.36": "1.7.2", - "1.0.37": "1.7.2" + "1.0.37": "1.7.2", + "1.0.38": "1.7.2" } From 69105b600a850d14efc991175d3d2c89f8fa4faa Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 14:49:29 +0000 Subject: [PATCH 11/68] Archive the 1.0.33 release notes --- docs/releases/1.0.md | 78 ++++++++++++++++++++++++++++++++++++++++++++ updates.md | 78 -------------------------------------------- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/docs/releases/1.0.md b/docs/releases/1.0.md index 59912591..cc8731f8 100644 --- a/docs/releases/1.0.md +++ b/docs/releases/1.0.md @@ -2,6 +2,84 @@ This document contains earlier published releases from the 1.0 line of the [current Self-hosted LiveSync release history](../../updates.md). Beta and release-candidate builds published before 1.0.0 are recorded in the [1.0 preview history](1.0-previews.md). Earlier release lines continue in the [0.25 history](0.25.md) and the [legacy history](legacy.md). +## 1.0.33 + +1st October, 2026 + +This has turned into quite a substantial release, and I think it brings meaningful improvements to the core of Self-hosted LiveSync. If you notice anything, please feel free to let me know. + +### Privacy and compatibility + +#### New Feature + +- An optional saved ID key can generate encrypted Chunk IDs and obfuscated Metadata document IDs independently of the current E2EE passphrase. + - New Vaults use a random key by default; existing Vaults keep their current ID configuration by default. You can also derive a key from the current E2EE passphrase, enter a separate source, or import a recovery code. The source is not retained; the saved key can be revealed locally as a recovery code. + - The saved key stays in place when the E2EE passphrase changes or E2EE is turned off. Share it with another device through a protected Setup URI. Changing document IDs on an existing remote requires the usual Rebuild and Fetch procedure. +- We can now keep the file properties used by Hidden File Sync and Customisation Sync private in CouchDB. + - **Encrypt internal file Properties** extends E2EE V2 and Property Encryption to their paths, times, sizes, and Chunk references. + - Existing configurations keep this preference disabled. New Vaults enable it for use when the required encryption settings are active. + - Update every synchronising device before enabling it. It protects future writes; a manual remote Rebuild is strongly recommended to protect existing properties. +- We can now see which unsupported feature prevents a client from synchronising with CouchDB. + - Clients check the features required by the remote before transferring data or resetting the local database for Fast Fetch. Receiving an unsupported requirement also stops active replication. + +- We can now compare ID generation performance on a desktop or mobile device through **Open review harness**, available with the developers' debug tools enabled. + - The copied report includes legacy and independent ID timings and, where available, approximate JavaScript heap samples. The measurement uses fixed test data and keeps our Vault and settings unchanged. + +#### Fixed + +- Updated `brace-expansion` and `markdown-it` to include upstream security fixes. +- We can now keep using an E2EE passphrase beginning with `%` after restarting Obsidian. (#1221) + - This plug-in encrypts it before saving the settings. If an earlier version saved it in plain text, re-enter the passphrase used to encrypt the existing data after updating. Treat that passphrase as exposed if the affected `data.json` was shared. +- A receiving device now retries an unavailable CouchDB Chunk when file Metadata arrives before that Chunk is visible, helping rapid edits reach the Vault after an initial on-demand lookup misses it. (#1224) + - Retries start after two seconds and continue with increasing delays while finite replication is active. When it ends, this plug-in checks locally and makes a final lookup if needed, without waiting out the remaining retry delay. +- We can now distinguish initial on-demand Chunk requests (`🛄`) from retries (`🔁`) in the status bar. These replace `🧩`; each pending Chunk appears in one category, including while a retry is waiting. + +### Synchronisation and storage + +#### Fixed + +- Received changes held during start-up or a fetch are applied when this plug-in becomes ready, without waiting for another change or a settings save. **Suspend database reflecting** continues to hold changes (#1200). +- Customisation Sync now compares full millisecond timestamps, so the freshness labels and **Select All Shiny** no longer mistake an older copy for a newer one because of timestamp truncation. (#1194) +- **Hide not applicable items** now hides identical Customisation Sync items and refreshes the list when toggled. Items with applicable differences stay visible. (#1193) + +### Interface and translation + +#### Improved + +- More settings and messages are now available in Russian. (#1187) + - Dialogues show generated QR codes, key pairs, and database sizes again. +- We can now use updated Spanish translations for settings and messages. (#1212) + +### Setup + +#### New Feature + +- We can now share a Setup URI with a displayed time limit, or choose **Compatible** for reuse without a time limit. + - **Time-bound** uses the current fixed seven-day UTC window, so the displayed end may be less than seven days away. Compatible retains the existing URI format; receiving devices still need to support the shared settings. + - The time condition applies when opening the URI. It does not revoke imported credentials or prevent reuse after rolling the device clock back. + +#### Improved + +- We can now distinguish the three Setup URI and QR code choices by their short labels and icons: initialise or overwrite the remote, join this device, or apply settings only. + +#### Fixed + +- We can now add a device or open a copied Vault without a compatibility pause solely because its device-local version record is absent. + - Existing version or settings incompatibilities still require review. A pause already saved by an earlier release still needs one explicit resume action. + +### Acknowledgements + +Thank you for your contributions! + +- @kimjansheden (#1219) +- @Immick (#1195, #1196) +- @bolikcraft (#1187) +- @speedy-axolotl (#1212) + +### Issue replies + +I am a little behind on replying to issues, but I am reading them and will respond as I work through them. I have had little uninterrupted time recently, and that should improve soon. + ## 1.0.32 27th September, 2026 diff --git a/updates.md b/updates.md index 3b4efc24..c8d1371e 100644 --- a/updates.md +++ b/updates.md @@ -134,81 +134,3 @@ Thank you for your contributions! Thank you for your contributions! - @kimjansheden (#1218) - -## 1.0.33 - -1st October, 2026 - -This has turned into quite a substantial release, and I think it brings meaningful improvements to the core of Self-hosted LiveSync. If you notice anything, please feel free to let me know. - -### Privacy and compatibility - -#### New Feature - -- An optional saved ID key can generate encrypted Chunk IDs and obfuscated Metadata document IDs independently of the current E2EE passphrase. - - New Vaults use a random key by default; existing Vaults keep their current ID configuration by default. You can also derive a key from the current E2EE passphrase, enter a separate source, or import a recovery code. The source is not retained; the saved key can be revealed locally as a recovery code. - - The saved key stays in place when the E2EE passphrase changes or E2EE is turned off. Share it with another device through a protected Setup URI. Changing document IDs on an existing remote requires the usual Rebuild and Fetch procedure. -- We can now keep the file properties used by Hidden File Sync and Customisation Sync private in CouchDB. - - **Encrypt internal file Properties** extends E2EE V2 and Property Encryption to their paths, times, sizes, and Chunk references. - - Existing configurations keep this preference disabled. New Vaults enable it for use when the required encryption settings are active. - - Update every synchronising device before enabling it. It protects future writes; a manual remote Rebuild is strongly recommended to protect existing properties. -- We can now see which unsupported feature prevents a client from synchronising with CouchDB. - - Clients check the features required by the remote before transferring data or resetting the local database for Fast Fetch. Receiving an unsupported requirement also stops active replication. - -- We can now compare ID generation performance on a desktop or mobile device through **Open review harness**, available with the developers' debug tools enabled. - - The copied report includes legacy and independent ID timings and, where available, approximate JavaScript heap samples. The measurement uses fixed test data and keeps our Vault and settings unchanged. - -#### Fixed - -- Updated `brace-expansion` and `markdown-it` to include upstream security fixes. -- We can now keep using an E2EE passphrase beginning with `%` after restarting Obsidian. (#1221) - - This plug-in encrypts it before saving the settings. If an earlier version saved it in plain text, re-enter the passphrase used to encrypt the existing data after updating. Treat that passphrase as exposed if the affected `data.json` was shared. -- A receiving device now retries an unavailable CouchDB Chunk when file Metadata arrives before that Chunk is visible, helping rapid edits reach the Vault after an initial on-demand lookup misses it. (#1224) - - Retries start after two seconds and continue with increasing delays while finite replication is active. When it ends, this plug-in checks locally and makes a final lookup if needed, without waiting out the remaining retry delay. -- We can now distinguish initial on-demand Chunk requests (`🛄`) from retries (`🔁`) in the status bar. These replace `🧩`; each pending Chunk appears in one category, including while a retry is waiting. - -### Synchronisation and storage - -#### Fixed - -- Received changes held during start-up or a fetch are applied when this plug-in becomes ready, without waiting for another change or a settings save. **Suspend database reflecting** continues to hold changes (#1200). -- Customisation Sync now compares full millisecond timestamps, so the freshness labels and **Select All Shiny** no longer mistake an older copy for a newer one because of timestamp truncation. (#1194) -- **Hide not applicable items** now hides identical Customisation Sync items and refreshes the list when toggled. Items with applicable differences stay visible. (#1193) - -### Interface and translation - -#### Improved - -- More settings and messages are now available in Russian. (#1187) - - Dialogues show generated QR codes, key pairs, and database sizes again. -- We can now use updated Spanish translations for settings and messages. (#1212) - -### Setup - -#### New Feature - -- We can now share a Setup URI with a displayed time limit, or choose **Compatible** for reuse without a time limit. - - **Time-bound** uses the current fixed seven-day UTC window, so the displayed end may be less than seven days away. Compatible retains the existing URI format; receiving devices still need to support the shared settings. - - The time condition applies when opening the URI. It does not revoke imported credentials or prevent reuse after rolling the device clock back. - -#### Improved - -- We can now distinguish the three Setup URI and QR code choices by their short labels and icons: initialise or overwrite the remote, join this device, or apply settings only. - -#### Fixed - -- We can now add a device or open a copied Vault without a compatibility pause solely because its device-local version record is absent. - - Existing version or settings incompatibilities still require review. A pause already saved by an earlier release still needs one explicit resume action. - -### Acknowledgements - -Thank you for your contributions! - -- @kimjansheden (#1219) -- @Immick (#1195, #1196) -- @bolikcraft (#1187) -- @speedy-axolotl (#1212) - -### Issue replies - -I am a little behind on replying to issues, but I am reading them and will respond as I work through them. I have had little uninterrupted time recently, and that should improve soon. From f0ea5cb3a0a52f4cde7cb4d1ff24d7fa24d56068 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 17 Oct 2026 18:22:45 +0000 Subject: [PATCH 12/68] Update software citation for version 1.0.38 --- CITATION.cff | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 86621d55..3f7067b9 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,9 +9,9 @@ authors: - name: "Self-hosted LiveSync Contributors" repository-code: "https://github.com/vrtmrz/obsidian-livesync" url: "https://github.com/vrtmrz/obsidian-livesync" -version: 1.0.23 -doi: 10.5281/zenodo.22247183 -date-released: "2026-09-05" +version: 1.0.38 +doi: 10.5281/zenodo.23438598 +date-released: "2026-10-17" license: MIT keywords: - obsidian From 30adf842eba0f2e39a9f8a7040d02f2f92ab012a Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 10:16:42 +0000 Subject: [PATCH 13/68] ci: skip test jobs for draft pull requests --- .github/workflows/cli-deno-tests.yml | 3 +++ .github/workflows/cli-docker-validation.yml | 5 ++++- .github/workflows/cli-p2p-tests.yml | 2 ++ .github/workflows/unit-ci.yml | 4 ++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 76abc996..66f13e77 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -22,6 +22,7 @@ on: - 'package.json' - 'package-lock.json' pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths: - '.github/workflows/cli-deno-tests.yml' - 'src/apps/cli/**' @@ -66,6 +67,7 @@ concurrency: jobs: prepare: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -125,6 +127,7 @@ jobs: retention-days: 1 test: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} needs: prepare runs-on: ubuntu-latest timeout-minutes: 60 diff --git a/.github/workflows/cli-docker-validation.yml b/.github/workflows/cli-docker-validation.yml index a993e9ed..5c76d551 100644 --- a/.github/workflows/cli-docker-validation.yml +++ b/.github/workflows/cli-docker-validation.yml @@ -2,6 +2,7 @@ name: CLI Docker validation on: pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths: - '.github/workflows/cli-docker.yml' - '.github/workflows/cli-docker-validation.yml' @@ -40,6 +41,7 @@ concurrency: jobs: bundle: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} name: Build shared Docker CLI bundle runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -72,6 +74,7 @@ jobs: retention-days: 1 validate: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} name: Docker E2E (${{ matrix.arch }}) needs: bundle runs-on: ${{ matrix.runner }} @@ -171,7 +174,7 @@ jobs: summary: name: Collect both Docker E2E results needs: validate - if: always() + if: ${{ always() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: diff --git a/.github/workflows/cli-p2p-tests.yml b/.github/workflows/cli-p2p-tests.yml index 17ffa039..478afad4 100644 --- a/.github/workflows/cli-p2p-tests.yml +++ b/.github/workflows/cli-p2p-tests.yml @@ -24,6 +24,7 @@ on: - '.dockerignore' - 'test/bench-network/**' pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths: - '.github/workflows/cli-p2p-tests.yml' - 'src/apps/cli/**' @@ -53,6 +54,7 @@ concurrency: jobs: compose-p2p-e2e: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} runs-on: ubuntu-latest timeout-minutes: 45 steps: diff --git a/.github/workflows/unit-ci.yml b/.github/workflows/unit-ci.yml index a3b94cfe..9df08f2d 100644 --- a/.github/workflows/unit-ci.yml +++ b/.github/workflows/unit-ci.yml @@ -44,6 +44,7 @@ on: - '.github/workflows/cli-docker.yml' - '.github/workflows/unit-ci.yml' pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths: - 'manifest.json' - 'updates.md' @@ -86,6 +87,7 @@ permissions: jobs: setup-tools: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} name: Self-hosted setup tools runs-on: ubuntu-latest timeout-minutes: 15 @@ -134,6 +136,7 @@ jobs: run: npm run test:docker-couchdb:stop || true unit-test: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} name: Unit Tests runs-on: ubuntu-latest timeout-minutes: 30 @@ -173,6 +176,7 @@ jobs: path: coverage/** integration-test: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} name: Integration Tests runs-on: ubuntu-latest timeout-minutes: 30 From 612e3eb9d5b22c7d60d142759c2ed223eaae8967 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 10:24:42 +0000 Subject: [PATCH 14/68] Document and test NFC reconstruction with logical deletions --- docs/design_docs/unicode_path_identity.md | 19 +++++-- docs/recovery.md | 4 ++ docs/settings.md | 2 +- test/e2e-obsidian/README.md | 2 +- .../scripts/cli-to-obsidian-sync.ts | 57 +++++++++++++++++-- updates.md | 7 +++ 6 files changed, 81 insertions(+), 10 deletions(-) diff --git a/docs/design_docs/unicode_path_identity.md b/docs/design_docs/unicode_path_identity.md index 4a9f4210..d88e1277 100644 --- a/docs/design_docs/unicode_path_identity.md +++ b/docs/design_docs/unicode_path_identity.md @@ -1,7 +1,7 @@ --- -date: 2026-10-17 +date: 2026-10-19 commonlib-version: "0.1.42" -self-hosted-livesync-version: "1.0.37" +self-hosted-livesync-version: "1.0.38" status: unreleased --- @@ -16,8 +16,9 @@ the [Unicode path identity ADR](../adr/2026_10_unicode_path_identity.md). Commonlib's [consumer contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md) defines the shared package boundary. -This records an unreleased change after Self-hosted LiveSync `1.0.37`, using the -published Commonlib `0.1.42` package as an exact dependency. +NFC identity shipped in Self-hosted LiveSync `1.0.38` with Commonlib `0.1.42`. +The deletion handling in reconstruction below is an unreleased refinement of +that baseline. ## Scope @@ -117,6 +118,16 @@ must resolve the ambiguous names or exclude one file from the synchronisation scope, then retry the reconstruction. This check is bounded by the host's listing contract and does not imply that every raw physical alias is visible. +For full Rebuild, Standard Fetch, and Fast Fetch, which replace the local +database, the Metadata enumeration requests conflict information. It excludes +`deleted: true` documents only when they have no unresolved conflict leaves. +A deleted winner with any unresolved conflict remains a collision candidate, +including when the other leaves are also logically deleted. Remote-only +Rebuild retains logically deleted Metadata and requires stored IDs to match +the selected identity. Fully tombstoned documents are already absent from +ordinary enumeration; a still non-deleted local copy remains subject to the +check until a remote deletion has been replicated. + ## Reconstruction and client operations Mode changes are never applied by an ordinary Metadata write. In Obsidian, the diff --git a/docs/recovery.md b/docs/recovery.md index de9c5c17..b73cf9fd 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -30,6 +30,10 @@ This migration reconstructs the databases from one authoritative Vault. It does 4. On each remaining client, select **Unicode NFC path identity** (`pathNormalizationVersion: 1`) before fetching. In Obsidian, choose **Reset Synchronisation on This Device**. In the CLI, set the same value in its settings and run `fetch` with `--force --write-settings`. 5. Confirm that every client uses version `1` and that the expected files are present before resuming synchronisation. Do not reconnect a version `0` client to the migrated database. +Rebuild and both Fetch paths ignore local logical deletions without unresolved conflicts when replacing the local database. Deletion or rename history alone therefore does not block this migration. Fully tombstoned documents are already absent from the Metadata check. Unresolved conflicts and ambiguous current files still require review, even when a conflict's winning revision is logically deleted. Remote-only Rebuild retains its existing ID checks and is not a substitute for the source Rebuild above. + +A remote deletion which has not reached this device does not establish that its local Metadata is deleted. Inspect the local state and preserve unsynchronised content before resolving a reported collision. Manual deletion of every database is not a prerequisite for this procedure. Clients already using NFC identity do not need to repeat the migration for this change. + The Inspector's **Repair this Metadata document ID** action repairs one local entry; it is not a bulk migration tool. Use the Rebuild and Fetch procedure above for path identity changes. This procedure does not establish native macOS or iOS filesystem behaviour. Obsidian's collision checks use the paths exposed by its Vault APIs; see [Unicode path identity](settings.md#unicode-path-identity) for that visibility limit. diff --git a/docs/settings.md b/docs/settings.md index ab3fc26e..c742b975 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -1062,7 +1062,7 @@ Version `1` requires a coordinated migration: set it on the source client before The selected value stays pending through ordinary settings saves and reloads. It becomes active only when the Rebuild or Fetch operation is successfully reserved; cancellation leaves it pending. -Distinct files or Metadata documents which derive the same identity stop reconstruction before either database is reset. This check uses the host's supported file listing; Obsidian checks the paths exposed by its Vault APIs and cannot detect physical aliases hidden by its index. Resolve the duplicate names or synchronisation scope first; Self-hosted LiveSync does not merge them automatically. Case-insensitive handling also applies the existing letter-case rule. +Distinct files or retained Metadata documents which derive the same identity stop reconstruction before either database is reset. When replacing the local database, Rebuild and Fetch exclude logical deletions without unresolved conflicts; a deleted winner with unresolved conflicts still participates in the check. This check uses the host's supported file listing; Obsidian checks the paths exposed by its Vault APIs and cannot detect physical aliases hidden by its index. Resolve the duplicate names or synchronisation scope first; Self-hosted LiveSync does not merge them automatically. Case-insensitive handling also applies the existing letter-case rule. ### 4. Compatibility (Internal API Usage) diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index aa3afb23..c846a364 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -172,7 +172,7 @@ Set `E2E_OBSIDIAN_ONLY_DIFFERENT_CHUNK_ID_KEYS=true` to run the focused case whe Set `E2E_OBSIDIAN_UNICODE_PATHS=true` to exercise path identity version 1 on Linux. This variant uses an NFD directory and a mixed NFC/NFD file name, checks reflection at Obsidian's NFC path, and returns an update and deletion to the original CLI physical path. Updates use `sync` and `mirror`; deletions use the polling daemon, because `mirror` retains tombstoned local files by default. It uses unencrypted Metadata by default to match issue #1259; add `E2E_OBSIDIAN_UNICODE_ENCRYPTION=true` for the encrypted variant. These checks do not establish native macOS or iOS filesystem behaviour. -Set `E2E_OBSIDIAN_UNICODE_MIGRATION=true` to begin with two CLI databases using legacy path identity. The source runs `rebuild --force --write-settings` after selecting version 1, and the second client runs `fetch --force --write-settings`. The scenario checks that the old document ID is replaced, the NFC feature is declared, and both physical NFD filenames are preserved before continuing the real Obsidian update and deletion checks. +Set `E2E_OBSIDIAN_UNICODE_MIGRATION=true` to begin with two CLI databases using legacy path identity and an obsolete, logically deleted NFC-equivalent Metadata document. The source runs `rebuild --force --write-settings` after selecting version 1, and the second client runs `fetch --force --write-settings`. The scenario checks that the old document ID is replaced, the NFC feature is declared, and both physical NFD filenames are preserved. In real Obsidian it also seeds an unconflicted logical deletion under the obsolete ID, runs Streaming Fast Fetch, and verifies replacement of that local Metadata with unchanged file content before continuing the update and deletion checks. The CouchDB fixture must allow browser CORS requests; this case rejects the native Request API fallback. The isolated Obsidian session starts with its CouchDB settings and device-local compatibility acknowledgement already in place. This keeps the scenario focused on cross-runtime data compatibility; unconfigured start-up and visible CouchDB onboarding are covered by their dedicated workflows. diff --git a/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts b/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts index 96267663..d71637a6 100644 --- a/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts +++ b/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts @@ -58,6 +58,7 @@ type CliFileInfo = { id: string; children: string[]; expectedCtime?: number; + legacyId?: string; }; function parseCommandLine(value: string): string[] { @@ -130,14 +131,15 @@ function resolveLiveSyncCliCommand(): LiveSyncCliCommand { return { executable: process.execPath, prefixArgs: [liveSyncCli] }; } -async function runLiveSyncCli(command: LiveSyncCliCommand, args: string[]): Promise { +async function runLiveSyncCli(command: LiveSyncCliCommand, args: string[], input?: string): Promise { return await new Promise((resolvePromise, reject) => { const timeoutMs = Number(process.env.E2E_LIVESYNC_CLI_TIMEOUT_MS ?? 60000); const child = spawn(command.executable, [...command.prefixArgs, ...args], { cwd: process.cwd(), env: process.env, - stdio: ["ignore", "pipe", "pipe"], + stdio: ["pipe", "pipe", "pipe"], }); + child.stdin.end(input); let stdout = ""; let stderr = ""; child.stdout.on("data", (chunk: Buffer) => { @@ -219,6 +221,7 @@ async function configureLiveSyncCli( passphrase: e2eePassphrase, usePathObfuscation: useEncryption, encryptInternalMetadata: useEncryption, + deleteMetadataOfDeletedFiles: false, pathNormalizationVersion, doctorProcessedVersion: "0.25.27", isConfigured: true, @@ -339,6 +342,27 @@ async function migrateCliDatabases( dbName: string, legacyFileInfo: CliFileInfo ): Promise { + await runLiveSyncCli( + command, + [databasePath, "--settings", settingsPath, "put", receivedNotePath], + "Obsolete composed-path content.\n" + ); + const aliasInfo = await runLiveSyncCli(command, [ + databasePath, + "--settings", + settingsPath, + "info", + receivedNotePath, + ]); + const deletedAliasId = (JSON.parse(aliasInfo.stdout) as CliFileInfo).id; + if (deletedAliasId === legacyFileInfo.id) + throw new Error("The legacy Unicode fixture needs distinct Metadata IDs."); + await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "rm", receivedNotePath]); + await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "sync"]); + await waitForCouchDbDocs(couchDb, dbName, (docs) => + docs.some((doc) => doc._id === deletedAliasId && doc.deleted === true) + ); + const secondaryDatabasePath = join(cliState, "secondary-database"); const secondarySettingsPath = join(cliState, "secondary-settings.json"); await mkdir(secondaryDatabasePath, { recursive: true }); @@ -406,8 +430,8 @@ async function migrateCliDatabases( } ); } - console.log("CLI Rebuild and Fetch migrated legacy document IDs while preserving both NFD physical files."); - return { ...fileInfo, expectedCtime: legacyFileInfo.expectedCtime }; + console.log("CLI Rebuild and Fetch passed with obsolete logical deletions and preserved both NFD physical files."); + return { ...fileInfo, expectedCtime: legacyFileInfo.expectedCtime, legacyId: legacyFileInfo.id }; } async function main(): Promise { @@ -509,6 +533,31 @@ async function main(): Promise { const received = await waitForVaultContent(vault.path, receivedNotePath); assertEqual(received, editedNoteContent, "The real Obsidian plug-in did not materialise the CLI-edited note."); + if (cliFileInfo.legacyId) { + const fetched = await evalObsidianJson<{ obsoleteRetained: boolean; content: string }>( + obsidianCli.binary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "if(core.settings.useRequestAPI) throw new Error('This case requires Streaming Fast Fetch.');", + "await core.services.setting.applyPartial({suspendParseReplicationResult:true,suspendFileWatching:true});", + `const legacyId=${JSON.stringify(cliFileInfo.legacyId)};`, + `const {_rev,...current}=await core.localDatabase.localDatabase.get(${JSON.stringify(cliFileInfo.id)});`, + `await core.localDatabase.localDatabase.put({...current,_id:legacyId,path:${JSON.stringify(notePath)},deleted:true});`, + "await core.serviceModules.rebuilder.$fetchLocalDBFast(true);", + "const rows=await core.localDatabase.localDatabase.allDocs();", + "return JSON.stringify({", + "obsoleteRetained:rows.rows.some(row=>row.id===legacyId),", + `content:await app.vault.adapter.read(${JSON.stringify(receivedNotePath)}),`, + "});", + "})()", + ].join(""), + session.cliEnv + ); + assertEqual(fetched.obsoleteRetained, false, "Fast Fetch retained obsolete local Metadata."); + assertEqual(fetched.content, editedNoteContent, "Fast Fetch changed the synchronised file content."); + console.log("Real Obsidian Fast Fetch replaced obsolete logical deletions and retained the expected file."); + } if (cliFileInfo.expectedCtime !== undefined) { const receivedCtime = await evalObsidianJson( obsidianCli.binary, diff --git a/updates.md b/updates.md index c8d1371e..436a4a56 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We can migrate to NFC path identity when deletion or rename history would otherwise block Rebuild or Fetch. (#1259) + - Logical deletions without unresolved conflicts no longer block operations which replace the local database. Conflicting histories and ambiguous current files still require review. + ## 1.0.38 17th October, 2026 From 63688db75421f477c4e7b9856ffddb730cd5ef66 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 11:22:42 +0000 Subject: [PATCH 15/68] Verify current Metadata after Fast Fetch --- .../scripts/cli-to-obsidian-sync.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts b/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts index d71637a6..d517c69e 100644 --- a/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts +++ b/test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts @@ -534,7 +534,13 @@ async function main(): Promise { const received = await waitForVaultContent(vault.path, receivedNotePath); assertEqual(received, editedNoteContent, "The real Obsidian plug-in did not materialise the CLI-edited note."); if (cliFileInfo.legacyId) { - const fetched = await evalObsidianJson<{ obsoleteRetained: boolean; content: string }>( + const fetched = await evalObsidianJson<{ + obsoleteRetained: boolean; + currentDeleted: boolean; + currentPath: string; + previousPath: string; + content: string; + }>( obsidianCli.binary, [ "(async()=>{", @@ -545,9 +551,13 @@ async function main(): Promise { `const {_rev,...current}=await core.localDatabase.localDatabase.get(${JSON.stringify(cliFileInfo.id)});`, `await core.localDatabase.localDatabase.put({...current,_id:legacyId,path:${JSON.stringify(notePath)},deleted:true});`, "await core.serviceModules.rebuilder.$fetchLocalDBFast(true);", + `const restored=await core.localDatabase.localDatabase.get(${JSON.stringify(cliFileInfo.id)});`, "const rows=await core.localDatabase.localDatabase.allDocs();", "return JSON.stringify({", "obsoleteRetained:rows.rows.some(row=>row.id===legacyId),", + "currentDeleted:restored.deleted===true,", + "currentPath:restored.path,", + "previousPath:current.path,", `content:await app.vault.adapter.read(${JSON.stringify(receivedNotePath)}),`, "});", "})()", @@ -555,8 +565,12 @@ async function main(): Promise { session.cliEnv ); assertEqual(fetched.obsoleteRetained, false, "Fast Fetch retained obsolete local Metadata."); + assertEqual(fetched.currentDeleted, false, "Fast Fetch returned logically deleted current Metadata."); + assertEqual(fetched.currentPath, fetched.previousPath, "Fast Fetch changed the current Metadata path."); assertEqual(fetched.content, editedNoteContent, "Fast Fetch changed the synchronised file content."); - console.log("Real Obsidian Fast Fetch replaced obsolete logical deletions and retained the expected file."); + console.log( + "Real Obsidian Fast Fetch replaced obsolete logical deletions and retained current Metadata and file content." + ); } if (cliFileInfo.expectedCtime !== undefined) { const receivedCtime = await evalObsidianJson( From 8aeb7c60eff7742d0cb4ac1f2cce3103e2e78733 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 12:49:02 +0000 Subject: [PATCH 16/68] Add read-only database diagnostics --- docs/troubleshooting.md | 6 + package-lock.json | 8 +- package.json | 3 +- src/common/events.ts | 2 + .../messages/LiveSyncProvisionalMessages.ts | 41 +++ .../CmdLocalDatabaseMainte.ts | 228 +---------------- .../CmdLocalDatabaseMainte.unit.spec.ts | 9 +- src/main.ts | 2 + .../features/SettingDialogue/PaneHatch.ts | 9 + .../dialogues/DatabaseDiagnosticDialog.svelte | 148 +++++++++++ .../databaseDiagnosticDialogTypes.ts | 9 + src/serviceFeatures/databaseDiagnostics.ts | 163 ++++++++++++ .../databaseDiagnostics.unit.spec.ts | 213 ++++++++++++++++ src/serviceFeatures/useDatabaseDiagnostics.ts | 76 ++++++ .../useDatabaseDiagnostics.unit.spec.ts | 122 +++++++++ test/e2e-obsidian/README.md | 3 + .../scripts/database-diagnostics.ts | 237 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + updates.md | 5 + 19 files changed, 1049 insertions(+), 236 deletions(-) create mode 100644 src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte create mode 100644 src/serviceFeatures/databaseDiagnosticDialogTypes.ts create mode 100644 src/serviceFeatures/databaseDiagnostics.ts create mode 100644 src/serviceFeatures/databaseDiagnostics.unit.spec.ts create mode 100644 src/serviceFeatures/useDatabaseDiagnostics.ts create mode 100644 src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/database-diagnostics.ts diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b4a2decc..5bdfe39d 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -163,6 +163,12 @@ Enable Obsidian's `Detect all file extensions`, then check LiveSync selectors, i ## Collect a report +For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. The inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete. + +The result dialogue provides Markdown and JSON reports with the same name policy. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. They are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled. + +**Analyse database usage** offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads. + Follow [How to report an issue](to_issue_reporting.md) for the complete reporting checklist, including Obsidian debug information and the privacy review required before sharing evidence. Run `Generate full report for opening the issue with debug info` to copy the current settings summary and recent verbose log lines. Remove credentials, remote URLs, Vault names, file contents, and other private information before sharing it. diff --git a/package-lock.json b/package-lock.json index 583f5928..fcfe4617 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.42", + "@vrtmrz/livesync-commonlib": "0.1.43", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4389,9 +4389,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.42", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.42.tgz", - "integrity": "sha512-LyWl0Mkrj9UsQXV0nOUaYJybcN3wLkpAAxCeHMZYOEizaUTQ7ezN+FgzN+OQxDRA8UXWnoatxh719NWzOwBpBw==", + "version": "0.1.43", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.43.tgz", + "integrity": "sha512-WM2k545hT0ctLv64WdKZ0PkHwyw9Fnlf9xssH7Uv9C0HEMm8CDtj6dfJsPOHs0+Z41Kfco1U/y6R5f5+xWZ00g==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 4f33396b..3103193f 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts", "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", + "test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", "test:e2e:obsidian:revision-repair": "tsx test/e2e-obsidian/scripts/revision-repair.ts", "test:e2e:obsidian:document-history-nav": "tsx test/e2e-obsidian/scripts/document-history-nav.ts", @@ -191,7 +192,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.42", + "@vrtmrz/livesync-commonlib": "0.1.43", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/src/common/events.ts b/src/common/events.ts index 911cc2ca..d358f997 100644 --- a/src/common/events.ts +++ b/src/common/events.ts @@ -19,6 +19,7 @@ export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor"; export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete"; export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage"; +export const EVENT_INSPECT_DB_INTEGRITY = "inspect-db-integrity"; export const EVENT_REQUEST_PERFORM_GC_V3 = "request-perform-gc-v3"; // export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size"; // export const EVENT_FILE_CHANGED = "file-changed"; @@ -37,6 +38,7 @@ declare global { [EVENT_REQUEST_RUN_DOCTOR]: string; [EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined; [EVENT_ANALYSE_DB_USAGE]: undefined; + [EVENT_INSPECT_DB_INTEGRITY]: undefined; [EVENT_REQUEST_PERFORM_GC_V3]: undefined; } } diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index e4e60160..073cbac7 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -251,6 +251,47 @@ export const liveSyncProvisionalEnglishMessages = { "An ID source is required to enable this option.": "An ID source is required to enable this option.", "The ID source or recovery code is invalid. Check it and try again.": "The ID source or recovery code is invalid. Check it and try again.", + "Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.": + "Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.", + Inspect: "Inspect", + "Inspect database integrity": "Inspect database integrity", + "Inspection completed. Review the findings and coverage before sharing.": + "Inspection completed. Review the findings and coverage before sharing.", + "Inspection partially completed. Some information could not be checked; review the reported reasons.": + "Inspection partially completed. Some information could not be checked; review the reported reasons.", + "Inspection failed. The report records the available evidence and reasons.": + "Inspection failed. The report records the available evidence and reasons.", + "Inspection cancelled. The report contains only the evidence collected so far.": + "Inspection cancelled. The report contains only the evidence collected so far.", + "Report copied.": "Report copied.", + "Copy failed. Select the report text and copy it manually.": + "Copy failed. Select the report text and copy it manually.", + "Download requested. You can also copy the report if downloads are unavailable on this device.": + "Download requested. You can also copy the report if downloads are unavailable on this device.", + "Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.": + "Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.", + "Read available local revisions and report their Chunk references. Settings are left unchanged.": + "Read available local revisions and report their Chunk references. Settings are left unchanged.", + "Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.": + "Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.", + "Names in this report": "Names in this report", + "Pseudonyms (recommended)": "Pseudonyms (recommended)", + "Original names": "Original names", + "Cancelling inspectionâ€Ļ": "Cancelling inspectionâ€Ļ", + "Items inspected": "Items inspected", + "Cancel inspection": "Cancel inspection", + "This report uses temporary pseudonyms. Counts and relationships remain visible.": + "This report uses temporary pseudonyms. Counts and relationships remain visible.", + "This report contains original paths and identifiers. Review them before sharing.": + "This report contains original paths and identifiers. Review them before sharing.", + "Partial Usage results are available as JSON so their coverage and reasons stay with the rows.": + "Partial Usage results are available as JSON so their coverage and reasons stay with the rows.", + "Report format": "Report format", + "Diagnostic report": "Diagnostic report", + "Copy report": "Copy report", + "Save report": "Save report", + "The inspection could not finish. Close this dialogue and try again after database startup.": + "The inspection could not finish. Close this dialogue and try again after database startup.", } as const; export type LiveSyncProvisionalMessageKey = keyof typeof liveSyncProvisionalEnglishMessages; diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts index 85b15d90..955813f3 100644 --- a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts @@ -8,14 +8,13 @@ import { type DocumentID, type EntryDoc, type EntryLeaf, - type FilePathWithPrefix, type MetaEntry, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { getNoFromRev } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; import { LiveSyncCommands } from "@/features/LiveSyncCommands"; import { serialized } from "octagonal-wheels/concurrency/lock_v2"; import { arrayToChunkedArray } from "octagonal-wheels/collection"; -import { EVENT_ANALYSE_DB_USAGE, EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events"; +import { EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events"; import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; import type { ReplicatorInstance } from "@vrtmrz/livesync-commonlib/replication"; import { delay } from "@vrtmrz/livesync-commonlib/compat/common/utils"; @@ -61,18 +60,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands { } onload(): void | Promise { // NO OP. - this.services.API.addCommand({ - id: "analyse-database", - name: "Analyse Database Usage (advanced)", - icon: "database-search", - checkCallback: (checking) => { - if (!this.settings.useAdvancedMode || !this._isDatabaseReady()) return false; - if (!checking) { - void this.analyseDatabase(); - } - return true; - }, - }); this.services.API.addCommand({ id: "gc-v3", name: "Garbage Collection V3 (advanced, beta)", @@ -88,7 +75,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands { return true; }, }); - eventHub.onEvent(EVENT_ANALYSE_DB_USAGE, () => this.analyseDatabase()); eventHub.onEvent(EVENT_REQUEST_PERFORM_GC_V3, () => this.gcv3()); } async allChunks(includeDeleted: boolean = false) { @@ -550,218 +536,6 @@ Success: ${successCount}, Errored: ${errored}`; await kvDB.set(DB_KEY_CHUNK_SET, chunkSet); } - // Analyse the database and report chunk usage. - async analyseDatabase() { - if (!(await this.ensureAvailable("Analyse Database Usage"))) return; - const db = this.localDatabase.localDatabase; - // Map of chunk ID to its info - type ChunkInfo = { - id: DocumentID; - refCount: number; - length: number; - }; - const chunkMap = new Map>(); - // Map of document ID to its info - type DocumentInfo = { - id: DocumentID; - rev: Rev; - chunks: Set; - uniqueChunks: Set; - sharedChunks: Set; - path: FilePathWithPrefix; - }; - const docMap = new Map>(); - const info = await db.info(); - // Total number of revisions to process (approximate) - const maxSeq = Number.parseInt(`${info.update_seq ?? 0}`, 10); - let processed = 0; - let read = 0; - let errored = 0; - // Fetch Tasks - const ft = [] as ReturnType[]; - // Fetch a specific revision of a document and make note of its chunks, or add chunk info. - const fetchRevision = async (id: DocumentID, rev: Rev, seq: string | number) => { - try { - processed++; - const doc = await db.get(id, { rev: rev }); - if (doc) { - if ("children" in doc) { - const id = doc._id; - const rev = doc._rev; - const children = (doc.children || []) as DocumentID[]; - const set = docMap.get(id) || new Set(); - set.add({ - id, - rev, - chunks: new Set(children), - uniqueChunks: new Set(), - sharedChunks: new Set(), - path: doc.path, - }); - docMap.set(id, set); - } else if (doc.type === EntryTypes.CHUNK) { - const id = doc._id; - if (chunkMap.has(id)) { - return; - } - if (doc._deleted) { - // Deleted chunk, skip (possibly resurrected later) - return; - } - const length = doc.data.length; - const set = chunkMap.get(id) || new Set(); - set.add({ id, length, refCount: 0 }); - chunkMap.set(id, set); - } - read++; - } else { - this._log(`Analysing Database: not found: ${id} / ${rev}`); - errored++; - } - } catch (error) { - this._log(`Error fetching document ${id} / ${rev}: $`, LOG_LEVEL_NOTICE); - this._log(error, LOG_LEVEL_VERBOSE); - errored++; - } - if (processed % 100 == 0) { - this._log(`Analysing database: ${read} (${errored}) / ${maxSeq} `, LOG_LEVEL_NOTICE, "db-analyse"); - } - }; - - // Enumerate all documents and their revisions. - const IDs = this.localDatabase.findEntryNames("", "", {}); - for await (const id of IDs) { - const revList = await this.localDatabase.getRaw(id as DocumentID, { - revs: true, - revs_info: true, - conflicts: true, - }); - const revInfos = revList._revs_info || []; - for (const revInfo of revInfos) { - // All available revisions should be processed. - // If the revision is not available, it means the revision is already tombstoned. - if (revInfo.status == "available") { - // Schedule fetch task - ft.push(fetchRevision(id as DocumentID, revInfo.rev, 0)); - } - } - } - // Wait for all fetch tasks to complete. - await Promise.all(ft); - // Reference count marking and unique/shared chunk classification. - for (const [, docRevs] of docMap) { - for (const docRev of docRevs) { - for (const chunkId of docRev.chunks) { - const chunkInfos = chunkMap.get(chunkId); - if (chunkInfos) { - for (const chunkInfo of chunkInfos) { - if (chunkInfo.refCount === 0) { - docRev.uniqueChunks.add(chunkId); - } else { - docRev.sharedChunks.add(chunkId); - } - chunkInfo.refCount++; - } - } - } - } - } - // Prepare results - const result = []; - // Calculate total size of chunks in the given set. - const getTotalSize = (ids: Set) => { - return [...ids].reduce((acc, chunkId) => { - const chunkInfos = chunkMap.get(chunkId); - if (chunkInfos) { - for (const chunkInfo of chunkInfos) { - acc += chunkInfo.length; - } - } - return acc; - }, 0); - }; - - // Compile results for each document revision - for (const doc of docMap.values()) { - for (const rev of doc) { - const title = `${rev.path} (${rev.rev})`; - const id = rev.id; - const revStr = `${getNoFromRev(rev.rev)}`; - const revHash = rev.rev.split("-")[1].substring(0, 6); - const path = rev.path; - const uniqueChunkCount = rev.uniqueChunks.size; - const sharedChunkCount = rev.sharedChunks.size; - const uniqueChunkSize = getTotalSize(rev.uniqueChunks); - const sharedChunkSize = getTotalSize(rev.sharedChunks); - result.push({ - title, - path, - rev: revStr, - revHash, - id, - uniqueChunkCount: uniqueChunkCount, - sharedChunkCount, - uniqueChunkSize: uniqueChunkSize, - sharedChunkSize: sharedChunkSize, - }); - } - } - - const titleMap = { - title: "Title", - id: "Document ID", - path: "Path", - rev: "Revision No", - revHash: "Revision Hash", - uniqueChunkCount: "Unique Chunk Count", - sharedChunkCount: "Shared Chunk Count", - uniqueChunkSize: "Unique Chunk Size", - sharedChunkSize: "Shared Chunk Size", - } as const; - // Enumerate orphan chunks (not referenced by any document) - const orphanChunks = [...chunkMap.entries()].filter(([chunkId, infos]) => { - const totalRefCount = [...infos].reduce((acc, info) => acc + info.refCount, 0); - return totalRefCount === 0; - }); - const orphanChunkSize = orphanChunks.reduce((acc, [chunkId, infos]) => { - for (const info of infos) { - acc += info.length; - } - return acc; - }, 0); - result.push({ - title: "__orphan", - id: "__orphan", - path: "__orphan", - rev: "1", - revHash: "xxxxx", - uniqueChunkCount: orphanChunks.length, - sharedChunkCount: 0, - uniqueChunkSize: orphanChunkSize, - sharedChunkSize: 0, - } as const); - - const csvSrc = result.map((e) => { - return [ - `${e.title.replace(/"/g, '""')}"`, - `${e.id}`, - `${e.path}`, - `${e.rev}`, - `${e.revHash}`, - `${e.uniqueChunkCount}`, - `${e.sharedChunkCount}`, - `${e.uniqueChunkSize}`, - `${e.sharedChunkSize}`, - ].join("\t"); - }); - // Add title row - csvSrc.unshift(Object.values(titleMap).join("\t")); - const csv = csvSrc.join("\n"); - - // Prompt to copy to clipboard - await this.services.UI.promptCopyToClipboard("Database Analysis data (TSV):", csv); - } - async compactDatabase() { const replicator = this.core.replicator; if (!canCompactCouchDBRemote(replicator)) return; diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts index 0d337484..89796f4f 100644 --- a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts @@ -42,6 +42,7 @@ import { } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { LocalDatabaseMaintenance } from "./CmdLocalDatabaseMainte"; import { ensureLocalDatabaseMaintenancePrerequisites } from "./maintenancePrerequisites"; +import { eventHub, EVENT_REQUEST_PERFORM_GC_V3 } from "@/common/events"; function createPrerequisites(settingsOverride: Partial = {}) { const askSelectStringDialogue = vi.fn( @@ -63,7 +64,7 @@ function createPrerequisites(settingsOverride: Partial } describe("LocalDatabaseMaintenance prerequisites", () => { - it("shows database analysis in Advanced mode and Garbage Collection only in applicable Edge Case mode", () => { + it("shows Garbage Collection only in applicable Edge Case mode", () => { const commands: Array<{ id: string; checkCallback?: (checking: boolean) => boolean | void; @@ -90,15 +91,15 @@ describe("LocalDatabaseMaintenance prerequisites", () => { _isDatabaseReady: vi.fn(() => true), }); + vi.mocked(eventHub.onEvent).mockClear(); maintenance.onload(); + expect(eventHub.onEvent).toHaveBeenCalledTimes(1); + expect(eventHub.onEvent).toHaveBeenCalledWith(EVENT_REQUEST_PERFORM_GC_V3, expect.any(Function)); - const analyse = commands.find(({ id }) => id === "analyse-database"); const garbageCollect = commands.find(({ id }) => id === "gc-v3"); - expect(analyse?.checkCallback?.(true)).toBe(false); expect(garbageCollect?.checkCallback?.(true)).toBe(false); settings.useAdvancedMode = true; - expect(analyse?.checkCallback?.(true)).toBe(true); expect(garbageCollect?.checkCallback?.(true)).toBe(false); settings.useEdgeCaseMode = true; diff --git a/src/main.ts b/src/main.ts index dd58dcc1..4dcec641 100644 --- a/src/main.ts +++ b/src/main.ts @@ -51,6 +51,7 @@ import { useInteractiveConflictResolutionFeature } from "./serviceFeatures/inter import { ConflictResolveModal } from "./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts"; import { useObsidianReplicationRibbonFeature } from "./serviceFeatures/obsidianReplicationRibbon.ts"; import { useStartupLifecycleFeature } from "./serviceFeatures/startupLifecycle"; +import { useDatabaseDiagnosticsFeature } from "./serviceFeatures/useDatabaseDiagnostics"; export type LiveSyncCore = LiveSyncBaseCore; export default class ObsidianLiveSyncPlugin extends Plugin { core: LiveSyncCore; @@ -195,6 +196,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { useP2PReplicatorCommands(core, replicator); useP2PReplicatorUI(core, core, replicator, createInteractiveP2PReplication(replicator)); useObsidianReplicationRibbonFeature(core); + useDatabaseDiagnosticsFeature(core); useRemoteConfiguration(core); useSetupProtocolFeature(core, setupManager); diff --git a/src/modules/features/SettingDialogue/PaneHatch.ts b/src/modules/features/SettingDialogue/PaneHatch.ts index d0653ee5..711556ff 100644 --- a/src/modules/features/SettingDialogue/PaneHatch.ts +++ b/src/modules/features/SettingDialogue/PaneHatch.ts @@ -16,6 +16,7 @@ import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; import { EVENT_ANALYSE_DB_USAGE, + EVENT_INSPECT_DB_INTEGRITY, EVENT_REQUEST_CHECK_REMOTE_SIZE, EVENT_REQUEST_RUN_DOCTOR, EVENT_REQUEST_RUN_FIX_INCOMPLETE, @@ -113,6 +114,14 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, await chooseAndCopyFileDatabaseInfo(this.core); }) ); + new Setting(paneEl) + .setName($msg("Inspect database integrity")) + .setDesc( + $msg("Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.") + ) + .addButton((button) => + button.setButtonText($msg("Inspect")).onClick(() => eventHub.emitEvent(EVENT_INSPECT_DB_INTEGRITY)) + ); new Setting(paneEl) .setName($msg("Analyse database usage")) .setDesc( diff --git a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte new file mode 100644 index 00000000..5403d1c9 --- /dev/null +++ b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte @@ -0,0 +1,148 @@ + + + +
+ {#if phase === "choice"} +

{translate("Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.")}

+

+ {options?.kind === "usage" + ? translate("Read available local revisions and report their Chunk references. Settings are left unchanged.") + : translate("Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.")} +

+ +
+ + +
+ {:else if phase === "running"} +

+ {cancelling ? translate("Cancelling inspectionâ€Ļ") : `${translate("Items inspected")}: ${processed}`} +

+ + {:else if phase === "result" && output} +

{statusText}

+

+ {output.nameMode === "pseudonym" + ? translate("This report uses temporary pseudonyms. Counts and relationships remain visible.") + : translate("This report contains original paths and identifiers. Review them before sharing.")} +

+ {#if options?.kind === "usage" && output.status !== "complete"} +

{translate("Partial Usage results are available as JSON so their coverage and reasons stay with the rows.")}

+ {/if} + + +

{actionStatus}

+
+ + + +
+ {:else if phase === "failed"} +

{translate("The inspection could not finish. Close this dialogue and try again after database startup.")}

+ + {/if} +
+ + diff --git a/src/serviceFeatures/databaseDiagnosticDialogTypes.ts b/src/serviceFeatures/databaseDiagnosticDialogTypes.ts new file mode 100644 index 00000000..ea142bd6 --- /dev/null +++ b/src/serviceFeatures/databaseDiagnosticDialogTypes.ts @@ -0,0 +1,9 @@ +import type { DiagnosticNameMode } from "@vrtmrz/livesync-commonlib/diagnostics"; +import type { DatabaseDiagnosticKind, DatabaseDiagnosticOutput } from "./databaseDiagnostics"; + +/** The dialogue can run or cancel an inspection without receiving the service container. */ +export interface DatabaseDiagnosticDialogOptions { + kind: DatabaseDiagnosticKind; + run(nameMode: DiagnosticNameMode, onProgress: (processed: number) => void): Promise; + cancel(): void; +} diff --git a/src/serviceFeatures/databaseDiagnostics.ts b/src/serviceFeatures/databaseDiagnostics.ts new file mode 100644 index 00000000..779a8aef --- /dev/null +++ b/src/serviceFeatures/databaseDiagnostics.ts @@ -0,0 +1,163 @@ +import { + analyseDatabaseIntegrity, + analyseDatabaseUsage, + createCouchDBDiagnosticSource, + createDiagnosticNameMapper, + createLocalDiagnosticSource, + formatDiagnosticJSON, + formatIntegrityMarkdown, + formatUsageTSV, + projectIntegrityReport, + projectUsageReport, + type DiagnosticContent, + type DiagnosticNameMode, + type DiagnosticStatus, +} from "@vrtmrz/livesync-commonlib/diagnostics"; +import { REMOTE_COUCHDB, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; +import type { NecessaryServicesInterfaces } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import commonlibPackage from "@vrtmrz/livesync-commonlib/package.json"; + +export type DatabaseDiagnosticKind = "usage" | "integrity"; +export type DatabaseDiagnosticHost = NecessaryServicesInterfaces< + "setting" | "database" | "path" | "API", + "storageAccess" +>; + +export interface DatabaseDiagnosticArtifact { + format: "json" | "markdown" | "tsv"; + filename: string; + content: string; +} + +export interface DatabaseDiagnosticOutput { + status: DiagnosticStatus; + nameMode: DiagnosticNameMode; + artifacts: DatabaseDiagnosticArtifact[]; +} + +export async function readDiagnosticStorage( + storage: Pick, + path: string +): Promise { + if (/^(?:[/\\]|[a-zA-Z]:)/.test(path) || path.includes("\0") || path.split(/[/\\]/).includes("..")) { + return { status: "unreadable", reason: "content-unreadable" }; + } + try { + const before = await storage.statHidden(path); + if (!before) return { status: "missing" }; + if (before.type !== "file") return { status: "unreadable", reason: "content-unreadable" }; + const bytes = new Uint8Array(await storage.readHiddenFileBinary(path)); + const after = await storage.statHidden(path); + if ( + !after || + after.type !== "file" || + before.mtime !== after.mtime || + before.ctime !== after.ctime || + before.size !== after.size || + bytes.byteLength !== after.size + ) { + return { status: "unreadable", reason: "storage-changed" }; + } + return { status: "available", bytes }; + } catch { + return { status: "unreadable", reason: "content-unreadable" }; + } +} + +/** Run only after the host has collected this run's name-display choice. */ +export async function runDatabaseDiagnostic( + host: DatabaseDiagnosticHost, + kind: DatabaseDiagnosticKind, + nameMode: DiagnosticNameMode, + options: { signal?: AbortSignal; onProgress?(processed: number): void } = {} +): Promise { + const configuration = JSON.stringify(host.services.setting.currentSettings()); + const settings = JSON.parse(configuration) as ObsidianLiveSyncSettings; + const database = host.services.database.localDatabase.localDatabase; + const isConfigurationCurrent = () => + JSON.stringify(host.services.setting.currentSettings()) === configuration && + host.services.database.localDatabase.localDatabase === database; + const mapper = createDiagnosticNameMapper(nameMode); + const versions = { + commonlib: commonlibPackage.version, + plugin: host.services.API.getPluginVersion(), + application: host.services.API.getAppVersion(), + }; + const local = createLocalDiagnosticSource(database); + if (kind === "usage") { + const result = await analyseDatabaseUsage({ + source: local, + signal: options.signal, + onProgress: (progress) => options.onProgress?.(progress.revisionsProcessed), + }); + if (!isConfigurationCurrent()) { + result.errors.push({ code: "configuration_changed" }); + if (result.status === "complete") result.status = "partial"; + } + const report = projectUsageReport(result, mapper, versions); + const artifacts: DatabaseDiagnosticArtifact[] = [ + { + format: "json", + filename: `database-usage-${result.status}.json`, + content: formatDiagnosticJSON(report), + }, + ]; + // A standalone legacy TSV has no coverage fields. Partial runs retain + // their rows in JSON together with the status and reasons. + if (result.status === "complete") { + artifacts.unshift({ format: "tsv", filename: "database-usage.tsv", content: formatUsageTSV(report) }); + } + return { status: result.status, nameMode, artifacts }; + } + const result = await analyseDatabaseIntegrity({ + local, + remote: + settings.remoteType === REMOTE_COUCHDB + ? () => + Promise.resolve( + createCouchDBDiagnosticSource({ + settings, + signal: options.signal, + fetch: (request, requestOptions) => + settings.useRequestAPI + ? host.services.API.nativeFetch(request, requestOptions) + : host.services.API.webCompatFetch(request, requestOptions), + }) + ) + : undefined, + remoteUnavailableReason: settings.remoteType === REMOTE_COUCHDB ? "source-unavailable" : "unsupported-remote", + path: { + getPath: (entry) => host.services.path.getPath(entry), + path2id: (path, prefix) => host.services.path.path2idWithSettings(path, settings, prefix), + }, + settings: { + encrypt: settings.encrypt, + usePathObfuscation: settings.usePathObfuscation, + handleFilenameCaseSensitive: settings.handleFilenameCaseSensitive, + idDerivationVersion: settings.idDerivationVersion ?? 0, + pathNormalizationVersion: settings.pathNormalizationVersion ?? 0, + }, + readStorage: (path) => readDiagnosticStorage(host.serviceModules.storageAccess, path), + isConfigurationCurrent, + signal: options.signal, + onProgress: (progress) => options.onProgress?.(progress.processed), + }); + const report = projectIntegrityReport(result, mapper, versions); + return { + status: result.status, + nameMode, + artifacts: [ + { + format: "markdown", + filename: `database-integrity-${result.status}.md`, + content: formatIntegrityMarkdown(report), + }, + { + format: "json", + filename: `database-integrity-${result.status}.json`, + content: formatDiagnosticJSON(report), + }, + ], + }; +} diff --git a/src/serviceFeatures/databaseDiagnostics.unit.spec.ts b/src/serviceFeatures/databaseDiagnostics.unit.spec.ts new file mode 100644 index 00000000..b572832d --- /dev/null +++ b/src/serviceFeatures/databaseDiagnostics.unit.spec.ts @@ -0,0 +1,213 @@ +import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_SETTINGS, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { readDiagnosticStorage, runDatabaseDiagnostic, type DatabaseDiagnosticHost } from "./databaseDiagnostics"; + +function fixture() { + const path = "Private/æ—ĨæœŦčĒž.md"; + const body = "private file contents"; + const bytes = new TextEncoder().encode(body); + const settings = { + ...DEFAULT_SETTINGS, + remoteType: REMOTE_P2P, + readChunksOnline: true, + passphrase: "private encryption passphrase", + couchDB_PASSWORD: "private credential", + couchDB_URI: "https://private-server.invalid", + }; + const documents = { + "f:private-actual-id": { + _id: "f:private-actual-id", + _rev: "1-private-revision", + _revs_info: [{ rev: "1-private-revision", status: "available" }], + type: "plain", + path, + ctime: 1, + mtime: 1, + size: bytes.byteLength, + children: ["h:private-chunk"], + }, + "h:private-chunk": { + _id: "h:private-chunk", + _rev: "1-private-chunk-revision", + _revs_info: [{ rev: "1-private-chunk-revision", status: "available" }], + type: "leaf", + data: body, + }, + }; + const get = vi.fn(async (id: string) => { + const doc = documents[id as keyof typeof documents]; + if (!doc) throw { status: 404 }; + return structuredClone(doc); + }); + const info = vi.fn(async () => ({ update_seq: "1-stable" })); + const put = vi.fn(() => { + throw new Error("Diagnostic attempted a write"); + }); + const applyPartial = vi.fn(); + const fetch = vi.fn(); + const storage = { + statHidden: vi.fn(async () => ({ type: "file" as const, ctime: 1, mtime: 1, size: bytes.byteLength })), + readHiddenFileBinary: vi.fn(async () => bytes.buffer), + }; + const host = { + services: { + setting: { currentSettings: () => settings, applyPartial }, + database: { + localDatabase: { + localDatabase: { + info, + get, + put, + allDocs: vi.fn(async () => ({ + rows: Object.keys(documents).map((id) => ({ id, value: { rev: "1-current" } })), + })), + }, + }, + }, + path: { + getPath: (doc: { path: string }) => doc.path, + path2idWithSettings: vi.fn(async (rawPath: string) => `f:expected:${rawPath}`), + }, + API: { + webCompatFetch: fetch, + nativeFetch: fetch, + getPluginVersion: () => "1.0.38", + getAppVersion: () => "1.12.7", + }, + }, + serviceModules: { storageAccess: storage }, + } as unknown as DatabaseDiagnosticHost; + return { host, settings, path, body, bytes, get, info, put, applyPartial, fetch, storage }; +} + +describe("database diagnostic host composition", () => { + it("keeps on-demand fetching enabled while analysing only existing local revisions", async () => { + const data = fixture(); + const before = JSON.stringify(data.settings); + const result = await runDatabaseDiagnostic(data.host, "usage", "pseudonym"); + expect(result.status).toBe("complete"); + expect(result.artifacts.map((artifact) => artifact.format)).toEqual(["tsv", "json"]); + expect(data.settings.readChunksOnline).toBe(true); + expect(JSON.stringify(data.settings)).toBe(before); + expect(data.applyPartial).not.toHaveBeenCalled(); + expect(data.fetch).not.toHaveBeenCalled(); + expect(data.put).not.toHaveBeenCalled(); + expect(data.get).toHaveBeenCalledWith("f:private-actual-id", { rev: "1-private-revision" }); + const report = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content); + expect(report.versions).toMatchObject({ plugin: "1.0.38", application: "1.12.7" }); + expect(report.versions.commonlib).toMatch(/^\d+\.\d+\.\d+/); + for (const artifact of result.artifacts) { + for (const secret of [ + data.path, + data.body, + data.settings.passphrase, + "private-actual-id", + "private-revision", + "private-server.invalid", + ]) { + expect(artifact.content).not.toContain(secret); + } + } + }); + + it("keeps local findings when the remote type is unsupported and projects both formats consistently", async () => { + const data = fixture(); + const result = await runDatabaseDiagnostic(data.host, "integrity", "pseudonym"); + expect(result.status).toBe("partial"); + expect(data.fetch).not.toHaveBeenCalled(); + expect(data.put).not.toHaveBeenCalled(); + expect(data.storage.readHiddenFileBinary).toHaveBeenCalledWith(data.path); + const json = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content); + expect(json.errors).toContainEqual({ source: "remote", reason: "unsupported-remote" }); + expect(json.observations[0]).toMatchObject({ + identity: "document-id-mismatch", + content: { status: "available" }, + }); + expect(json.observations[0].comparisons).toContainEqual({ + against: json.observations[0].vault.reference, + relation: "equal", + }); + for (const artifact of result.artifacts) { + expect(artifact.content).toContain(json.observations[0].path); + expect(artifact.content).not.toContain(data.path); + expect(artifact.content).not.toContain(data.body); + expect(artifact.content).not.toContain(data.settings.passphrase); + } + }); + + it("includes names only when requested and never includes credentials or file contents", async () => { + const data = fixture(); + const result = await runDatabaseDiagnostic(data.host, "usage", "original"); + for (const artifact of result.artifacts) { + expect(artifact.content).toContain(data.path); + expect(artifact.content).toContain("f:private-actual-id"); + expect(artifact.content).not.toContain(data.body); + expect(artifact.content).not.toContain(data.settings.couchDB_PASSWORD); + expect(artifact.content).not.toContain(data.settings.couchDB_URI); + } + }); + + it("retains cancellation and configuration changes in JSON instead of presenting a complete TSV", async () => { + const cancelled = fixture(); + const controller = new AbortController(); + controller.abort(); + const stopped = await runDatabaseDiagnostic(cancelled.host, "usage", "pseudonym", { + signal: controller.signal, + }); + expect(stopped.status).toBe("cancelled"); + expect(stopped.artifacts.map((artifact) => artifact.format)).toEqual(["json"]); + expect(cancelled.get).not.toHaveBeenCalled(); + + const changed = fixture(); + changed.info.mockImplementationOnce(async () => ({ update_seq: "1-stable" })); + changed.info.mockImplementationOnce(async () => { + changed.settings.couchDB_URI = "https://changed.invalid"; + return { update_seq: "1-stable" }; + }); + const output = await runDatabaseDiagnostic(changed.host, "usage", "original"); + expect(output.status).toBe("partial"); + expect(output.artifacts.map((artifact) => artifact.format)).toEqual(["json"]); + expect(JSON.parse(output.artifacts[0].content).errors).toContainEqual({ code: "configuration_changed" }); + }); +}); + +describe("diagnostic Vault reads", () => { + it("distinguishes an absent file, a read error, and a file changing during its read", async () => { + const data = fixture(); + data.storage.statHidden.mockResolvedValueOnce(null as never); + expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ status: "missing" }); + data.storage.readHiddenFileBinary.mockRejectedValueOnce(new Error("private filename and content")); + expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ + status: "unreadable", + reason: "content-unreadable", + }); + data.storage.statHidden.mockResolvedValueOnce({ + type: "file", + ctime: 1, + mtime: 1, + size: data.bytes.byteLength, + }); + data.storage.statHidden.mockResolvedValueOnce({ + type: "file", + ctime: 1, + mtime: 2, + size: data.bytes.byteLength, + }); + expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ + status: "unreadable", + reason: "storage-changed", + }); + }); + + it("rejects paths outside the Vault before touching storage", async () => { + const data = fixture(); + for (const path of ["../private", "/private", "C:\\private", "x/../private", "x\0y"]) { + expect(await readDiagnosticStorage(data.storage, path)).toEqual({ + status: "unreadable", + reason: "content-unreadable", + }); + } + expect(data.storage.statHidden).not.toHaveBeenCalled(); + expect(data.storage.readHiddenFileBinary).not.toHaveBeenCalled(); + }); +}); diff --git a/src/serviceFeatures/useDatabaseDiagnostics.ts b/src/serviceFeatures/useDatabaseDiagnostics.ts new file mode 100644 index 00000000..71021b1c --- /dev/null +++ b/src/serviceFeatures/useDatabaseDiagnostics.ts @@ -0,0 +1,76 @@ +import { + createServiceFeature, + type NecessaryServices, +} from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events"; +import { $msg } from "@/common/translation"; +import DatabaseDiagnosticDialog from "@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte"; +import { runDatabaseDiagnostic, type DatabaseDiagnosticKind } from "./databaseDiagnostics"; +import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes"; + +type DatabaseDiagnosticsFeatureHost = NecessaryServices< + "setting" | "database" | "path" | "API" | "UI" | "appLifecycle", + "storageAccess" +>; + +export const useDatabaseDiagnosticsFeature = createServiceFeature((host: DatabaseDiagnosticsFeatureHost) => { + const { services } = host; + let active: AbortController | undefined; + const open = async (kind: DatabaseDiagnosticKind): Promise => { + if (active) return; + const controller = new AbortController(); + active = controller; + try { + await services.UI.dialogManager.open<"close", DatabaseDiagnosticDialogOptions>(DatabaseDiagnosticDialog, { + kind, + run: async (nameMode, onProgress) => { + if (!services.database.isDatabaseReady()) throw new Error("local-database-unavailable"); + return await runDatabaseDiagnostic(host, kind, nameMode, { + signal: controller.signal, + onProgress, + }); + }, + cancel: () => controller.abort(), + }); + } catch { + // Closing the host during unload can reject the dialogue result. + // Diagnostic errors are presented inside the dialogue without raw logs. + } finally { + controller.abort(); + if (active === controller) active = undefined; + } + }; + services.appLifecycle.onInitialise.addHandler(() => { + services.API.addCommand({ + id: "analyse-database", + name: "Analyse Database Usage (advanced)", + icon: "database-search", + checkCallback: (checking) => { + if (!services.setting.currentSettings().useAdvancedMode || !services.database.isDatabaseReady()) + return false; + if (!checking) void open("usage"); + return true; + }, + }); + services.API.addCommand({ + id: "inspect-database-integrity", + name: $msg("Inspect database integrity"), + icon: "database-search", + checkCallback: (checking) => { + if (!services.database.isDatabaseReady()) return false; + if (!checking) void open("integrity"); + return true; + }, + }); + return Promise.resolve(true); + }); + const stopUsage = services.context.events.onEvent(EVENT_ANALYSE_DB_USAGE, () => void open("usage")); + const stopIntegrity = services.context.events.onEvent(EVENT_INSPECT_DB_INTEGRITY, () => void open("integrity")); + services.appLifecycle.onUnload.addHandler(() => { + active?.abort(); + stopUsage(); + stopIntegrity(); + return Promise.resolve(true); + }); + return { open }; +}); diff --git a/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts new file mode 100644 index 00000000..0e9669e0 --- /dev/null +++ b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from "vitest"; +import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes"; + +vi.mock("@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte", () => ({ default: {} })); +vi.mock("./databaseDiagnostics", () => ({ + runDatabaseDiagnostic: vi.fn(async () => ({ status: "complete", nameMode: "pseudonym", artifacts: [] })), +})); +vi.mock("@/common/translation", () => ({ $msg: (value: string) => value })); + +import { runDatabaseDiagnostic } from "./databaseDiagnostics"; +import { useDatabaseDiagnosticsFeature } from "./useDatabaseDiagnostics"; +import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events"; + +function fixture() { + const initialise: Array<() => Promise> = []; + const unload: Array<() => Promise> = []; + const commands: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = []; + const listeners = new Map void>(); + const unsubscribe = vi.fn((event: string) => listeners.delete(event)); + const settings = { useAdvancedMode: false }; + let ready = true; + let close!: () => void; + const openDialog = vi.fn( + (_component: unknown, _options: DatabaseDiagnosticDialogOptions) => + new Promise((resolve) => { + close = resolve; + }) + ); + const host = { + services: { + setting: { currentSettings: () => settings }, + database: { isDatabaseReady: () => ready }, + path: {}, + API: { addCommand: (command: (typeof commands)[number]) => commands.push(command) }, + UI: { dialogManager: { open: openDialog } }, + appLifecycle: { + onInitialise: { addHandler: (handler: (typeof initialise)[number]) => initialise.push(handler) }, + onUnload: { addHandler: (handler: (typeof unload)[number]) => unload.push(handler) }, + }, + context: { + events: { + onEvent: (event: string, handler: () => void) => { + listeners.set(event, handler); + return () => unsubscribe(event); + }, + }, + }, + }, + serviceModules: { storageAccess: {} }, + } as unknown as Parameters[0]; + const view = useDatabaseDiagnosticsFeature(host); + return { + host, + view, + initialise, + unload, + commands, + listeners, + settings, + openDialog, + unsubscribe, + close: () => close(), + setReady: (value: boolean) => { + ready = value; + }, + }; +} + +describe("database diagnostics serviceFeature", () => { + it("retains the existing Advanced-mode Usage command gate in its new owner", async () => { + const data = fixture(); + for (const initialise of data.initialise) await initialise(); + const usage = data.commands.find((command) => command.id === "analyse-database")!; + const integrity = data.commands.find((command) => command.id === "inspect-database-integrity")!; + expect(usage.checkCallback(true)).toBe(false); + expect(integrity.checkCallback(true)).toBe(true); + data.settings.useAdvancedMode = true; + expect(usage.checkCallback(true)).toBe(true); + data.setReady(false); + expect(usage.checkCallback(true)).toBe(false); + expect(integrity.checkCallback(true)).toBe(false); + expect(data.openDialog).not.toHaveBeenCalled(); + }); + + it("uses the existing dialogue manager and performs no analysis until the run choice is submitted", async () => { + vi.mocked(runDatabaseDiagnostic).mockClear(); + const data = fixture(); + const opened = data.view.open("usage"); + expect(data.openDialog).toHaveBeenCalledTimes(1); + expect(runDatabaseDiagnostic).not.toHaveBeenCalled(); + const options = data.openDialog.mock.calls[0][1]; + expect(Object.keys(options).sort()).toEqual(["cancel", "kind", "run"]); + await options.run("original", () => {}); + expect(runDatabaseDiagnostic).toHaveBeenCalledWith( + data.host, + "usage", + "original", + expect.objectContaining({ signal: expect.any(AbortSignal) }) + ); + const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!; + await data.view.open("integrity"); + expect(data.openDialog).toHaveBeenCalledTimes(1); + data.close(); + await opened; + expect(signal.aborted).toBe(true); + }); + + it("wires both Hatch actions and releases its listeners and active request on unload", async () => { + const data = fixture(); + expect(data.listeners.has(EVENT_ANALYSE_DB_USAGE)).toBe(true); + expect(data.listeners.has(EVENT_INSPECT_DB_INTEGRITY)).toBe(true); + const opened = data.view.open("integrity"); + await data.openDialog.mock.calls[0][1].run("pseudonym", () => {}); + const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!; + for (const unload of data.unload) await unload(); + expect(signal.aborted).toBe(true); + expect(data.listeners.size).toBe(0); + expect(data.unsubscribe).toHaveBeenCalledTimes(2); + data.close(); + await opened; + }); +}); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index aa3afb23..ecef9569 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -74,6 +74,7 @@ After changing plug-in source, use the focused wrapper rather than invoking a sc ```bash npm run test:e2e:obsidian:focused -- settings-ui +npm run test:e2e:obsidian:focused -- database-diagnostics npm run test:e2e:obsidian:focused -- configuration-encryption npm run test:e2e:obsidian:focused -- configuration-encryption-sync npm run test:e2e:obsidian:focused -- two-vault-sync @@ -85,6 +86,8 @@ npm run test:e2e:obsidian:focused -- couchdb-http-cache The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. +`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including the choice before inspection, original names and pseudonyms, retained local findings when the remote is unavailable, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice dialogue in mobile emulation. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. Commonlib's separate CouchDB integration test verifies encrypted reads and missing Security Seed handling against a real server. + `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts new file mode 100644 index 00000000..d6e59714 --- /dev/null +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -0,0 +1,237 @@ +import assert from "node:assert/strict"; +import type { Page } from "playwright"; +import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { captureObsidianElement, captureObsidianPage, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts"; +import { assertMobileDialogueLayout, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +const fixturePath = "Private/diagnostic-į§˜å¯†.md"; +const fixtureContent = "Diagnostic fixture content which must never appear in a report.\n"; +const shadowId = "diagnostic-shadow-document"; +const timeout = 15000; + +type DiagnosticPageGlobal = typeof globalThis & { + app: { commands: { executeCommandById(id: string): boolean } }; + diagnosticClipboard?: string; + diagnosticDownload?: { name: string; content: string }; +}; + +async function openCommand(page: Page, kind: "usage" | "integrity") { + const id = kind === "usage" ? "analyse-database" : "inspect-database-integrity"; + const opened = await page.evaluate( + (command) => (globalThis as DiagnosticPageGlobal).app.commands.executeCommandById(command), + `obsidian-livesync:${id}` + ); + assert.equal(opened, true); + const dialogue = page.locator(".database-diagnostic"); + await dialogue.locator("select").selectOption("pseudonym"); + assert.equal(await dialogue.getAttribute("data-diagnostic-phase"), "choice"); + assert.equal(await dialogue.locator("textarea").count(), 0); + return dialogue; +} + +async function readDatabaseState(cli: string, session: ObsidianLiveSyncSession) { + return await evalObsidianJson<{ sequence: unknown; settings: string; names: string[] }>( + cli, + "(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;return JSON.stringify({sequence:(await core.localDatabase.localDatabase.info()).update_seq,settings:JSON.stringify(core.services.setting.currentSettings()),names:app.vault.getFiles().map(f=>f.path).sort()});})()", + session.cliEnv + ); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = requireObsidianCli(); + const vault = await createTemporaryVault("obsidian-livesync-diagnostics-"); + let session: ObsidianLiveSyncSession | undefined; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli, + vault, + pluginData: { + displayLanguage: "def", + doctorProcessedVersion: DoctorRegulation.version, + isConfigured: true, + useAdvancedMode: true, + readChunksOnline: true, + liveSync: false, + remoteType: "", + couchDB_URI: "", + couchDB_DBNAME: "", + couchDB_USER: "", + couchDB_PASSWORD: "", + remoteConfigurations: {}, + activeConfigurationId: "", + notifyThresholdOfRemoteStorageSize: -1, + periodicReplication: false, + syncAfterMerge: false, + syncOnEditorSave: false, + syncOnFileOpen: false, + syncOnSave: false, + syncOnStart: false, + useEden: false, + }, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + }); + await waitForLiveSyncCoreReady(cli, session.cliEnv); + await evalObsidianJson( + cli, + `(async()=>{await app.vault.createFolder('Private');await app.vault.create(${JSON.stringify(fixturePath)},${JSON.stringify(fixtureContent)});return JSON.stringify(true);})()`, + session.cliEnv + ); + await waitForLocalDatabaseEntry(cli, session.cliEnv, fixturePath); + await evalObsidianJson( + cli, + `(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;await core.services.fileProcessing.commitPendingFileEvents();const db=core.localDatabase.localDatabase;const id=await core.services.path.path2id(${JSON.stringify(fixturePath)});const original=await db.get(id);const shadow={...original,_id:${JSON.stringify(shadowId)}};delete shadow._rev;await db.put(shadow);return JSON.stringify(true);})()`, + session.cliEnv + ); + const before = await readDatabaseState(cli, session); + let usageJSON = ""; + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const choice = await openCommand(page, "usage"); + await choice.getByRole("button", { name: "Cancel", exact: true }).click(); + await choice.waitFor({ state: "hidden", timeout }); + const dialogue = await openCommand(page, "usage"); + assert.equal(await dialogue.locator("select").inputValue(), "pseudonym"); + await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + const tsv = await dialogue.locator("textarea").inputValue(); + assert.equal(tsv.split("\n")[0].split("\t").length, 9); + assert.ok(!tsv.includes(fixturePath)); + assert.ok(!tsv.includes(shadowId)); + assert.ok(!tsv.includes(fixtureContent.trim())); + await dialogue.locator("select").selectOption("json"); + usageJSON = await dialogue.locator("textarea").inputValue(); + const report = JSON.parse(usageJSON); + assert.equal(report.status, "complete"); + assert.ok(report.versions.commonlib); + assert.ok(report.rows.some((row: { path: string }) => row.path.startsWith("file-"))); + + // Capture exports inside the isolated renderer without changing the + // host clipboard or leaving downloads in a user's download directory. + await page.evaluate(`(() => { + Object.defineProperty(navigator.clipboard, "writeText", { + configurable: true, + value: async (text) => { globalThis.diagnosticClipboard = text; }, + }); + document.addEventListener("click", (event) => { + const link = event.target; + if (!(link instanceof HTMLAnchorElement) || !link.download || !link.href.startsWith("blob:")) return; + event.preventDefault(); + void fetch(link.href).then((response) => response.text()).then((content) => { + globalThis.diagnosticDownload = { name: link.download, content }; + }); + }, true); + })()`); + await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); + assert.equal( + await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), + usageJSON + ); + await dialogue.getByRole("button", { name: "Save report", exact: true }).click(); + await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { + timeout, + }); + assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { + name: "database-usage-complete.json", + content: usageJSON, + }); + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); + + const original = await openCommand(page, "usage"); + await original.locator("select").selectOption("original"); + await original.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + const rawTSV = await original.locator("textarea").inputValue(); + assert.ok(rawTSV.includes(fixturePath)); + assert.ok(rawTSV.includes(shadowId)); + assert.ok(!rawTSV.includes(fixtureContent.trim())); + await original.getByRole("button", { name: "Close", exact: true }).click(); + await original.waitFor({ state: "hidden", timeout }); + + const settingsNavigator = await openLiveSyncSettings(page, timeout); + const hatch = await settingsNavigator.openPage("Hatch"); + const setting = hatch + .locator(".setting-item") + .filter({ has: page.getByText("Inspect database integrity", { exact: true }) }); + await setting.getByRole("button", { name: "Inspect", exact: true }).click(); + const integrity = page.locator(".database-diagnostic"); + assert.equal(await integrity.locator("select").inputValue(), "pseudonym"); + await integrity.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await integrity.locator("select").selectOption("json"); + const reportText = await integrity.locator("textarea").inputValue(); + const integrityReport = JSON.parse(reportText); + assert.equal(integrityReport.status, "partial"); + assert.ok( + integrityReport.errors.some((error: { reason: string }) => error.reason === "source-unavailable") + ); + assert.ok( + integrityReport.observations.some( + (observation: { identity: string }) => observation.identity === "document-id-mismatch" + ) + ); + assert.ok( + integrityReport.observations.some((observation: { comparisons: { relation: string }[] }) => + observation.comparisons.some((comparison) => comparison.relation === "equal") + ) + ); + for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) + assert.ok(!reportText.includes(privateValue)); + }); + const desktopScreenshot = await captureObsidianElement( + session.remoteDebuggingPort, + "database-diagnostics-desktop.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.locator(".database-diagnostic").getByRole("button", { name: "Close", exact: true }).click(); + await page.keyboard.press("Escape"); + }); + const after = await readDatabaseState(cli, session); + assert.deepEqual(after, before, "Diagnostics must leave settings, revisions, and Vault files unchanged"); + + await setObsidianMobileTestMode(session.remoteDebuggingPort, true, timeout); + await waitForLiveSyncCoreReady(cli, session.cliEnv); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await openCommand(page, "integrity"); + }); + const mobileScreenshot = await captureObsidianElement( + session.remoteDebuggingPort, + "database-diagnostics-mobile.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); + await assertMobileDialogueLayout(page, modal, "Database diagnostic choice"); + await modal.getByRole("button", { name: "Cancel", exact: true }).click(); + }); + console.log(JSON.stringify({ status: "passed", desktopScreenshot, mobileScreenshot, readOnly: true })); + } catch (error) { + if (session) { + await captureObsidianPage( + session.remoteDebuggingPort, + "database-diagnostics-failure.png", + async () => {} + ).catch(() => {}); + } + throw error; + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index b0199b89..7ce9514e 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -8,6 +8,7 @@ const focusedScenarios = new Set([ "smoke", "onboarding-invitation", "dialog-mounts", + "database-diagnostics", "conflict-dialog-policy", "revision-repair", "document-history-nav", diff --git a/updates.md b/updates.md index c8d1371e..7b8c9756 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Diagnostics + +- We can inspect normal-file Metadata and current conflicts in the local database and CouchDB together, with byte comparisons for suspect groups. Partial results retain the available evidence when a source is unavailable. The inspection leaves databases and settings unchanged. +- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons. + ## 1.0.38 17th October, 2026 From 0ca499778e2a098eca964a0e2be47c02b42b82a8 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 12:55:47 +0000 Subject: [PATCH 17/68] Use Commonlib 0.1.43 for NFC reconstruction --- docs/design_docs/unicode_path_identity.md | 14 +++++++++++--- package-lock.json | 8 ++++---- package.json | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/design_docs/unicode_path_identity.md b/docs/design_docs/unicode_path_identity.md index d88e1277..6827d9b0 100644 --- a/docs/design_docs/unicode_path_identity.md +++ b/docs/design_docs/unicode_path_identity.md @@ -1,6 +1,6 @@ --- date: 2026-10-19 -commonlib-version: "0.1.42" +commonlib-version: "0.1.43" self-hosted-livesync-version: "1.0.38" status: unreleased --- @@ -17,8 +17,8 @@ Commonlib's [consumer contract](https://github.com/vrtmrz/livesync-commonlib/blo defines the shared package boundary. NFC identity shipped in Self-hosted LiveSync `1.0.38` with Commonlib `0.1.42`. -The deletion handling in reconstruction below is an unreleased refinement of -that baseline. +The deletion handling in reconstruction below uses Commonlib `0.1.43` and +remains unreleased in Self-hosted LiveSync. ## Scope @@ -241,6 +241,14 @@ NFC duplicate, and remote deletion through the polling CLI daemon. The `mirror` command retains tombstoned local files by default; its update behaviour was tested separately from daemon deletion reflection. +The published Commonlib `0.1.43` package also passed plain and encrypted NFC +migration with obsolete logical deletions, plus the legacy encrypted ASCII +scenario, on Linux with Obsidian `1.12.7` and CouchDB `3.5.0`. Both migration +variants checked CLI Rebuild, second-client Standard Fetch, and real Obsidian +Streaming Fast Fetch. Fast Fetch removed the obsolete Metadata and retained +the current NFC document, its active state, and its stored path. The existing +content, creation-time, physical-name, update, and deletion checks also passed. + ## Non-goals This design does not: diff --git a/package-lock.json b/package-lock.json index 583f5928..fcfe4617 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.42", + "@vrtmrz/livesync-commonlib": "0.1.43", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4389,9 +4389,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.42", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.42.tgz", - "integrity": "sha512-LyWl0Mkrj9UsQXV0nOUaYJybcN3wLkpAAxCeHMZYOEizaUTQ7ezN+FgzN+OQxDRA8UXWnoatxh719NWzOwBpBw==", + "version": "0.1.43", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.43.tgz", + "integrity": "sha512-WM2k545hT0ctLv64WdKZ0PkHwyw9Fnlf9xssH7Uv9C0HEMm8CDtj6dfJsPOHs0+Z41Kfco1U/y6R5f5+xWZ00g==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 4f33396b..936bba06 100644 --- a/package.json +++ b/package.json @@ -191,7 +191,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.42", + "@vrtmrz/livesync-commonlib": "0.1.43", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", From c23b055cd684fcc768cad316da8c218222c49595 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 13:49:01 +0000 Subject: [PATCH 18/68] Verify database diagnostics and keep mobile report controls visible --- docs/troubleshooting.md | 2 +- package.json | 1 + .../dialogues/DatabaseDiagnosticDialog.svelte | 2 + test/e2e-obsidian/README.md | 21 +- .../scripts/database-diagnostics-couchdb.ts | 263 ++++++++++++++++++ .../scripts/database-diagnostics.ts | 180 ++++++++++-- test/e2e-obsidian/scripts/run-focused.ts | 1 + test/e2e-obsidian/scripts/two-vault-sync.ts | 1 + 8 files changed, 452 insertions(+), 19 deletions(-) create mode 100644 test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5bdfe39d..afd14d67 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -167,7 +167,7 @@ For a database-wide inspection, use **Hatch** → **Inspect database integrity** The result dialogue provides Markdown and JSON reports with the same name policy. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. They are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled. -**Analyse database usage** offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads. +For storage accounting, use **Hatch** → **Analyse database usage** → **Analyse**. It offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads. Follow [How to report an issue](to_issue_reporting.md) for the complete reporting checklist, including Obsidian debug information and the privacy review required before sharing evidence. diff --git a/package.json b/package.json index 3103193f..983ca49f 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", "test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts", + "test:e2e:obsidian:database-diagnostics-couchdb": "tsx test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", "test:e2e:obsidian:revision-repair": "tsx test/e2e-obsidian/scripts/revision-repair.ts", "test:e2e:obsidian:document-history-nav": "tsx test/e2e-obsidian/scripts/document-history-nav.ts", diff --git a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte index 5403d1c9..87c85856 100644 --- a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte @@ -141,8 +141,10 @@ diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index ecef9569..e84cc4da 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -75,6 +75,7 @@ After changing plug-in source, use the focused wrapper rather than invoking a sc ```bash npm run test:e2e:obsidian:focused -- settings-ui npm run test:e2e:obsidian:focused -- database-diagnostics +npm run test:e2e:obsidian:focused -- database-diagnostics-couchdb npm run test:e2e:obsidian:focused -- configuration-encryption npm run test:e2e:obsidian:focused -- configuration-encryption-sync npm run test:e2e:obsidian:focused -- two-vault-sync @@ -86,7 +87,9 @@ npm run test:e2e:obsidian:focused -- couchdb-http-cache The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. -`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including the choice before inspection, original names and pseudonyms, retained local findings when the remote is unavailable, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice dialogue in mobile emulation. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. Commonlib's separate CouchDB integration test verifies encrypted reads and missing Security Seed handling against a real server. +`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including the choice before inspection, original names and pseudonyms, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking result layout, format selection, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. + +`database-diagnostics-couchdb` needs the CouchDB fixture with CORS enabled. It checks unencrypted and encrypted databases, both browser fetch and Obsidian's native Request API, and both name policies through the real integrity dialogue. A remote-only document with a mismatched ID must produce a finding with readable Chunks and equal content. JSON and Markdown must omit file contents and credentials. The scenario observes real requests and requires GET-only access, unchanged local settings, revisions, and Vault contents, and unchanged remote documents and Security Seed. Commonlib's separate CouchDB integration tests additionally cover a missing Security Seed without creating one. `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. @@ -311,6 +314,22 @@ BRAT acceptance validates installation of the exact published artefacts: install A short physical-device check is optional when a specific concern remains about localised time text, input, clipboard interaction, or responsiveness. The camera and operating-system dispatch paths are unchanged by this integration and do not require routine revalidation. Emulated mobile establishes layout and interaction, but does not establish native device performance or verify the published installation path. +### Database diagnostic release checks + +For a release containing database diagnostics, run these scenarios sequentially against the same artefacts: + +```bash +npm run test:e2e:obsidian:database-diagnostics +npm run test:e2e:obsidian:database-diagnostics-couchdb +E2E_OBSIDIAN_ONLY_INDEPENDENT_IDS=true npm run test:e2e:obsidian:two-vault-sync +``` + +The first scenario needs no service; the other two need CouchDB. The last checks an encrypted note round trip between two real Obsidian Vaults, with Path Obfuscation and independent IDs, and rejection of incompatible keys. These checks complement Commonlib's classification, conflict, privacy, and cancellation tests. + +Set `E2E_OBSIDIAN_ARTIFACT_ROOT` to a directory containing the exact release `main.js`, `manifest.json`, and `styles.css` when testing downloaded artefacts. These three scenarios accept that directory. Invoke their direct commands above, because the focused wrapper rebuilds local artefacts. Record the commit, Obsidian version, artefact hashes, and results. If testing a local build, compare its hashes with the published release before relying on that evidence for BRAT acceptance. + +When these checks pass for the release artefacts, BRAT acceptance can be limited to installation or update, enabling the plug-in, and a cold start. Check any other changes included in that release using their own regression scenarios. The diagnostic export checks verify report payloads; they do not exercise the operating system's clipboard or download implementation on iOS or Android. + ### Environment variables - `OBSIDIAN_BINARY`: explicit Obsidian executable path. diff --git a/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts b/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts new file mode 100644 index 00000000..cf1276f7 --- /dev/null +++ b/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts @@ -0,0 +1,263 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + fetchAllCouchDbDocs, + fetchCouchDbDatabaseInfo, + fetchCouchDbDocument, + fetchCouchDbLocalDocs, + loadCouchDbConfig, + makeUniqueDatabaseName, + putCouchDbDocument, + type CouchDbConfig, +} from "../runner/couchdb.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + pushLocalChanges, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000"; +const fixturePath = "Diagnostic/remote-į§˜å¯†.md"; +const fixtureContent = "Diagnostic remote fixture contents must stay outside reports.\n"; +const passphrase = "diagnostic-remote-fixture-passphrase"; +const timeout = 20000; + +interface IntegrityReport { + status: string; + coverage: { local: { status: string }; remote: { status: string; contentReads: number } }; + observations: { + source: string; + identity: string; + content: { status: string; chunksRead: number }; + comparisons: { relation: string }[]; + }[]; + errors: unknown[]; +} + +async function localState(cli: string, session: ObsidianLiveSyncSession) { + const state = await evalObsidianJson( + cli, + "(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;const files=await Promise.all(app.vault.getFiles().map(async file=>[file.path,await app.vault.read(file)]));files.sort((a,b)=>a[0].localeCompare(b[0]));return JSON.stringify({sequence:(await core.localDatabase.localDatabase.info()).update_seq,settings:core.services.setting.currentSettings(),files});})()", + session.cliEnv + ); + return createHash("sha256").update(JSON.stringify(state)).digest("hex"); +} + +async function remoteState(config: CouchDbConfig, name: string) { + const [info, documents, localDocuments] = await Promise.all([ + fetchCouchDbDatabaseInfo(config, name), + fetchAllCouchDbDocs(config, name), + fetchCouchDbLocalDocs(config, name), + ]); + // Hash the snapshot so a failed assertion cannot print Security Seed data. + return createHash("sha256") + .update(JSON.stringify([info.update_seq, documents, localDocuments])) + .digest("hex"); +} + +async function observeRequests(cli: string, session: ObsidianLiveSyncSession) { + await evalObsidianJson( + cli, + `(() => { + const api=app.plugins.plugins['obsidian-livesync'].core.services.API; + const state={calls:[],restore:undefined}; + const originals={}; + for(const name of ['webCompatFetch','nativeFetch']) { + originals[name]=api[name]; + api[name]=function(request,options) { + state.calls.push({transport:name,method:options?.method??(typeof request==='string'?'GET':request.method)}); + return originals[name].call(this,request,options); + }; + } + state.restore=()=>{for(const name of Object.keys(originals)) api[name]=originals[name];}; + globalThis.diagnosticRequests=state; + return JSON.stringify(true); + })()`, + session.cliEnv + ); +} + +async function finishRequests(cli: string, session: ObsidianLiveSyncSession) { + return await evalObsidianJson<{ transport: string; method: string }[]>( + cli, + "(()=>{const state=globalThis.diagnosticRequests;if(!state)return JSON.stringify([]);state.restore();delete globalThis.diagnosticRequests;return JSON.stringify(state.calls);})()", + session.cliEnv + ); +} + +async function runCase(config: CouchDbConfig, encrypted: boolean) { + const binary = requireObsidianBinary(); + const cli = requireObsidianCli(); + const name = makeUniqueDatabaseName(config.dbPrefix, "diagnostic"); + const vault = await createTemporaryVault("obsidian-livesync-remote-diagnostic-"); + let session: ObsidianLiveSyncSession | undefined; + let created = false; + try { + await createCouchDbDatabase(config, name); + created = true; + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli, + vault, + artifactRoot: process.env.E2E_OBSIDIAN_ARTIFACT_ROOT?.trim() || undefined, + pluginData: createE2eCouchDbPluginData( + { ...config, dbName: name }, + { + doctorProcessedVersion: DoctorRegulation.version, + useAdvancedMode: true, + encrypt: encrypted, + passphrase: encrypted ? passphrase : "", + usePathObfuscation: encrypted, + encryptInternalMetadata: encrypted, + E2EEAlgorithm: "v2", + useRequestAPI: true, + periodicReplication: false, + syncAfterMerge: false, + syncOnEditorSave: false, + syncOnFileOpen: false, + notifyThresholdOfRemoteStorageSize: -1, + } + ), + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + }); + await waitForLiveSyncCoreReady(cli, session.cliEnv); + await prepareRemote(cli, session.cliEnv); + await evalObsidianJson( + cli, + `(async()=>{await app.vault.createFolder('Diagnostic');await app.vault.create(${JSON.stringify(fixturePath)},${JSON.stringify(fixtureContent)});return JSON.stringify(true);})()`, + session.cliEnv + ); + const entry = await waitForLocalDatabaseEntry(cli, session.cliEnv, fixturePath); + await pushLocalChanges(cli, session.cliEnv); + const original = await fetchCouchDbDocument(config, name, entry.id); + assert.ok(entry.children.length > 0); + if (encrypted) assert.ok(!JSON.stringify(original).includes(fixturePath)); + const shadowId = await evalObsidianJson( + cli, + "(async()=>JSON.stringify(await app.plugins.plugins['obsidian-livesync'].core.services.path.path2id('Diagnostic/different-identity.md')))()", + session.cliEnv + ); + const shadow = { ...original, _id: shadowId }; + delete shadow._rev; + await putCouchDbDocument(config, name, shadow); + + for (const native of [false, true]) { + await evalObsidianJson( + cli, + `(async()=>{await app.plugins.plugins['obsidian-livesync'].core.services.setting.applyPartial({useRequestAPI:${native}});return JSON.stringify(true);})()`, + session.cliEnv + ); + const localBefore = await localState(cli, session); + const remoteBefore = await remoteState(config, name); + await observeRequests(cli, session); + let calls: { transport: string; method: string }[] = []; + try { + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + for (const names of ["pseudonym", "original"]) { + const opened = await page.evaluate( + "app.commands.executeCommandById('obsidian-livesync:inspect-database-integrity')" + ); + assert.equal(opened, true); + const dialogue = page.locator(".database-diagnostic"); + await dialogue.locator("select").selectOption(names); + await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await dialogue.locator("select").selectOption("json"); + const text = await dialogue.locator("textarea").inputValue(); + const report = JSON.parse(text) as IntegrityReport; + assert.equal(report.status, "complete"); + assert.equal(report.coverage.local.status, "complete"); + assert.equal(report.coverage.remote.status, "complete"); + assert.ok(report.coverage.remote.contentReads > 0); + assert.deepEqual(report.errors, []); + const suspect = report.observations.find( + (row) => row.source === "remote" && row.identity === "document-id-mismatch" + ); + assert.ok(suspect); + assert.equal(suspect.content.status, "available"); + assert.ok(suspect.content.chunksRead > 0); + assert.ok(suspect.comparisons.some((comparison) => comparison.relation === "equal")); + await dialogue.locator("select").selectOption("markdown"); + const markdown = await dialogue.locator("textarea").inputValue(); + for (const reportText of [text, markdown]) { + for (const secret of [fixtureContent.trim(), passphrase, config.password, config.uri]) { + assert.ok( + !reportText.includes(secret), + "A report exposed fixture contents or credentials" + ); + } + if (names === "pseudonym") { + for (const privateName of [fixturePath, entry.id, shadowId, ...entry.children]) { + assert.ok( + !reportText.includes(privateName), + "A pseudonym report exposed a raw identifier" + ); + } + } else assert.ok(reportText.includes(fixturePath)); + } + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); + } + }); + } finally { + calls = await finishRequests(cli, session); + } + assert.ok(calls.some((call) => call.transport === (native ? "nativeFetch" : "webCompatFetch"))); + assert.ok( + calls.every((call) => call.method === "GET"), + "Diagnostic requests must be read-only" + ); + assert.equal( + await localState(cli, session), + localBefore, + "Diagnostics changed the local database, settings, or Vault" + ); + assert.equal( + await remoteState(config, name), + remoteBefore, + "Diagnostics changed remote documents or Security Seed" + ); + } + console.log( + JSON.stringify({ + encrypted, + transports: ["webCompatFetch", "nativeFetch"], + names: ["pseudonym", "original"], + readOnly: true, + }) + ); + } finally { + try { + if (session) await session.app.stop(); + } finally { + try { + await vault.dispose(); + } finally { + if (created) await deleteCouchDbDatabase(config, name); + } + } + } +} + +async function main() { + const config = await loadCouchDbConfig(); + await assertCouchDbReachable(config); + for (const encrypted of [false, true]) await runCase(config, encrypted); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts index d6e59714..58050847 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -15,11 +15,12 @@ import { createTemporaryVault } from "../runner/vault.ts"; const fixturePath = "Private/diagnostic-į§˜å¯†.md"; const fixtureContent = "Diagnostic fixture content which must never appear in a report.\n"; -const shadowId = "diagnostic-shadow-document"; +const shadowId = "zz-diagnostic-shadow-document"; const timeout = 15000; type DiagnosticPageGlobal = typeof globalThis & { app: { commands: { executeCommandById(id: string): boolean } }; + databaseDiagnosticReadGate?: { reached: boolean }; diagnosticClipboard?: string; diagnosticDownload?: { name: string; content: string }; }; @@ -46,6 +47,36 @@ async function readDatabaseState(cli: string, session: ObsidianLiveSyncSession) ); } +async function installDiagnosticExportInterception(page: Page): Promise { + await page.evaluate(`(() => { + globalThis.diagnosticClipboard = undefined; + globalThis.diagnosticDownload = undefined; + Object.defineProperty(navigator.clipboard, "writeText", { + configurable: true, + value: async (text) => { globalThis.diagnosticClipboard = text; }, + }); + if (!globalThis.databaseDiagnosticExportInterceptionInstalled) { + document.addEventListener("click", (event) => { + const link = event.target; + if (!(link instanceof HTMLAnchorElement) || !link.download || !link.href.startsWith("blob:")) return; + event.preventDefault(); + void fetch(link.href).then((response) => response.text()).then((content) => { + globalThis.diagnosticDownload = { name: link.download, content }; + }); + }, true); + globalThis.databaseDiagnosticExportInterceptionInstalled = true; + } + })()`); +} + +async function restoreDiagnosticReadGate(cli: string, session: ObsidianLiveSyncSession): Promise { + await evalObsidianJson( + cli, + `(async()=>{const gate=globalThis.databaseDiagnosticReadGate;if(gate){gate.release();const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase.localDatabase;db.get=gate.originalGet;delete globalThis.databaseDiagnosticReadGate;}return JSON.stringify(true);})()`, + session.cliEnv + ); +} + async function main(): Promise { const binary = requireObsidianBinary(); const cli = requireObsidianCli(); @@ -56,6 +87,7 @@ async function main(): Promise { binary, cliBinary: cli, vault, + artifactRoot: process.env.E2E_OBSIDIAN_ARTIFACT_ROOT?.trim() || undefined, pluginData: { displayLanguage: "def", doctorProcessedVersion: DoctorRegulation.version, @@ -95,6 +127,7 @@ async function main(): Promise { ); const before = await readDatabaseState(cli, session); let usageJSON = ""; + let usageFixtureRow: { path: string; id: string; revision: string } | undefined; await withObsidianPage(session.remoteDebuggingPort, async (page) => { const choice = await openCommand(page, "usage"); await choice.getByRole("button", { name: "Cancel", exact: true }).click(); @@ -114,23 +147,15 @@ async function main(): Promise { assert.equal(report.status, "complete"); assert.ok(report.versions.commonlib); assert.ok(report.rows.some((row: { path: string }) => row.path.startsWith("file-"))); + const fixtureRow = report.rows.find((row: { path: string; id: string; revision: string }) => + row.path.startsWith("file-") + ); + assert.ok(fixtureRow); + usageFixtureRow = { path: fixtureRow.path, id: fixtureRow.id, revision: fixtureRow.revision }; // Capture exports inside the isolated renderer without changing the // host clipboard or leaving downloads in a user's download directory. - await page.evaluate(`(() => { - Object.defineProperty(navigator.clipboard, "writeText", { - configurable: true, - value: async (text) => { globalThis.diagnosticClipboard = text; }, - }); - document.addEventListener("click", (event) => { - const link = event.target; - if (!(link instanceof HTMLAnchorElement) || !link.download || !link.href.startsWith("blob:")) return; - event.preventDefault(); - void fetch(link.href).then((response) => response.text()).then((content) => { - globalThis.diagnosticDownload = { name: link.download, content }; - }); - }, true); - })()`); + await installDiagnosticExportInterception(page); await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); assert.equal( await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), @@ -197,8 +222,69 @@ async function main(): Promise { await page.locator(".database-diagnostic").getByRole("button", { name: "Close", exact: true }).click(); await page.keyboard.press("Escape"); }); + const cancellationOrder = await evalObsidianJson<{ fixtureId: string; orderedIds: string[] }>( + cli, + `(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;const db=core.localDatabase.localDatabase;return JSON.stringify({fixtureId:await core.services.path.path2id(${JSON.stringify(fixturePath)}),orderedIds:(await db.allDocs({include_docs:false})).rows.map(row=>row.id)});})()`, + session.cliEnv + ); + const fixturePosition = cancellationOrder.orderedIds.indexOf(cancellationOrder.fixtureId); + const gatePosition = cancellationOrder.orderedIds.indexOf(shadowId); + assert.ok(fixturePosition >= 0, "The cancellation fixture must exist in the local database"); + assert.ok(gatePosition >= 0, "The cancellation read gate must exist in the local database"); + assert.ok(fixturePosition < gatePosition, "The cancellation read gate must follow the report fixture"); + assert.ok(usageFixtureRow, "The complete report must contain the cancellation fixture"); + const expectedFixtureRow = usageFixtureRow; + const cancellationEnv = session.cliEnv; + + try { + await evalObsidianJson( + cli, + `(async()=>{const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase.localDatabase;const gate={reached:false,release:null,releasePromise:null,originalGet:db.get};gate.releasePromise=new Promise(resolve=>{gate.release=resolve;});globalThis.databaseDiagnosticReadGate=gate;db.get=async function(id,options){if(id===${JSON.stringify(shadowId)}&&options?.rev&&!gate.reached){gate.reached=true;await gate.releasePromise;}return await gate.originalGet.call(this,id,options);};return JSON.stringify(true);})()`, + session.cliEnv + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await openCommand(page, "usage"); + await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.waitForFunction( + () => (globalThis as DiagnosticPageGlobal).databaseDiagnosticReadGate?.reached === true, + undefined, + { timeout } + ); + assert.equal(await dialogue.getAttribute("data-diagnostic-phase"), "running"); + await dialogue.getByRole("button", { name: "Cancel inspection", exact: true }).click(); + await dialogue.getByText("Cancelling inspectionâ€Ļ", { exact: true }).waitFor({ timeout }); + await evalObsidianJson( + cli, + `(async()=>{globalThis.databaseDiagnosticReadGate?.release();return JSON.stringify(true);})()`, + cancellationEnv + ); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + const cancelledText = await dialogue.locator("textarea").inputValue(); + const cancelledReport = JSON.parse(cancelledText); + assert.equal(cancelledReport.status, "cancelled"); + assert.ok( + cancelledReport.rows.some( + (row: { path: string; id: string; revision: string }) => + row.path === expectedFixtureRow.path && + row.id === expectedFixtureRow.id && + row.revision === expectedFixtureRow.revision + ), + "Cancellation must retain report rows collected before the gated database read" + ); + for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) + assert.ok(!cancelledText.includes(privateValue)); + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); + }); + } finally { + await restoreDiagnosticReadGate(cli, session); + } const after = await readDatabaseState(cli, session); - assert.deepEqual(after, before, "Diagnostics must leave settings, revisions, and Vault files unchanged"); + assert.deepEqual( + after, + before, + "Diagnostics, including cancellation, must leave settings, revisions, and Vault files unchanged" + ); await setObsidianMobileTestMode(session.remoteDebuggingPort, true, timeout); await waitForLiveSyncCoreReady(cli, session.cliEnv); @@ -214,8 +300,68 @@ async function main(): Promise { const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); await assertMobileDialogueLayout(page, modal, "Database diagnostic choice"); await modal.getByRole("button", { name: "Cancel", exact: true }).click(); + await modal.waitFor({ state: "hidden", timeout }); }); - console.log(JSON.stringify({ status: "passed", desktopScreenshot, mobileScreenshot, readOnly: true })); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await openCommand(page, "usage"); + await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); + await assertMobileDialogueLayout(page, modal, "Database diagnostic result"); + + const format = dialogue.locator("select"); + await format.selectOption("json"); + const reportText = await dialogue.locator("textarea").inputValue(); + const report = JSON.parse(reportText); + assert.equal(report.status, "complete"); + for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) + assert.ok(!reportText.includes(privateValue)); + await format.selectOption("tsv"); + const tsv = await dialogue.locator("textarea").inputValue(); + assert.equal(tsv.split("\n")[0].split("\t").length, 9); + assert.ok(!tsv.includes(fixturePath)); + assert.ok(!tsv.includes(shadowId)); + assert.ok(!tsv.includes(fixtureContent.trim())); + await format.selectOption("json"); + }); + const mobileResultScreenshot = await captureObsidianElement( + session.remoteDebuggingPort, + "database-diagnostics-mobile-result.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = page.locator(".database-diagnostic"); + await installDiagnosticExportInterception(page); + const reportText = await dialogue.locator("textarea").inputValue(); + await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); + await dialogue.getByText("Report copied.", { exact: true }).waitFor({ timeout }); + const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); + await assertMobileDialogueLayout(page, modal, "Database diagnostic copied result"); + assert.equal( + await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), + reportText + ); + await dialogue.getByRole("button", { name: "Save report", exact: true }).click(); + await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { + timeout, + }); + assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { + name: "database-usage-complete.json", + content: reportText, + }); + await assertMobileDialogueLayout(page, modal, "Database diagnostic saved result"); + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); + }); + console.log( + JSON.stringify({ + status: "passed", + desktopScreenshot, + mobileScreenshot, + mobileResultScreenshot, + readOnly: true, + }) + ); } catch (error) { if (session) { await captureObsidianPage( diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 7ce9514e..7f766c67 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -9,6 +9,7 @@ const focusedScenarios = new Set([ "onboarding-invitation", "dialog-mounts", "database-diagnostics", + "database-diagnostics-couchdb", "conflict-dialog-policy", "revision-repair", "document-history-nav", diff --git a/test/e2e-obsidian/scripts/two-vault-sync.ts b/test/e2e-obsidian/scripts/two-vault-sync.ts index f5ef0174..8771ba66 100644 --- a/test/e2e-obsidian/scripts/two-vault-sync.ts +++ b/test/e2e-obsidian/scripts/two-vault-sync.ts @@ -1268,6 +1268,7 @@ async function runIndependentIdRoundTrip( rejectedSession = await startObsidianLiveSyncSession({ binary: context.binary, cliBinary: context.cliBinary, + artifactRoot: process.env.E2E_OBSIDIAN_ARTIFACT_ROOT, vault: rejectedVault, localStorageEntries: createE2eObsidianDeviceLocalState(rejectedVault.name), pluginData: createE2eCouchDbPluginData( From 7c59c9adbc6b2f961497ae9d68feb3cd7ea1de63 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 18:22:23 +0000 Subject: [PATCH 19/68] Document diagnostic actions in settings and issue reporting --- docs/settings.md | 8 ++++++++ docs/to_issue_reporting.md | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/docs/settings.md b/docs/settings.md index c742b975..91101117 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -830,6 +830,14 @@ Select a file to copy its local database information. The command **Copy databas The report includes the Vault-relative path, document and chunk identifiers, local database revisions, conflicts, and local chunk availability. It does not query the remote or include file contents. Paths and identifiers can still be private metadata, so review the report before sharing it. +#### Inspect database integrity + +Select **Inspect** to choose original names or temporary pseudonyms and inspect normal-file Metadata and current conflicts locally and in CouchDB. The command **Inspect database integrity** opens the same dialogue. The inspection is read-only; its findings do not authorise deletion or perform repairs. See [Collect a report](troubleshooting.md#collect-a-report) for coverage, report formats, privacy, and cancellation. + +#### Analyse database usage + +Select **Analyse** to choose a name policy and report local revision and Chunk usage. The command **Analyse Database Usage (advanced)** opens the same dialogue when [advanced features](#enable-advanced-features) are enabled. Complete runs offer TSV and JSON; incomplete runs retain their status and reasons in JSON. Existing on-demand Chunk fetching stays unchanged. See [Collect a report](troubleshooting.md#collect-a-report) for accounting limits and export instructions. + #### Write logs into the file Setting key: writeLogToTheFile diff --git a/docs/to_issue_reporting.md b/docs/to_issue_reporting.md index 7512f8a9..1643f1c3 100644 --- a/docs/to_issue_reporting.md +++ b/docs/to_issue_reporting.md @@ -44,6 +44,10 @@ For a large report, you may share a GitHub Gist after reviewing and redacting it ## Additional evidence when relevant +### A problem involving database integrity or usage + +Use **Hatch** → **Inspect database integrity** for database-wide consistency findings, or **Analyse database usage** for local revision and Chunk accounting. Follow [Collect a report](troubleshooting.md#collect-a-report) for the name choice, formats, and inspection limits. Temporary pseudonyms are the default; review the report before sharing it. Retain the status and reported reasons when sharing a partial or cancelled result. + ### A problem involving one file Run `Copy database information for the active file`, or use **Hatch** → **Copy database information for a file** to select another file. From b332be69245585799cd1d3b802feca0bd5df7eec Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 18:49:54 +0000 Subject: [PATCH 20/68] Show diagnostic name and report format choices as radio buttons --- docs/troubleshooting.md | 4 +- .../dialogues/DatabaseDiagnosticDialog.svelte | 44 ++++++---- test/e2e-obsidian/README.md | 2 +- .../scripts/database-diagnostics-couchdb.ts | 16 +++- .../scripts/database-diagnostics.ts | 88 ++++++++++++++++--- updates.md | 2 +- 6 files changed, 122 insertions(+), 34 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index afd14d67..e8649667 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -163,9 +163,9 @@ Enable Obsidian's `Detect all file extensions`, then check LiveSync selectors, i ## Collect a report -For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. The inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete. +For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. Both choices are visible as radio buttons, with pseudonyms selected initially. The inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete. -The result dialogue provides Markdown and JSON reports with the same name policy. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. They are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled. +The result dialogue provides Markdown and JSON reports with the same name policy. Available formats are displayed as radio buttons. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. They are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled. For storage accounting, use **Hatch** → **Analyse database usage** → **Analyse**. It offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads. diff --git a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte index 87c85856..b7b56b5f 100644 --- a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte @@ -90,13 +90,19 @@ ? translate("Read available local revisions and report their Chunk references. Settings are left unchanged.") : translate("Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.")}

- +
+ {translate("Names in this report")} +
+ + +
+
@@ -118,14 +124,17 @@ {#if options?.kind === "usage" && output.status !== "complete"}

{translate("Partial Usage results are available as JSON so their coverage and reasons stay with the rows.")}

{/if} - {/each} - - +
+

{actionStatus}

@@ -142,9 +151,14 @@ diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index e48d9220..0a31cae4 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -87,7 +87,7 @@ npm run test:e2e:obsidian:focused -- couchdb-http-cache The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. -`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including the choice before inspection, original names and pseudonyms, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking result layout, format selection, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. +`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. `database-diagnostics-couchdb` needs the CouchDB fixture with CORS enabled. It checks unencrypted and encrypted databases, both browser fetch and Obsidian's native Request API, and both name policies through the real integrity dialogue. A remote-only document with a mismatched ID must produce a finding with readable Chunks and equal content. JSON and Markdown must omit file contents and credentials. The scenario observes real requests and requires GET-only access, unchanged local settings, revisions, and Vault contents, and unchanged remote documents and Security Seed. Commonlib's separate CouchDB integration tests additionally cover a missing Security Seed without creating one. diff --git a/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts b/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts index cf1276f7..d51279bf 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts @@ -171,10 +171,19 @@ async function runCase(config: CouchDbConfig, encrypted: boolean) { ); assert.equal(opened, true); const dialogue = page.locator(".database-diagnostic"); - await dialogue.locator("select").selectOption(names); + await dialogue.waitFor({ state: "visible", timeout }); + const nameOptions = dialogue.getByRole("group", { name: "Names in this report", exact: true }); + assert.equal(await nameOptions.getByRole("radio").count(), 2); + const nameLabel = names === "pseudonym" ? "Pseudonyms (recommended)" : "Original names"; + await nameOptions.getByRole("radio", { name: nameLabel, exact: true }).check(); + assert.equal(await nameOptions.getByRole("radio", { checked: true }).count(), 1); await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); - await dialogue.locator("select").selectOption("json"); + const formats = dialogue.getByRole("group", { name: "Report format", exact: true }); + assert.equal(await formats.getByRole("radio").count(), 2); + for (const name of ["MARKDOWN", "JSON"]) + assert.equal(await formats.getByRole("radio", { name, exact: true }).isVisible(), true); + await formats.getByRole("radio", { name: "JSON", exact: true }).check(); const text = await dialogue.locator("textarea").inputValue(); const report = JSON.parse(text) as IntegrityReport; assert.equal(report.status, "complete"); @@ -189,7 +198,8 @@ async function runCase(config: CouchDbConfig, encrypted: boolean) { assert.equal(suspect.content.status, "available"); assert.ok(suspect.content.chunksRead > 0); assert.ok(suspect.comparisons.some((comparison) => comparison.relation === "equal")); - await dialogue.locator("select").selectOption("markdown"); + await formats.getByRole("radio", { name: "MARKDOWN", exact: true }).check(); + assert.equal(await formats.getByRole("radio", { checked: true }).count(), 1); const markdown = await dialogue.locator("textarea").inputValue(); for (const reportText of [text, markdown]) { for (const secret of [fixtureContent.trim(), passphrase, config.password, config.uri]) { diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts index 58050847..d4f44ae9 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -1,5 +1,10 @@ import assert from "node:assert/strict"; -import type { Page } from "playwright"; +import type { Locator, Page } from "playwright"; +import { + assertLocatorHasMinimumTouchTarget, + assertLocatorWithinSafeArea, + assertNoHorizontalOverflow, +} from "@vrtmrz/obsidian-test-session"; import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc"; import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; import { evalObsidianJson } from "../runner/cli.ts"; @@ -10,7 +15,7 @@ import { } from "../runner/liveSyncWorkflow.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { captureObsidianElement, captureObsidianPage, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts"; -import { assertMobileDialogueLayout, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; +import { assertMobileDialogueLayout, iPhoneSafeArea, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; import { createTemporaryVault } from "../runner/vault.ts"; const fixturePath = "Private/diagnostic-į§˜å¯†.md"; @@ -33,12 +38,48 @@ async function openCommand(page: Page, kind: "usage" | "integrity") { ); assert.equal(opened, true); const dialogue = page.locator(".database-diagnostic"); - await dialogue.locator("select").selectOption("pseudonym"); + await dialogue.waitFor({ state: "visible", timeout }); + assert.equal( + await dialogue.getByRole("group", { name: "Names in this report", exact: true }).getByRole("radio").count(), + 2, + "Both name choices must be visible as radio buttons before inspection" + ); + const names = dialogue.getByRole("group", { name: "Names in this report", exact: true }); + for (const name of ["Pseudonyms (recommended)", "Original names"]) + assert.equal(await names.getByRole("radio", { name, exact: true }).isVisible(), true); + assert.equal(await names.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).isChecked(), true); + assert.equal(await names.getByRole("radio", { checked: true }).count(), 1); assert.equal(await dialogue.getAttribute("data-diagnostic-phase"), "choice"); assert.equal(await dialogue.locator("textarea").count(), 0); return dialogue; } +async function assertFormatChoices(dialogue: Locator, formats: string[]): Promise { + const group = dialogue.getByRole("group", { name: "Report format", exact: true }); + assert.equal(await group.getByRole("radio").count(), formats.length); + for (const name of formats) assert.equal(await group.getByRole("radio", { name, exact: true }).isVisible(), true); + assert.equal(await group.getByRole("radio", { checked: true }).count(), 1); +} + +async function chooseFormat(dialogue: Locator, format: string): Promise { + const group = dialogue.getByRole("group", { name: "Report format", exact: true }); + const radio = group.getByRole("radio", { name: format.toUpperCase(), exact: true }); + await radio.check(); + assert.equal(await radio.isChecked(), true); + assert.equal(await group.getByRole("radio", { checked: true }).count(), 1); +} + +async function assertMobileRadioTargets(page: Page, dialogue: Locator, name: string): Promise { + const radios = dialogue.getByRole("group", { name, exact: true }).getByRole("radio"); + for (let index = 0; index < (await radios.count()); index++) { + const target = radios.nth(index).locator(".."); + const label = `${name}: ${(await target.innerText()).trim()}`; + await assertLocatorHasMinimumTouchTarget(page, target, { label }); + await assertLocatorWithinSafeArea(page, target, { label, safeAreaInsets: iPhoneSafeArea }); + await assertNoHorizontalOverflow(page, target, { label }); + } +} + async function readDatabaseState(cli: string, session: ObsidianLiveSyncSession) { return await evalObsidianJson<{ sequence: unknown; settings: string; names: string[] }>( cli, @@ -130,18 +171,33 @@ async function main(): Promise { let usageFixtureRow: { path: string; id: string; revision: string } | undefined; await withObsidianPage(session.remoteDebuggingPort, async (page) => { const choice = await openCommand(page, "usage"); + await choice.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).focus(); + await page.keyboard.press("ArrowRight"); + assert.equal(await choice.getByRole("radio", { name: "Original names", exact: true }).isChecked(), true); + await page.keyboard.press("ArrowLeft"); + assert.equal( + await choice.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).isChecked(), + true + ); await choice.getByRole("button", { name: "Cancel", exact: true }).click(); await choice.waitFor({ state: "hidden", timeout }); const dialogue = await openCommand(page, "usage"); - assert.equal(await dialogue.locator("select").inputValue(), "pseudonym"); + assert.equal( + await dialogue.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).isChecked(), + true + ); await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await assertFormatChoices(dialogue, ["TSV", "JSON"]); + assert.equal(await dialogue.getByRole("radio", { name: "TSV", exact: true }).isChecked(), true); const tsv = await dialogue.locator("textarea").inputValue(); assert.equal(tsv.split("\n")[0].split("\t").length, 9); assert.ok(!tsv.includes(fixturePath)); assert.ok(!tsv.includes(shadowId)); assert.ok(!tsv.includes(fixtureContent.trim())); - await dialogue.locator("select").selectOption("json"); + await dialogue.getByRole("radio", { name: "TSV", exact: true }).focus(); + await page.keyboard.press("ArrowRight"); + assert.equal(await dialogue.getByRole("radio", { name: "JSON", exact: true }).isChecked(), true); usageJSON = await dialogue.locator("textarea").inputValue(); const report = JSON.parse(usageJSON); assert.equal(report.status, "complete"); @@ -173,7 +229,7 @@ async function main(): Promise { await dialogue.waitFor({ state: "hidden", timeout }); const original = await openCommand(page, "usage"); - await original.locator("select").selectOption("original"); + await original.getByRole("radio", { name: "Original names", exact: true }).check(); await original.getByRole("button", { name: "Begin inspection", exact: true }).click(); await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); const rawTSV = await original.locator("textarea").inputValue(); @@ -190,10 +246,15 @@ async function main(): Promise { .filter({ has: page.getByText("Inspect database integrity", { exact: true }) }); await setting.getByRole("button", { name: "Inspect", exact: true }).click(); const integrity = page.locator(".database-diagnostic"); - assert.equal(await integrity.locator("select").inputValue(), "pseudonym"); + assert.equal( + await integrity.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).isChecked(), + true + ); await integrity.getByRole("button", { name: "Begin inspection", exact: true }).click(); await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); - await integrity.locator("select").selectOption("json"); + await assertFormatChoices(integrity, ["MARKDOWN", "JSON"]); + assert.equal(await integrity.getByRole("radio", { name: "MARKDOWN", exact: true }).isChecked(), true); + await chooseFormat(integrity, "json"); const reportText = await integrity.locator("textarea").inputValue(); const integrityReport = JSON.parse(reportText); assert.equal(integrityReport.status, "partial"); @@ -262,6 +323,7 @@ async function main(): Promise { const cancelledText = await dialogue.locator("textarea").inputValue(); const cancelledReport = JSON.parse(cancelledText); assert.equal(cancelledReport.status, "cancelled"); + await assertFormatChoices(dialogue, ["JSON"]); assert.ok( cancelledReport.rows.some( (row: { path: string; id: string; revision: string }) => @@ -299,6 +361,7 @@ async function main(): Promise { await withObsidianPage(session.remoteDebuggingPort, async (page) => { const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); await assertMobileDialogueLayout(page, modal, "Database diagnostic choice"); + await assertMobileRadioTargets(page, modal, "Names in this report"); await modal.getByRole("button", { name: "Cancel", exact: true }).click(); await modal.waitFor({ state: "hidden", timeout }); }); @@ -309,20 +372,21 @@ async function main(): Promise { const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); await assertMobileDialogueLayout(page, modal, "Database diagnostic result"); - const format = dialogue.locator("select"); - await format.selectOption("json"); + await assertFormatChoices(dialogue, ["TSV", "JSON"]); + await assertMobileRadioTargets(page, dialogue, "Report format"); + await chooseFormat(dialogue, "json"); const reportText = await dialogue.locator("textarea").inputValue(); const report = JSON.parse(reportText); assert.equal(report.status, "complete"); for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) assert.ok(!reportText.includes(privateValue)); - await format.selectOption("tsv"); + await chooseFormat(dialogue, "tsv"); const tsv = await dialogue.locator("textarea").inputValue(); assert.equal(tsv.split("\n")[0].split("\t").length, 9); assert.ok(!tsv.includes(fixturePath)); assert.ok(!tsv.includes(shadowId)); assert.ok(!tsv.includes(fixtureContent.trim())); - await format.selectOption("json"); + await chooseFormat(dialogue, "json"); }); const mobileResultScreenshot = await captureObsidianElement( session.remoteDebuggingPort, diff --git a/updates.md b/updates.md index 977d24dd..dcdc8352 100644 --- a/updates.md +++ b/updates.md @@ -22,7 +22,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Diagnostics - We can inspect normal-file Metadata and current conflicts in the local database and CouchDB together, with byte comparisons for suspect groups. Partial results retain the available evidence when a source is unavailable. The inspection leaves databases and settings unchanged. -- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons. +- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Both name modes and the available report formats stay visible as radio buttons. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons. ## 1.0.38 From 32874ea97f28f2bb44cfcd1cee0d4dc781d63333 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 20 Oct 2026 08:23:56 +0000 Subject: [PATCH 21/68] Polish diagnostic reporting and local pseudonym follow-up --- docs/settings.md | 44 +- docs/to_issue_reporting.md | 95 +++- docs/troubleshooting.md | 36 +- .../messages/LiveSyncProvisionalMessages.ts | 46 +- .../features/SettingDialogue/PaneHatch.ts | 107 +++-- .../dialogues/DatabaseDiagnosticDialog.svelte | 159 ++++++- .../DiagnosticFileLookupDialog.svelte | 367 ++++++++++++++++ .../databaseDiagnosticDialogTypes.ts | 12 +- src/serviceFeatures/databaseDiagnostics.ts | 88 +++- .../databaseDiagnostics.unit.spec.ts | 148 ++++++- src/serviceFeatures/diagnosticFileLookup.ts | 346 +++++++++++++++ .../diagnosticFileLookup.unit.spec.ts | 362 +++++++++++++++ .../diagnosticFileLookupTypes.ts | 36 ++ src/serviceFeatures/fileDatabaseInfo.ts | 89 ++-- .../fileDatabaseInfo.unit.spec.ts | 87 +++- src/serviceFeatures/useDatabaseDiagnostics.ts | 3 +- .../useDatabaseDiagnostics.unit.spec.ts | 25 ++ test/e2e-obsidian/README.md | 8 +- .../scripts/database-diagnostics-couchdb.ts | 100 ++++- .../scripts/database-diagnostics.ts | 413 +++++++++++++++++- updates.md | 8 +- 21 files changed, 2425 insertions(+), 154 deletions(-) create mode 100644 src/modules/services/LiveSyncUI/dialogues/DiagnosticFileLookupDialog.svelte create mode 100644 src/serviceFeatures/diagnosticFileLookup.ts create mode 100644 src/serviceFeatures/diagnosticFileLookup.unit.spec.ts create mode 100644 src/serviceFeatures/diagnosticFileLookupTypes.ts diff --git a/docs/settings.md b/docs/settings.md index 91101117..0885589c 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -820,9 +820,13 @@ Open the dialogue ## 7. Hatch -### 1. Reporting Issue +### 1. Reporting Tools -#### Make report to inform the issue +This panel groups the five actions that collect reports or inspect database state. These actions provide diagnostic information; they do not repair or reset the database. + +#### Prepare a report to inform an issue + +Select **Copy Report to clipboard** to collect diagnostic information and recent logs for a support report. Review the report before sharing it because it can contain sensitive information. #### Copy database information for a file @@ -830,20 +834,48 @@ Select a file to copy its local database information. The command **Copy databas The report includes the Vault-relative path, document and chunk identifiers, local database revisions, conflicts, and local chunk availability. It does not query the remote or include file contents. Paths and identifiers can still be private metadata, so review the report before sharing it. +#### Find a file from a report pseudonym + +Paste a path pseudonym from a diagnostic report, then select **Find file**. The lookup checks the current Vault and local normal-file Metadata only; it does not contact the remote database. It returns every matching local candidate, and shows each candidate's real Vault path and document ID only in this local dialogue. Multiple local document IDs for one path remain separate candidates. The follow-up for a Vault-only candidate marks Metadata as not inspected, because no local document was selected. No match can mean that the file was renamed or deleted, or that the matching Metadata exists only remotely. + +For a selected candidate, open an existing Vault file or prepare a shareable local report using the original report's pseudonym salt and the exact selected document ID. The report includes current, conflict, and history status, Chunk availability, merge bases, and Vault size facts; it does not include file contents or credentials. Its history status listing does not scan every historical body. + +Path pseudonyms use the report's public eight-character salt. Someone with the report can test a guessed path against a pseudonym. The same salt reproduces path and identifier pseudonyms in a targeted follow-up; older ordinal pseudonyms cannot be looked up with this tool. + #### Inspect database integrity -Select **Inspect** to choose original names or temporary pseudonyms and inspect normal-file Metadata and current conflicts locally and in CouchDB. The command **Inspect database integrity** opens the same dialogue. The inspection is read-only; its findings do not authorise deletion or perform repairs. See [Collect a report](troubleshooting.md#collect-a-report) for coverage, report formats, privacy, and cancellation. +Select **Inspect** to choose original names or temporary pseudonyms and inspect normal-file Metadata and current conflicts. **Include remote database** is selected by default; clear it to inspect only the local database and Vault. This choice applies to the current run only, and the report marks remote coverage as not checked when excluded. The command **Inspect database integrity** opens the same dialogue. The inspection is read-only; its findings do not authorise deletion or perform repairs. See [Collect a report](troubleshooting.md#collect-a-report) for coverage, report formats, privacy, and cancellation, and the [compact Markdown reading guide](troubleshooting.md#read-the-compact-markdown-integrity-report) for interpreting its state and review index. #### Analyse database usage Select **Analyse** to choose a name policy and report local revision and Chunk usage. The command **Analyse Database Usage (advanced)** opens the same dialogue when [advanced features](#enable-advanced-features) are enabled. Complete runs offer TSV and JSON; incomplete runs retain their status and reasons in JSON. Existing on-demand Chunk fetching stays unchanged. See [Collect a report](troubleshooting.md#collect-a-report) for accounting limits and export instructions. +### 2. Checks and Recovery + +This panel groups checks that may lead to changes to settings or database contents. + +#### Setting Doctor + +Run Setting Doctor to check for configuration mismatches or suboptimal settings. Applying its recommendations can change settings or request a database rebuild, so review the recommendations before proceeding. + +#### Scan for broken files + +Scan for files which are not stored correctly in the database. The scan can offer repair actions that change database contents; review each proposed action before confirming it. + +### 3. TroubleShooting + +This panel keeps the remote storage notification check and file logging controls separate from report collection and recovery checks. + +#### Reset notification threshold and check the remote database usage + +Reset the remote storage size threshold and check the remote storage size again. This changes the saved notification threshold. + #### Write logs into the file Setting key: writeLogToTheFile Warning! This will have a serious impact on performance. And the logs will not be synchronised under the default name. Please be careful with logs; they often contain your confidential information. -### 2. Scram Switches +### 4. Scram Switches Emergency controls to suspend synchronisation processes in order to prevent database corruption. If a critical mismatch or sync error occurs, the plug-in may automatically enter a Scram state and suspend operations. @@ -859,7 +891,7 @@ Setting key: suspendParseReplicationResult Stop reflecting database changes to storage files. -### 3. Recovery and Repair +### 5. Recovery and Repair #### Recreate chunks for current Vault files @@ -879,7 +911,7 @@ After confirmation, resolve every conflict by modification time. This logically #### Check and convert non-path-obfuscated files -### 4. Reset +### 6. Reset #### Back to non-configured diff --git a/docs/to_issue_reporting.md b/docs/to_issue_reporting.md index 1643f1c3..dc64edc1 100644 --- a/docs/to_issue_reporting.md +++ b/docs/to_issue_reporting.md @@ -4,6 +4,16 @@ Thank you for helping improve Self-hosted LiveSync. A concise report with the ri Use the [issue report template](https://github.com/vrtmrz/obsidian-livesync/issues/new?template=issue-report.md) for the report itself. Use [Troubleshooting](troubleshooting.md) to diagnose a symptom or choose a recovery action. +If a maintainer requests additional evidence, follow the relevant procedure below. These reports supplement the required information in this guide. + +| Requested evidence | Procedure | +| --------------------------------------------------------------------- | ------------------------------------- | +| Database integrity, current conflicts, or differences between sources | [Integrity report](#integrity-report) | +| Local revision and Chunk usage | [Usage report](#usage-report) | +| More information about a path pseudonym in an existing report | [File follow-up](#file-follow-up) | + +Maintainers can link directly to each procedure and adapt the [example requests](#requesting-diagnostics). + ## Preserve the original symptom Do not reset a database, rebuild a remote, change transport, or enable P2P merely to see whether the problem disappears. These actions can change the evidence and may make the original cause harder to identify. @@ -46,7 +56,7 @@ For a large report, you may share a GitHub Gist after reviewing and redacting it ### A problem involving database integrity or usage -Use **Hatch** → **Inspect database integrity** for database-wide consistency findings, or **Analyse database usage** for local revision and Chunk accounting. Follow [Collect a report](troubleshooting.md#collect-a-report) for the name choice, formats, and inspection limits. Temporary pseudonyms are the default; review the report before sharing it. Retain the status and reported reasons when sharing a partial or cancelled result. +Follow the [Integrity report](#integrity-report) procedure for database-wide consistency findings, or the [Usage report](#usage-report) procedure for local revision and Chunk accounting. [Collect a report](troubleshooting.md#collect-a-report) explains the inspection limits and how to read the results. ### A problem involving one file @@ -54,6 +64,8 @@ Run `Copy database information for the active file`, or use **Hatch** → **Copy This report describes only the local database on that device. It includes the Vault-relative path, document and chunk identifiers, local revisions, conflicts, and local chunk availability. It does not query the remote or include file contents. Review paths and identifiers as private metadata before sharing them. +If the request identifies a path pseudonym from an existing report, use [File follow-up](#file-follow-up) to retain that report's pseudonyms. + ### A problem which crosses a restart Use `Write logs into the file` under **Hatch** only when the in-memory report cannot cover the restart. Persistent logging affects performance and can record private information. Disable it after reproducing the problem, review the log before sharing it, and remove the log file when it is no longer needed. @@ -62,8 +74,89 @@ Use `Write logs into the file` under **Hatch** only when the in-memory report ca Include network evidence only when the ordinary LiveSync log cannot show the rejected response. Follow [Inspect a network failure](troubleshooting.md#inspect-a-network-failure), and remove request paths, remote addresses, authority and authorisation values, cookies, credentials, payload identifiers, and response secrets before sharing screenshots or copied data. +## Requested diagnostic reports + +Use the device and Vault named in the request. Label each result with a device role such as 'Device A: edits originate here', and include when it was collected. If an action below is unavailable, report the installed plug-in version and the missing action. + +These reporting tools inspect existing state without repairing or resetting it. Preserve the original symptom as described above. + +### Integrity report + +1. Open Self-hosted LiveSync settings → **Hatch** → **Reporting Tools** → **Inspect database integrity**, then select **Inspect**. +2. Keep **Pseudonyms (recommended)** selected. +3. Leave **Include remote database** selected unless the request specifies a local-only inspection. For a local-only inspection, clear it. This controls diagnostic requests for this run; it does not change normal synchronisation settings. +4. Select **Begin inspection**. If you cancel, retain the resulting report and its cancellation status. +5. Choose **MARKDOWN**, then **Copy report** or **Save report**. Share the complete output after reviewing it. Use **JSON** instead if the maintainer requests it. + +Keep the status, versions, settings, scope, coverage, counts, findings, and errors together. A partial or cancelled result is useful evidence. If the report is too large for a comment, share the saved report or a reviewed Gist, and identify its device role in the issue. + +The remote inspection currently supports CouchDB. An excluded or unavailable remote remains visible in the report; changing remote settings is not part of this procedure. Content checks cover suspect groups, so `not-checked` is meaningful. A completed inspection does not establish that every file is healthy. See [Read the compact Markdown integrity report](troubleshooting.md#read-the-compact-markdown-integrity-report) for the state notation and Review index. + +### Usage report + +1. Open Self-hosted LiveSync settings → **Hatch** → **Reporting Tools** → **Analyse database usage**, then select **Analyse**. +2. Keep **Pseudonyms (recommended)** selected, then select **Begin inspection**. +3. For a complete result, choose **TSV**, unless the maintainer requests **JSON**. Partial or cancelled results use **JSON** to retain their status and reasons. +4. Select **Copy report** or **Save report**, review the output, and share it with the device role and collection time. + +This analysis reads the local database. Its revision-reading total can grow during discovery, followed by a separate accounting stage. Reported sizes describe stored string units and do not measure the physical database file. Cancelling during accounting omits unfinished statistics. + +### File follow-up + +Use this procedure when a maintainer identifies a path token from an Integrity or Usage report. The complete token has the form `$`. Copy the token itself, including the salt and `$`, without surrounding Markdown backticks. An observation reference such as `local:12` or `remote:12`, a document pseudonym, or a revision pseudonym is not a path token. + +1. On the requested device, open Self-hosted LiveSync settings → **Hatch** → **Reporting Tools** → **Find a file from a report pseudonym**, then select **Find file**. +2. Paste the complete token into **Report path pseudonym**, then select **Find file** in the dialogue. +3. For a matching candidate, select **Prepare follow-up report**. +4. Choose **MARKDOWN** or **JSON**, then **Copy report** or **Save report**. Review the generated output before sharing it, and identify the original report, token, and device role. +5. If several candidates match the path, use **Back to matches** and prepare a separate report for each, unless the maintainer has specified a particular candidate. Keep the reports separate so their document IDs can be compared. + +Use the token from the original request throughout. The follow-up reuses its salt, which keeps pseudonyms comparable with the original report. A new database-wide report generates a new salt, so its pseudonyms will differ. + +The candidate list displays real paths and document IDs locally. Share the generated pseudonymised report rather than a screenshot of that list. A Metadata candidate can still be inspected when its current Vault file is missing. A Vault-only candidate reports that local Metadata was not inspected. + +If there are no matches, report that outcome and the device role. The lookup searches current Vault paths and current local normal-file Metadata; it does not search the remote or older history. A rename, deletion, or remote-only path can therefore have no local match. An older ordinal pseudonym cannot be used for this lookup. + +The follow-up report inspects the selected local document, current conflicts, and selected merge bases, and lists available revision-history status. It omits file contents and credentials, does not query the remote, and does not scan every historical file body. Pseudonyms use a public salt, so someone with the report can test guessed paths against a token. + ## Sharing the report Paste reports into the matching collapsible sections in the issue template, or provide a link to an already-redacted Gist. A separate plug-in log is normally unnecessary because the full LiveSync report already contains the recent verbose log history. +Integrity and file follow-up Markdown already include a collapsible section and a code block. Paste that output directly into an issue comment, outside any existing code block. Keep each device's report separate, and preserve the original token when adding a follow-up. + If a maintainer asks for a more specialised diagnostic, collect only that additional evidence and review it again before publishing it. + +## Requesting diagnostics + +For maintainers: choose the procedure which answers the current question, specify the device role, and replace the placeholders before posting a request. The examples use direct links to the relevant steps. For Integrity, choose one remote setting explicitly; inclusion is the default. If the requested action requires a newer build, identify the version to use. + +### Request an integrity report + +```markdown +Would you be willing to collect an [Integrity report](https://github.com/vrtmrz/obsidian-livesync/blob/main/docs/to_issue_reporting.md#integrity-report) on **[DEVICE ROLE]**, using **Pseudonyms (recommended)** with **Include remote database [selected / cleared]**? + +Please share the complete Markdown output after reviewing it, including its status, scope, coverage, and errors. A partial or cancelled result is useful too. Include the device role and collection time. +``` + +### Request a usage report + +```markdown +Would you be willing to collect a [Usage report](https://github.com/vrtmrz/obsidian-livesync/blob/main/docs/to_issue_reporting.md#usage-report) on **[DEVICE ROLE]**, using **Pseudonyms (recommended)**? + +Please share the reviewed TSV output for a complete result, or JSON for a partial or cancelled result, with the device role and collection time. +``` + +### Request a file follow-up + +Include the original report link or label, its observation reference where available, and the complete path token. Keep the revision generation and document pseudonym available when discussing a particular branch. A reference such as `remote:12` locates an observation within that report; the path token is what the user enters into the lookup. + +```markdown +Would you be willing to follow the [File follow-up steps](https://github.com/vrtmrz/obsidian-livesync/blob/main/docs/to_issue_reporting.md#file-follow-up) on **[DEVICE ROLE]** for this entry? + +- Original report: [REPORT LINK OR LABEL] +- Observation: `[REFERENCE, IF AVAILABLE]` +- Path token: `[COMPLETE SALT$HASH TOKEN]` + +Please share the generated pseudonymised Markdown or JSON report. If several candidates match, share a separate report for each. If none match, tell us that outcome. Include the device role and collection time; the generated reports are sufficient without sharing real paths or a screenshot of the candidate list. +``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e8649667..c68fe8fc 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -163,9 +163,37 @@ Enable Obsidian's `Detect all file extensions`, then check LiveSync selectors, i ## Collect a report -For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. Both choices are visible as radio buttons, with pseudonyms selected initially. The inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete. +For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. Both choices are visible as radio buttons, with pseudonyms selected initially. **Include remote database** is selected by default; clear it to inspect the local database and current Vault files without making diagnostic requests to the remote. This choice applies to one run, and the report marks the remote as excluded with coverage not checked. When included, the inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete. -The result dialogue provides Markdown and JSON reports with the same name policy. Available formats are displayed as radio buttons. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. They are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled. +The result dialogue provides Markdown and JSON reports with the same name policy. Available formats are displayed as radio buttons. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. Each report uses a random eight-character public salt; path tokens are derived from the exact stored Vault-relative path without lower-casing or Unicode NFC normalisation. The same salt reproduces path, document, revision, and Chunk pseudonyms for a targeted follow-up. Because the salt is public, a guessed path can be checked against its token. Earlier ordinal pseudonyms cannot be looked up this way. Pseudonyms are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled. + +Integrity observations include recorded Metadata type, size, and total, unique, and embedded Chunk-reference counts. When suspect content has already been read, they also include measured decoded and Vault sizes, their differences from recorded or reconstructed sizes, and the Vault-to-Metadata modification-time difference. Case-only and NFC-equivalent path groups are listed once. The inspection uses content already needed for its comparisons and does not add database, HTTP, or Vault reads. Values which were not inspected remain absent or `not-checked`. + +### Read the compact Markdown integrity report + +Markdown keeps its status, settings, scope, coverage, and count summary, and adds separate counts for content and Vault checks which were not performed. **State counts** count observations, not unique files; each local or remote observation contributes to its state total. The **Review index** lists references to observations which match review conditions, before **Findings**. Those conditions include identity mismatches, current conflicts, case- or NFC-equivalent paths, non-NFC paths, a missing group in the other inspected database, missing or unreadable content or Vault files, unavailable Chunks, different or unavailable comparisons, non-zero measured differences, and absent Metadata. These are review candidates, not proof of corruption. A report with no review-index entry still does not confirm that unchecked content is healthy. JSON data and its schema are unchanged. + +Each observation is shown in a compact form, while retaining every observation and its full paths, IDs, revisions, and references for follow-up: + +```text +### `remote:4410` source=remote +path: `$` +doc: `` rev=`` expected=self +state: consistent-winner-logicallyLive-NFC-newnote conflicts=0 +paths: case=none nfcPeers=none | otherDB=present +data: recorded=47584B chunks=1/1/0 read=0 +check: content=not-checked vault=not-checked@`vault:remote:4410` +``` + +The `state` values always appear in this order: identity, current branch, logical deletion, exact stored path normalisation, and Metadata type. Identity is `consistent`, `documentIdMismatch`, or `namespaceMismatch`; JSON retains `document-id-mismatch` and `namespace-mismatch`. `winner` is the selected current branch and can coexist with conflicts, shown by `conflicts`. `logicallyLive` and `logicallyDeleted` describe the deletion marker. `NFC` means the stored path is NFC-normalised; `nonNFC` means it is not. The Metadata type is `plain`, `newnote`, `notes`, or `unknown`; `notRecorded` means Metadata is absent. A `consistent` identity says only that the ID matches its expected form; it does not show that content is readable or equal across sources. + +In `paths`, `case` and `nfcPeers` describe case-equivalent and NFC-equivalent paths: `none` means no equivalent path was found, `yes` means one was found, and `yes@` followed by a reference identifies the available group. `otherDB` says whether the same identity and path group occurs in the other inspected source. It does not compare content. If that source was excluded or unavailable, the field is `not-checked`. + +In `data`, `chunks` is total/unique/embedded Chunk references, and `read` counts Chunk reads, not HTTP requests. A `?` means a value was not recorded; it is distinct from zero, missing, unreadable, and `not-checked`. If Metadata is absent, its recorded size and Chunk counts are `?`. In `doc`, `expected=self` means the actual document ID exactly matches the expected ID. Otherwise, the report shows the expected ID or `?` when it was not recorded. + +The `check` statuses distinguish `available`, `missing`, `unreadable`, and `not-checked`; a recorded reason follows the status in parentheses. An optional `measure` line includes only available values: decoded content size and Vault size in bytes, `deltaContent` (decoded content minus recorded Metadata size), `deltaVault` (Vault minus decoded content), and `deltaMtime` (Vault minus Metadata modification time) in milliseconds. Positive differences have a `+` prefix, negative differences retain `-`, and zero is `0`. These measurements describe the evidence collected and do not select an authoritative version. Unavailable Chunk names, comparison results, path-equivalence groups, and errors retain their reasons and references. All observations remain in the report's collapsible section and fenced copy. + +Usage progress first follows available-revision discovery, so its `n/m` total can grow and remains provisional until discovery ends. It then shows separate progress for accounting. Cancellation during revision reading keeps the collected evidence; cancellation during accounting omits incomplete statistics. For storage accounting, use **Hatch** → **Analyse database usage** → **Analyse**. It offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads. @@ -175,6 +203,10 @@ Run `Generate full report for opening the issue with debug info` to copy the cur When a problem concerns one file, run **Copy database information for the active file**, or use **Hatch** → **Copy database information for a file** to select another file. The report describes this device's local database view, including the Vault-relative path, document and chunk identifiers, local database revisions, conflicts, and local chunk availability. It does not query the remote server or include file contents. Treat paths and identifiers as private metadata before sharing. +To follow up on a path pseudonym from a report, use **Hatch** → **Find a file from a report pseudonym**, paste the token, and select **Find file**. The lookup checks the current Vault and local normal-file Metadata only; it does not contact the remote database. It lists every local candidate, with real paths and document IDs visible only in the local dialogue. Multiple IDs for the same path remain distinct. A Vault-only candidate means local Metadata was not inspected. No match may mean the file was renamed or deleted, or that the matching Metadata exists only remotely. + +For a selected candidate, the tool can open an existing Vault file or prepare a shareable local report using the original salt and selected document ID. That report shows current, conflict, and history status, Chunk availability, merge bases, and Vault size facts, without file contents or credentials. Its history status is a listing; it does not inspect every historical body. + Use `Show log` for live inspection. Logs are intentionally kept in memory for a limited time to reduce accidental disclosure. Enable `Write logs into the file` only while reproducing a problem, then disable it and remove the file after review because persistent logging affects performance and may contain private data. ![Write logs into the file](../images/write_logs_into_the_file.png) diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 073cbac7..34ae100b 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,43 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "Preparing follow-up reportâ€Ļ": "Preparing follow-up reportâ€Ļ", + "Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.", + "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.": + "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.", + "The lookup does not read file contents or contact a remote database. Follow-up reports include current conflicts and merge bases.": + "The lookup does not read file contents or contact a remote database. Follow-up reports include current conflicts and merge bases.", + "Local file diagnostic report": "Local file diagnostic report", + "Checked ${COUNT} local entries.": "Checked ${COUNT} local entries.", + "Enter a valid report path pseudonym.": "Enter a valid report path pseudonym.", + "Search cancelled. Matches found so far are shown.": "Search cancelled. Matches found so far are shown.", + "No matches were found in current Vault paths or current local Metadata. Older history and remote-only files are not searched.": + "No matches were found in current Vault paths or current local Metadata. Older history and remote-only files are not searched.", + "Found ${COUNT} local matches.": "Found ${COUNT} local matches.", + "The local search could not finish.": "The local search could not finish.", + "The file could not be opened.": "The file could not be opened.", + "The follow-up report could not be prepared.": "The follow-up report could not be prepared.", + "Report path pseudonym": "Report path pseudonym", + Source: "Source", + Metadata: "Metadata", + Vault: "Vault", + "Metadata document ID": "Metadata document ID", + "No current Metadata document": "No current Metadata document", + "Current Vault file": "Current Vault file", + Present: "Present", + Missing: "Missing", + "Open file": "Open file", + "Prepare follow-up report": "Prepare follow-up report", + "This report uses pseudonyms. Review its coverage before sharing.": + "This report uses pseudonyms. Review its coverage before sharing.", + "Back to matches": "Back to matches", + "Reading revisions": "Reading revisions", + "Calculating database usage": "Calculating database usage", + "Discovering revisions": "Discovering revisions", + "Find a file from a report pseudonym": "Find a file from a report pseudonym", + "Find matching paths in this Vault and local Metadata, then prepare a follow-up report using the same pseudonyms.": + "Find matching paths in this Vault and local Metadata, then prepare a follow-up report using the same pseudonyms.", + "Find file": "Find file", "⚠ī¸ Initialise or overwrite the remote": "⚠ī¸ Initialise or overwrite the remote", "🔗 Join this device": "🔗 Join this device", "⚙ī¸ Apply settings only (advanced)": "⚙ī¸ Apply settings only (advanced)", @@ -268,12 +305,13 @@ export const liveSyncProvisionalEnglishMessages = { "Copy failed. Select the report text and copy it manually.", "Download requested. You can also copy the report if downloads are unavailable on this device.": "Download requested. You can also copy the report if downloads are unavailable on this device.", - "Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.": - "Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.", + "Pseudonyms include a public salt, so guessed paths can be checked. Original names include private filenames. Reports omit file contents and credentials.": + "Pseudonyms include a public salt, so guessed paths can be checked. Original names include private filenames. Reports omit file contents and credentials.", "Read available local revisions and report their Chunk references. Settings are left unchanged.": "Read available local revisions and report their Chunk references. Settings are left unchanged.", - "Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.": - "Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.", + "Inspect normal-file Metadata, current conflicts, and suspect content against the Vault. Remote inspection supports CouchDB. Older history is excluded; findings do not authorise deletion.": + "Inspect normal-file Metadata, current conflicts, and suspect content against the Vault. Remote inspection supports CouchDB. Older history is excluded; findings do not authorise deletion.", + "Include remote database": "Include remote database", "Names in this report": "Names in this report", "Pseudonyms (recommended)": "Pseudonyms (recommended)", "Original names": "Original names", diff --git a/src/modules/features/SettingDialogue/PaneHatch.ts b/src/modules/features/SettingDialogue/PaneHatch.ts index 711556ff..48f6e4de 100644 --- a/src/modules/features/SettingDialogue/PaneHatch.ts +++ b/src/modules/features/SettingDialogue/PaneHatch.ts @@ -10,7 +10,7 @@ import { import { createBlob, escapeMarkdownValue, readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger"; import { shouldBeIgnored } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; -import { Menu, diff_match_patch, setIcon } from "@/deps.ts"; +import { Menu, TFile, diff_match_patch, setIcon } from "@/deps.ts"; import { $msg } from "@/common/translation"; import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; @@ -62,10 +62,70 @@ import { executeMetadataIdentityRepair, MetadataIdentityRepairExecutions, } from "@/serviceFeatures/metadataIdentityRepair.ts"; +import { showDiagnosticFileLookup } from "@/serviceFeatures/diagnosticFileLookup.ts"; export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, { addPanel }: PageFunctions): void { // const hatchWarn = this.createEl(paneEl, "div", { text: `To stop the boot up sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` }); // hatchWarn.addClass("op-warn-info"); - void addPanel(paneEl, $msg("Setting.TroubleShooting")).then((paneEl) => { + void addPanel(paneEl, "Reporting Tools").then((paneEl) => { + new Setting(paneEl).setName($msg("Prepare the 'report' to create an issue")).addButton((button) => + button + .setButtonText($msg("Copy Report to clipboard")) + .setCta() + .setDisabled(false) + .onClick(async () => { + await this.app.commands.executeCommandById("obsidian-livesync:dump-debug-info"); + }) + ); + new Setting(paneEl) + .setName($msg("Copy database information for a file")) + .setDesc( + $msg( + "Copy revision, conflict, and local chunk availability information, including document and chunk identifiers but not file contents." + ) + ) + .addButton((button) => + button.setButtonText($msg("Choose file")).onClick(async () => { + await chooseAndCopyFileDatabaseInfo(this.core); + }) + ); + new Setting(paneEl) + .setName($msg("Find a file from a report pseudonym")) + .setDesc( + $msg( + "Find matching paths in this Vault and local Metadata, then prepare a follow-up report using the same pseudonyms." + ) + ) + .addButton((button) => + button.setButtonText($msg("Find file")).onClick(async () => { + await showDiagnosticFileLookup(this.core, async (path) => { + const file = this.app.vault.getAbstractFileByPath(path); + if (file instanceof TFile) await this.app.workspace.getLeaf(false).openFile(file); + }); + }) + ); + new Setting(paneEl) + .setName($msg("Inspect database integrity")) + .setDesc( + $msg("Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.") + ) + .addButton((button) => + button.setButtonText($msg("Inspect")).onClick(() => eventHub.emitEvent(EVENT_INSPECT_DB_INTEGRITY)) + ); + new Setting(paneEl) + .setName($msg("Analyse database usage")) + .setDesc( + $msg( + "Analyse database usage and generate a TSV report for diagnosis yourself. You can paste the generated report with any spreadsheet you like." + ) + ) + .addButton((button) => + button.setButtonText($msg("Analyse")).onClick(() => { + eventHub.emitEvent(EVENT_ANALYSE_DB_USAGE); + }) + ); + }); + + void addPanel(paneEl, "Checks and Recovery").then((paneEl) => { new Setting(paneEl) .setName($msg("Setting.TroubleShooting.Doctor")) .setDesc($msg("Setting.TroubleShooting.Doctor.Desc")) @@ -92,48 +152,9 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, eventHub.emitEvent(EVENT_REQUEST_RUN_FIX_INCOMPLETE); }) ); + }); - new Setting(paneEl).setName($msg("Prepare the 'report' to create an issue")).addButton((button) => - button - .setButtonText($msg("Copy Report to clipboard")) - .setCta() - .setDisabled(false) - .onClick(async () => { - await this.app.commands.executeCommandById("obsidian-livesync:dump-debug-info"); - }) - ); - new Setting(paneEl) - .setName($msg("Copy database information for a file")) - .setDesc( - $msg( - "Copy revision, conflict, and local chunk availability information, including document and chunk identifiers but not file contents." - ) - ) - .addButton((button) => - button.setButtonText($msg("Choose file")).onClick(async () => { - await chooseAndCopyFileDatabaseInfo(this.core); - }) - ); - new Setting(paneEl) - .setName($msg("Inspect database integrity")) - .setDesc( - $msg("Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.") - ) - .addButton((button) => - button.setButtonText($msg("Inspect")).onClick(() => eventHub.emitEvent(EVENT_INSPECT_DB_INTEGRITY)) - ); - new Setting(paneEl) - .setName($msg("Analyse database usage")) - .setDesc( - $msg( - "Analyse database usage and generate a TSV report for diagnosis yourself. You can paste the generated report with any spreadsheet you like." - ) - ) - .addButton((button) => - button.setButtonText($msg("Analyse")).onClick(() => { - eventHub.emitEvent(EVENT_ANALYSE_DB_USAGE); - }) - ); + void addPanel(paneEl, $msg("Setting.TroubleShooting")).then((paneEl) => { new Setting(paneEl) .setName($msg("Reset notification threshold and check the remote database usage")) .setDesc($msg("Reset the remote storage size threshold and check the remote storage size again.")) diff --git a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte index b7b56b5f..46ed9056 100644 --- a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte @@ -1,21 +1,31 @@ -
+
{#if phase === "choice"} -

{translate("Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.")}

+

{translate("Pseudonyms include a public salt, so guessed paths can be checked. Original names include private filenames. Reports omit file contents and credentials.")}

{options?.kind === "usage" ? translate("Read available local revisions and report their Chunk references. Settings are left unchanged.") - : translate("Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.")} + : translate("Inspect normal-file Metadata, current conflicts, and suspect content against the Vault. Remote inspection supports CouchDB. Older history is excluded; findings do not authorise deletion.")}

{translate("Names in this report")} @@ -103,14 +189,21 @@
+ {#if options?.kind === "integrity"} + + {/if}
{:else if phase === "running"} -

- {cancelling ? translate("Cancelling inspectionâ€Ļ") : `${translate("Items inspected")}: ${processed}`} -

+
+ {#if cancelling} +

{translate("Cancelling inspectionâ€Ļ")}

+ {/if} @@ -149,16 +242,48 @@
diff --git a/src/modules/services/LiveSyncUI/dialogues/DiagnosticFileLookupDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/DiagnosticFileLookupDialog.svelte new file mode 100644 index 00000000..75cab5cd --- /dev/null +++ b/src/modules/services/LiveSyncUI/dialogues/DiagnosticFileLookupDialog.svelte @@ -0,0 +1,367 @@ + + + +
+ {#if phase !== "report"} + {#if phase !== "results"} +

+ {translate("Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.")} +

+

+ {translate( + "The lookup does not read file contents or contact a remote database. Follow-up reports include current conflicts and merge bases." + )} +

+ {/if} + +
+ + +
+ + {#if phase === "running"} + + {/if} + +
+
+ {:else if completedToken} +

{translate("Report path pseudonym")}: {completedToken}

+ {/if} + + {#if phase === "input" && actionStatus} +

{actionStatus}

+ {/if} + + {#if phase === "running"} +

{lookupStatus}

+ {:else if phase === "input" && lookupResult?.valid === false} +

{lookupStatus}

+ {:else if phase === "results" && lookupResult?.valid} +

{lookupStatus}

+ {#if lookupResult.candidates.length > 0} +
    + {#each lookupResult.candidates as candidate} +
  • +
    + {candidate.path} +
    +
    {translate("Source")}
    +
    {candidate.source === "metadata" ? translate("Metadata") : translate("Vault")}
    +
    {translate("Metadata document ID")}
    +
    {candidate.documentId ?? translate("No current Metadata document")}
    +
    {translate("Current Vault file")}
    +
    {candidate.storageExists ? translate("Present") : translate("Missing")}
    +
    +
    +
    + + +
    +
    • + {/each} +
+ {/if} +

{preparingReport ? translate("Preparing follow-up reportâ€Ļ") : actionStatus}

+ {:else if phase === "report" && output} +

{translate("This report uses pseudonyms. Review its coverage before sharing.")}

+
+ {translate("Report format")} +
+ {#each output.artifacts as artifact} + + {/each} +
+
+ +

{actionStatus}

+
+ + + + +
+ {/if} +
+ + diff --git a/src/serviceFeatures/databaseDiagnosticDialogTypes.ts b/src/serviceFeatures/databaseDiagnosticDialogTypes.ts index ea142bd6..c7686214 100644 --- a/src/serviceFeatures/databaseDiagnosticDialogTypes.ts +++ b/src/serviceFeatures/databaseDiagnosticDialogTypes.ts @@ -1,9 +1,17 @@ import type { DiagnosticNameMode } from "@vrtmrz/livesync-commonlib/diagnostics"; -import type { DatabaseDiagnosticKind, DatabaseDiagnosticOutput } from "./databaseDiagnostics"; +import type { + DatabaseDiagnosticKind, + DatabaseDiagnosticOutput, + DatabaseDiagnosticProgress, +} from "./databaseDiagnostics"; /** The dialogue can run or cancel an inspection without receiving the service container. */ export interface DatabaseDiagnosticDialogOptions { kind: DatabaseDiagnosticKind; - run(nameMode: DiagnosticNameMode, onProgress: (processed: number) => void): Promise; + run( + nameMode: DiagnosticNameMode, + onProgress: (progress: DatabaseDiagnosticProgress) => void, + includeRemote?: boolean + ): Promise; cancel(): void; } diff --git a/src/serviceFeatures/databaseDiagnostics.ts b/src/serviceFeatures/databaseDiagnostics.ts index 779a8aef..7519746b 100644 --- a/src/serviceFeatures/databaseDiagnostics.ts +++ b/src/serviceFeatures/databaseDiagnostics.ts @@ -17,6 +17,7 @@ import { REMOTE_COUCHDB, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync- import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; import type { NecessaryServicesInterfaces } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; import commonlibPackage from "@vrtmrz/livesync-commonlib/package.json"; +import { LOG_LEVEL_INFO, Logger } from "octagonal-wheels/common/logger"; export type DatabaseDiagnosticKind = "usage" | "integrity"; export type DatabaseDiagnosticHost = NecessaryServicesInterfaces< @@ -36,6 +37,15 @@ export interface DatabaseDiagnosticOutput { artifacts: DatabaseDiagnosticArtifact[]; } +export interface DatabaseDiagnosticProgress { + phase: "usage" | "accounting" | "metadata" | "content"; + source?: "local" | "remote"; + processed: number; + total?: number; + totalIsFinal?: boolean; + elapsedMs: number; +} + export async function readDiagnosticStorage( storage: Pick, path: string @@ -59,7 +69,7 @@ export async function readDiagnosticStorage( ) { return { status: "unreadable", reason: "storage-changed" }; } - return { status: "available", bytes }; + return { status: "available", bytes, mtime: after.mtime }; } catch { return { status: "unreadable", reason: "content-unreadable" }; } @@ -70,8 +80,13 @@ export async function runDatabaseDiagnostic( host: DatabaseDiagnosticHost, kind: DatabaseDiagnosticKind, nameMode: DiagnosticNameMode, - options: { signal?: AbortSignal; onProgress?(processed: number): void } = {} + options: { + signal?: AbortSignal; + onProgress?(progress: DatabaseDiagnosticProgress): void; + includeRemote?: boolean; + } = {} ): Promise { + const started = performance.now(); const configuration = JSON.stringify(host.services.setting.currentSettings()); const settings = JSON.parse(configuration) as ObsidianLiveSyncSettings; const database = host.services.database.localDatabase.localDatabase; @@ -79,6 +94,12 @@ export async function runDatabaseDiagnostic( JSON.stringify(host.services.setting.currentSettings()) === configuration && host.services.database.localDatabase.localDatabase === database; const mapper = createDiagnosticNameMapper(nameMode); + const httpRequestCounters = { httpRequests: 0, httpFailedRequests: 0, httpRequestDurationMs: 0 }; + const getHttpRequestCounters = () => ({ + httpRequests: httpRequestCounters.httpRequests, + httpFailedRequests: httpRequestCounters.httpFailedRequests, + httpRequestDurationMs: Math.round(httpRequestCounters.httpRequestDurationMs), + }); const versions = { commonlib: commonlibPackage.version, plugin: host.services.API.getPluginVersion(), @@ -89,7 +110,14 @@ export async function runDatabaseDiagnostic( const result = await analyseDatabaseUsage({ source: local, signal: options.signal, - onProgress: (progress) => options.onProgress?.(progress.revisionsProcessed), + onProgress: (progress) => + options.onProgress?.({ + phase: progress.phase === "accounting" ? "accounting" : "usage", + processed: progress.processed, + total: progress.total, + totalIsFinal: progress.totalIsFinal, + elapsedMs: Math.round(performance.now() - started), + }), }); if (!isConfigurationCurrent()) { result.errors.push({ code: "configuration_changed" }); @@ -110,23 +138,41 @@ export async function runDatabaseDiagnostic( } return { status: result.status, nameMode, artifacts }; } - const result = await analyseDatabaseIntegrity({ + const includeRemote = options.includeRemote !== false; + const integrityOptions: Parameters[0] = { local, remote: - settings.remoteType === REMOTE_COUCHDB + includeRemote && settings.remoteType === REMOTE_COUCHDB ? () => Promise.resolve( createCouchDBDiagnosticSource({ settings, signal: options.signal, - fetch: (request, requestOptions) => - settings.useRequestAPI - ? host.services.API.nativeFetch(request, requestOptions) - : host.services.API.webCompatFetch(request, requestOptions), + fetch: async (request, requestOptions) => { + const requestStarted = performance.now(); + httpRequestCounters.httpRequests++; + try { + const response = settings.useRequestAPI + ? await host.services.API.nativeFetch(request, requestOptions) + : await host.services.API.webCompatFetch(request, requestOptions); + if (!response.ok) httpRequestCounters.httpFailedRequests++; + return response; + } catch (error) { + httpRequestCounters.httpFailedRequests++; + throw error; + } finally { + httpRequestCounters.httpRequestDurationMs += performance.now() - requestStarted; + } + }, }) ) : undefined, - remoteUnavailableReason: settings.remoteType === REMOTE_COUCHDB ? "source-unavailable" : "unsupported-remote", + remoteUnavailableReason: includeRemote + ? settings.remoteType === REMOTE_COUCHDB + ? "source-unavailable" + : "unsupported-remote" + : undefined, + includeRemote, path: { getPath: (entry) => host.services.path.getPath(entry), path2id: (path, prefix) => host.services.path.path2idWithSettings(path, settings, prefix), @@ -141,10 +187,17 @@ export async function runDatabaseDiagnostic( readStorage: (path) => readDiagnosticStorage(host.serviceModules.storageAccess, path), isConfigurationCurrent, signal: options.signal, - onProgress: (progress) => options.onProgress?.(progress.processed), - }); + onProgress: (progress) => options.onProgress?.(progress), + onLog: (event) => + Logger( + `Database integrity inspection: ${JSON.stringify({ ...event, ...getHttpRequestCounters() })}`, + LOG_LEVEL_INFO + ), + }; + const result = await analyseDatabaseIntegrity(integrityOptions); + const formattingStarted = performance.now(); const report = projectIntegrityReport(result, mapper, versions); - return { + const output: DatabaseDiagnosticOutput = { status: result.status, nameMode, artifacts: [ @@ -160,4 +213,13 @@ export async function runDatabaseDiagnostic( }, ], }; + const requestCounters = getHttpRequestCounters(); + Logger( + `Database integrity report: status=${result.status}, observations=${result.observations.length}, ` + + `httpRequests=${requestCounters.httpRequests}, httpFailedRequests=${requestCounters.httpFailedRequests}, ` + + `httpRequestDurationMs=${requestCounters.httpRequestDurationMs}, ` + + `formattingMs=${Math.round(performance.now() - formattingStarted)}, elapsedMs=${Math.round(performance.now() - started)}`, + LOG_LEVEL_INFO + ); + return output; } diff --git a/src/serviceFeatures/databaseDiagnostics.unit.spec.ts b/src/serviceFeatures/databaseDiagnostics.unit.spec.ts index b572832d..3d17ea88 100644 --- a/src/serviceFeatures/databaseDiagnostics.unit.spec.ts +++ b/src/serviceFeatures/databaseDiagnostics.unit.spec.ts @@ -1,12 +1,31 @@ import { describe, expect, it, vi } from "vitest"; -import { DEFAULT_SETTINGS, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + DEFAULT_SETTINGS, + REMOTE_COUCHDB, + REMOTE_P2P, + type ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; + +const diagnosticsMocks = vi.hoisted(() => ({ analyseOptions: vi.fn() })); + +vi.mock("@vrtmrz/livesync-commonlib/diagnostics", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + analyseDatabaseIntegrity: (options: Parameters[0]) => { + diagnosticsMocks.analyseOptions(options); + return actual.analyseDatabaseIntegrity(options); + }, + }; +}); + import { readDiagnosticStorage, runDatabaseDiagnostic, type DatabaseDiagnosticHost } from "./databaseDiagnostics"; function fixture() { const path = "Private/æ—ĨæœŦčĒž.md"; const body = "private file contents"; const bytes = new TextEncoder().encode(body); - const settings = { + const settings: ObsidianLiveSyncSettings = { ...DEFAULT_SETTINGS, remoteType: REMOTE_P2P, readChunksOnline: true, @@ -81,6 +100,41 @@ function fixture() { } describe("database diagnostic host composition", () => { + it("passes growing Usage totals and the separate accounting phase to the dialogue", async () => { + const onProgress = vi.fn(); + await runDatabaseDiagnostic(fixture().host, "usage", "pseudonym", { onProgress }); + const progress = onProgress.mock.calls.map(([value]) => value); + expect(progress).toContainEqual( + expect.objectContaining({ + phase: "usage", + processed: 0, + total: 0, + totalIsFinal: false, + }) + ); + expect(progress).toContainEqual( + expect.objectContaining({ + phase: "usage", + processed: 2, + total: 2, + totalIsFinal: true, + }) + ); + const accounting = progress.filter((value) => value.phase === "accounting"); + expect(accounting.length).toBeGreaterThan(1); + const final = accounting[accounting.length - 1]; + expect(final).toMatchObject({ processed: final.total, totalIsFinal: true }); + }); + + it("preserves phase, source, and totals for the inspection dialogue", async () => { + const onProgress = vi.fn(); + await runDatabaseDiagnostic(fixture().host, "integrity", "pseudonym", { onProgress }); + expect(onProgress).toHaveBeenCalledWith( + expect.objectContaining({ phase: "metadata", source: "local", processed: 0 }) + ); + expect(onProgress).toHaveBeenCalledWith(expect.objectContaining({ phase: "content", processed: 1, total: 1 })); + }); + it("keeps on-demand fetching enabled while analysing only existing local revisions", async () => { const data = fixture(); const before = JSON.stringify(data.settings); @@ -113,6 +167,9 @@ describe("database diagnostic host composition", () => { it("keeps local findings when the remote type is unsupported and projects both formats consistently", async () => { const data = fixture(); const result = await runDatabaseDiagnostic(data.host, "integrity", "pseudonym"); + expect(diagnosticsMocks.analyseOptions).toHaveBeenLastCalledWith( + expect.objectContaining({ includeRemote: true }) + ); expect(result.status).toBe("partial"); expect(data.fetch).not.toHaveBeenCalled(); expect(data.put).not.toHaveBeenCalled(); @@ -135,6 +192,82 @@ describe("database diagnostic host composition", () => { } }); + it("excludes the remote source for a local integrity run", async () => { + const data = fixture(); + data.settings.remoteType = REMOTE_COUCHDB; + data.settings.couchDB_DBNAME = "private-database"; + data.settings.useRequestAPI = true; + data.fetch + .mockImplementationOnce(async () => { + await new Promise((resolve) => setTimeout(resolve, 2)); + return { ok: false, status: 503 } as Response; + }) + .mockRejectedValue(new Error("simulated diagnostic request failure")); + + diagnosticsMocks.analyseOptions.mockClear(); + const result = await runDatabaseDiagnostic(data.host, "integrity", "pseudonym", { + includeRemote: false, + }); + + expect(data.fetch).not.toHaveBeenCalled(); + const passedOptions = diagnosticsMocks.analyseOptions.mock.calls[0][0]; + expect(passedOptions.includeRemote).toBe(false); + expect(passedOptions.remote).toBeUndefined(); + expect(result.artifacts.map((artifact) => artifact.format)).toEqual(["markdown", "json"]); + const report = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content); + expect(report.scope.remote).toBe("excluded"); + expect(report.coverage.remote.status).toBe("not-checked"); + expect(report.errors.some((error: { source: string }) => error.source === "remote")).toBe(false); + }); + + it("logs cumulative HTTP dispatch, failure, and duration counters for remote inspection", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + try { + const nonOk = fixture(); + nonOk.settings.remoteType = REMOTE_COUCHDB; + nonOk.settings.couchDB_DBNAME = "private-database"; + nonOk.settings.useRequestAPI = true; + nonOk.fetch.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 2)); + return { ok: false, status: 503 } as Response; + }); + await runDatabaseDiagnostic(nonOk.host, "integrity", "pseudonym"); + + const thrown = fixture(); + thrown.settings.remoteType = REMOTE_COUCHDB; + thrown.settings.couchDB_DBNAME = "private-database"; + thrown.settings.useRequestAPI = true; + thrown.fetch.mockRejectedValue(new Error("simulated diagnostic request failure")); + await runDatabaseDiagnostic(thrown.host, "integrity", "pseudonym"); + + const messages = log.mock.calls.map(([message]) => String(message)); + const inspectionLogs = messages.filter((message) => message.includes("Database integrity inspection:")); + expect(inspectionLogs.length).toBeGreaterThan(0); + const inspectionEvents = inspectionLogs.map((message) => { + const json = message.slice(message.indexOf("{")); + return JSON.parse(json); + }); + expect(inspectionEvents.some((event) => event.httpRequests > 0)).toBe(true); + expect(inspectionEvents.some((event) => event.httpFailedRequests > 0)).toBe(true); + expect(inspectionEvents.every((event) => typeof event.httpRequestDurationMs === "number")).toBe(true); + + const summaries = messages.filter((message) => message.includes("Database integrity report:")); + expect(summaries).toHaveLength(2); + for (const [index, summary] of summaries.entries()) { + const match = /httpRequests=(\d+), httpFailedRequests=(\d+), httpRequestDurationMs=(\d+)/.exec(summary); + expect(match).not.toBeNull(); + const data = index === 0 ? nonOk : thrown; + expect(Number(match![1])).toBe(data.fetch.mock.calls.length); + expect(Number(match![2])).toBe(data.fetch.mock.calls.length); + expect(Number(match![3])).toBeGreaterThanOrEqual(0); + if (index === 0) expect(Number(match![3])).toBeGreaterThan(0); + } + } finally { + log.mockRestore(); + } + }); + it("includes names only when requested and never includes credentials or file contents", async () => { const data = fixture(); const result = await runDatabaseDiagnostic(data.host, "usage", "original"); @@ -172,6 +305,17 @@ describe("database diagnostic host composition", () => { }); describe("diagnostic Vault reads", () => { + it("reuses the validated snapshot timestamp without adding stat or content reads", async () => { + const data = fixture(); + expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ + status: "available", + bytes: data.bytes, + mtime: 1, + }); + expect(data.storage.statHidden).toHaveBeenCalledTimes(2); + expect(data.storage.readHiddenFileBinary).toHaveBeenCalledTimes(1); + }); + it("distinguishes an absent file, a read error, and a file changing during its read", async () => { const data = fixture(); data.storage.statHidden.mockResolvedValueOnce(null as never); diff --git a/src/serviceFeatures/diagnosticFileLookup.ts b/src/serviceFeatures/diagnosticFileLookup.ts new file mode 100644 index 00000000..f9bdf7f3 --- /dev/null +++ b/src/serviceFeatures/diagnosticFileLookup.ts @@ -0,0 +1,346 @@ +import { $msg } from "@/common/translation"; +import { delay } from "octagonal-wheels/promises"; +import { createDiagnosticNameMapper, parseDiagnosticPathToken } from "@vrtmrz/livesync-commonlib/diagnostics"; +import { EntryTypes, type FilePath, type FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { ICHeader, ICXHeader, PSCHeader } from "@vrtmrz/livesync-commonlib/compat/common/models/fileaccess.const"; +import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; +import type { ComponentHasResult } from "@/modules/services/LiveSyncUI/svelteDialog"; +import DiagnosticFileLookupDialog from "@/modules/services/LiveSyncUI/dialogues/DiagnosticFileLookupDialog.svelte"; +import type { + FileDatabaseInfo, + FileDatabaseInfoCore, + FileDatabaseMergeBaseInfo, + InspectFileDatabaseInfoOptions, + RevisionDatabaseInfo, +} from "./fileDatabaseInfo"; +import { inspectFileDatabaseInfo } from "./fileDatabaseInfo"; +import type { + DiagnosticFileCandidate, + DiagnosticFileLookupDialogOptions, + DiagnosticFileLookupResult, + FindDiagnosticFilesOptions, +} from "./diagnosticFileLookupTypes"; +import type { DatabaseDiagnosticArtifact, DatabaseDiagnosticOutput } from "./databaseDiagnostics"; + +type CurrentFileMetadata = { + _id: string; + path: FilePathWithPrefix | FilePath; + type: string; + _deleted?: boolean; +}; + +type FileLookupDialogManager = { + open( + component: ComponentHasResult, + initialData?: TInitial + ): Promise; +}; + +type FileLookupUI = FileDatabaseInfoCore["services"]["UI"] & { + dialogManager: FileLookupDialogManager; +}; + +function isCurrentFileMetadata(value: unknown): value is CurrentFileMetadata { + if (value === null || typeof value !== "object") return false; + const entry = value as Record; + if ( + typeof entry._id !== "string" || + typeof entry.path !== "string" || + (entry.type !== EntryTypes.NOTE_BINARY && entry.type !== EntryTypes.NOTE_PLAIN) || + entry._deleted === true + ) { + return false; + } + const path = entry.path; + return ![ICHeader, ICXHeader, PSCHeader].some((prefix) => path.startsWith(prefix)); +} + +function cancelledResult(processed: number, candidates: DiagnosticFileCandidate[]): DiagnosticFileLookupResult { + return { valid: true, status: "cancelled", processed, candidates }; +} + +async function yieldToEventLoop(): Promise { + await delay(0); +} + +export async function findDiagnosticFiles( + core: FileDatabaseInfoCore, + token: string, + options: FindDiagnosticFilesOptions = {} +): Promise { + const parsedToken = parseDiagnosticPathToken(token); + if (!parsedToken) { + return { valid: false, status: "invalid", processed: 0, candidates: [] }; + } + const mapper = createDiagnosticNameMapper("pseudonym", parsedToken.salt); + const candidates: DiagnosticFileCandidate[] = []; + let processed = 0; + const reportProgress = () => options.onProgress?.(processed); + reportProgress(); + if (options.signal?.aborted) { + reportProgress(); + return cancelledResult(processed, candidates); + } + const vaultPaths = new Set(await core.storageAccess.getFileNames()); + const matchingMetadataPaths = new Set(); + if (options.signal?.aborted) { + reportProgress(); + return cancelledResult(processed, candidates); + } + + for await (const entry of core.localDatabase.findAllDocs()) { + processed++; + if (isCurrentFileMetadata(entry)) { + const databasePath = entry.path; + const path = stripAllPrefixes(databasePath); + if (mapper.path(path) === token) { + matchingMetadataPaths.add(path); + candidates.push({ + path, + databasePath, + documentId: entry._id, + storageExists: vaultPaths.has(path), + source: "metadata", + }); + } + } + if (processed % 128 === 0) { + reportProgress(); + await yieldToEventLoop(); + if (options.signal?.aborted) { + reportProgress(); + return cancelledResult(processed, candidates); + } + } + } + if (options.signal?.aborted) { + reportProgress(); + return cancelledResult(processed, candidates); + } + + for (const path of vaultPaths) { + processed++; + if (mapper.path(path) === token && !matchingMetadataPaths.has(path)) { + candidates.push({ path, storageExists: true, source: "vault" }); + } + if (processed % 128 === 0) { + reportProgress(); + await yieldToEventLoop(); + if (options.signal?.aborted) { + reportProgress(); + return cancelledResult(processed, candidates); + } + } + } + reportProgress(); + if (options.signal?.aborted) return cancelledResult(processed, candidates); + return { valid: true, status: "complete", processed, candidates }; +} + +function finiteMetric(value: number | undefined, fallback = 0): number { + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +function nonnegativeMetric(value: number | undefined): number { + return Math.max(0, finiteMetric(value)); +} + +function revisionStatus(value: string): "available" | "missing" | "unknown" { + if (value === "available" || value === "missing") return value; + return "unknown"; +} + +function storageType(value: string): "absent" | "notes" | "newnote" | "plain" | "unknown" { + if (value === "absent" || value === "notes" || value === "newnote" || value === "plain") return value; + return "unknown"; +} + +function localChunkState(value: string): "available" | "deleted" | "missing" | "unknown" { + if (value === "available" || value === "deleted" || value === "missing") return value; + return "unknown"; +} + +function projectRevisionInfo(info: RevisionDatabaseInfo, mapper: ReturnType) { + return { + documentId: mapper.id(info.documentId), + revision: info.revision === null ? null : mapper.revision(info.revision), + current: info.current, + deleted: info.deleted, + storageType: storageType(info.storageType), + storageLayout: info.storageLayout === "legacy-inline" ? "legacy-inline" : "chunked", + ctime: finiteMetric(info.ctime), + mtime: finiteMetric(info.mtime), + recordedSize: nonnegativeMetric(info.recordedSize), + revisionHistory: info.revisionHistory.map(({ revision, status }) => ({ + revision: mapper.revision(revision), + status: revisionStatus(status), + })), + chunkReferences: nonnegativeMetric(info.chunkReferences), + uniqueChunkReferences: nonnegativeMetric(info.uniqueChunkReferences), + embeddedChunkReferences: nonnegativeMetric(info.embeddedChunkReferences), + locallyStoredChunkReferences: nonnegativeMetric(info.locallyStoredChunkReferences), + contentAvailableLocally: info.contentAvailableLocally, + chunks: info.chunks.map((chunk) => ({ + id: mapper.chunk(chunk.id), + referenceCount: nonnegativeMetric(chunk.referenceCount), + embedded: chunk.embedded, + storedInLocalDatabase: chunk.storedInLocalDatabase, + localDatabaseState: localChunkState(chunk.localDatabaseState), + localDatabaseRevision: + chunk.localDatabaseRevision === null ? null : mapper.revision(chunk.localDatabaseRevision), + })), + }; +} + +function projectMergeBase(info: FileDatabaseMergeBaseInfo, mapper: ReturnType) { + return { + winnerRevision: mapper.revision(info.winnerRevision), + conflictRevision: mapper.revision(info.conflictRevision), + revision: info.revision === null ? null : mapper.revision(info.revision), + metadataAvailableLocally: info.metadataAvailableLocally, + contentAvailableLocally: info.contentAvailableLocally, + missingChunkIds: info.missingChunkIds.map((id) => mapper.chunk(id)), + unavailableSharedRevisions: info.unavailableSharedRevisions.map((revision) => mapper.revision(revision)), + }; +} + +function projectStorage(info: FileDatabaseInfo["storage"]) { + return { + exists: info.exists, + ...(info.ctime === undefined ? {} : { ctime: finiteMetric(info.ctime) }), + ...(info.mtime === undefined ? {} : { mtime: finiteMetric(info.mtime) }), + ...(info.size === undefined ? {} : { size: nonnegativeMetric(info.size) }), + }; +} + +function projectFileInfo( + info: FileDatabaseInfo, + source: DiagnosticFileCandidate["source"], + mapper: ReturnType +) { + return { + path: mapper.path(info.path), + databasePath: mapper.path(info.databasePath), + source, + storage: projectStorage(info.storage), + database: { + source: "local database on this device" as const, + remoteQueried: false as const, + inspected: true, + exists: info.database.exists, + ...(info.database.documentId === undefined ? {} : { documentId: mapper.id(info.database.documentId) }), + currentRevision: + info.database.currentRevision === null ? null : mapper.revision(info.database.currentRevision), + conflictCount: nonnegativeMetric(info.database.conflictCount), + conflictRevisions: info.database.conflictRevisions.map((revision) => mapper.revision(revision)), + unavailableConflictRevisions: info.database.unavailableConflictRevisions.map((revision) => + mapper.revision(revision) + ), + revisions: info.database.revisions.map((revision) => projectRevisionInfo(revision, mapper)), + mergeBases: info.database.mergeBases.map((mergeBase) => projectMergeBase(mergeBase, mapper)), + }, + }; +} + +async function inspectVaultOnlyCandidate( + core: FileDatabaseInfoCore, + candidate: DiagnosticFileCandidate, + mapper: ReturnType, + signal?: AbortSignal +) { + const exists = await core.storageAccess.isExistsIncludeHidden(candidate.path); + if (signal?.aborted) throw new DOMException("File inspection cancelled", "AbortError"); + const stat = exists ? await core.storageAccess.statHidden(candidate.path) : null; + return { + path: mapper.path(candidate.path), + source: "vault" as const, + storage: projectStorage( + stat ? { exists: true, ctime: stat.ctime, mtime: stat.mtime, size: stat.size } : { exists: false } + ), + database: { + source: "local database on this device" as const, + remoteQueried: false as const, + inspected: false, + reason: "no-local-target" as const, + }, + }; +} + +export async function buildPseudonymousFileReport( + core: FileDatabaseInfoCore, + candidate: DiagnosticFileCandidate, + salt: string, + signal?: AbortSignal +): Promise { + if (signal?.aborted) throw new DOMException("File inspection cancelled", "AbortError"); + const mapper = createDiagnosticNameMapper("pseudonym", salt); + if (!mapper.salt) throw new Error("Invalid diagnostic path token"); + if (candidate.source === "metadata" && candidate.documentId === undefined) { + throw new Error("A Metadata candidate requires its exact document ID"); + } + const info = + candidate.source === "metadata" && candidate.documentId !== undefined + ? await inspectFileDatabaseInfo(core, candidate.path, { + documentId: candidate.documentId, + signal, + ...(candidate.databasePath === undefined ? {} : { databasePath: candidate.databasePath }), + } satisfies InspectFileDatabaseInfoOptions) + : undefined; + const file = info + ? projectFileInfo(info, candidate.source, mapper) + : await inspectVaultOnlyCandidate(core, candidate, mapper, signal); + if (signal?.aborted) throw new DOMException("File inspection cancelled", "AbortError"); + const report = { + schemaVersion: 1, + kind: "file", + status: "complete", + nameMode: "pseudonym", + salt: mapper.salt, + scope: { + source: "local database on this device", + remoteQueried: false, + vaultContentRead: false, + fileContents: "omitted", + revisions: "current revision, current conflict revisions, and selected merge-base revisions", + revisionHistory: "status metadata is listed for current and conflict revisions where available", + historicalFileBodiesScanned: false, + }, + file, + }; + const json = JSON.stringify(report, null, 2); + const markdown = `
\n${$msg("Local file diagnostic report")}\n\n\`\`\`json\n${json}\n\`\`\`\n
`; + const artifacts: DatabaseDiagnosticArtifact[] = [ + { format: "json", filename: "diagnostic-file-report.json", content: json }, + { format: "markdown", filename: "diagnostic-file-report.md", content: markdown }, + ]; + return { status: "complete", nameMode: "pseudonym", artifacts }; +} + +export async function showDiagnosticFileLookup( + core: FileDatabaseInfoCore, + openFile?: (path: string) => Promise | void +): Promise { + let activeController: AbortController | undefined; + const options: DiagnosticFileLookupDialogOptions = { + find: async (token, onProgress) => { + activeController?.abort(); + activeController = new AbortController(); + return await findDiagnosticFiles(core, token, { signal: activeController.signal, onProgress }); + }, + prepareReport: async (candidate, token) => { + const parsed = parseDiagnosticPathToken(token); + if (!parsed) throw new Error("Invalid diagnostic path token"); + activeController?.abort(); + activeController = new AbortController(); + return await buildPseudonymousFileReport(core, candidate, parsed.salt, activeController.signal); + }, + openFile, + cancel: () => activeController?.abort(), + }; + const ui = core.services.UI as FileLookupUI; + try { + await ui.dialogManager.open(DiagnosticFileLookupDialog, options); + } finally { + activeController?.abort(); + } +} diff --git a/src/serviceFeatures/diagnosticFileLookup.unit.spec.ts b/src/serviceFeatures/diagnosticFileLookup.unit.spec.ts new file mode 100644 index 00000000..3079df4a --- /dev/null +++ b/src/serviceFeatures/diagnosticFileLookup.unit.spec.ts @@ -0,0 +1,362 @@ +import { createDiagnosticNameMapper } from "@vrtmrz/livesync-commonlib/diagnostics"; +import { describe, expect, it, vi } from "vitest"; +import { buildPseudonymousFileReport, findDiagnosticFiles, showDiagnosticFileLookup } from "./diagnosticFileLookup"; +import type { DiagnosticFileLookupDialogOptions } from "./diagnosticFileLookupTypes"; + +type TestDocument = { + _id: string; + _rev?: string; + _conflicts?: string[]; + _revs_info?: Array<{ rev: string; status: string }>; + _deleted?: boolean; + children?: string[]; + ctime?: number; + data?: string; + deleted?: boolean; + eden?: Record; + mtime?: number; + path?: string; + size?: number; + type?: string; +}; + +async function* entries(documents: TestDocument[]) { + for (const document of documents) yield document; +} + +function createCore(vaultPaths: string[] = [], documents: TestDocument[] = []) { + const existingPaths = new Set(vaultPaths); + const chunkRevisions = new Map([["h:secret-chunk", "1-secret-chunk-revision"]]); + const core = { + settings: {}, + storageAccess: { + getFileNames: vi.fn(async () => [...vaultPaths]), + getFilesIncludeHidden: vi.fn(async () => [...vaultPaths]), + isExistsIncludeHidden: vi.fn(async (path: string) => existingPaths.has(path)), + statHidden: vi.fn(async (path: string) => + existingPaths.has(path) ? { ctime: 10, mtime: 20, size: 30, type: "file" } : undefined + ), + }, + localDatabase: { + allDocsRaw: vi.fn(async ({ keys }: { keys: string[] }) => ({ + rows: keys.flatMap((key) => { + const rev = chunkRevisions.get(key); + return rev ? [{ id: key, key, value: { rev } }] : []; + }), + })), + findAllDocs: vi.fn(() => entries(documents)), + getDBEntryFromMeta: vi.fn(), + getDBEntry: vi.fn(), + localDatabase: { + get: vi.fn(async (id: string) => { + const document = documents.find(({ _id }) => _id === id); + if (document) return document; + throw Object.assign(new Error("missing"), { status: 404 }); + }), + }, + }, + services: { + path: { path2id: vi.fn(async () => "f:path-derived") }, + UI: { + promptCopyToClipboard: vi.fn(async () => true), + dialogManager: { open: vi.fn() }, + }, + }, + }; + return core; +} + +function diagnosticPath(path: string, salt = "salt1234"): string { + return createDiagnosticNameMapper("pseudonym", salt).path(path); +} + +describe("diagnostic file lookup", () => { + it.each(["metadata", "chunks"] as const)( + "stops follow-up reads when the dialogue closes during a %s read", + async (pendingRead) => { + const path = "cancelled.md"; + const selected = { + _id: "f:cancelled", + _rev: "3-current", + _conflicts: ["3-conflict"], + path, + type: "plain", + children: ["h:secret-chunk"], + }; + const core = createCore([path], [selected]); + let markStarted!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let releaseRead!: () => void; + const pending = new Promise((resolve) => { + releaseRead = resolve; + }); + if (pendingRead === "metadata") { + core.localDatabase.localDatabase.get.mockImplementationOnce(async () => { + markStarted(); + await pending; + return selected; + }); + } else { + core.localDatabase.allDocsRaw.mockImplementationOnce(async () => { + markStarted(); + await pending; + return { rows: [] }; + }); + } + core.services.UI.dialogManager.open.mockImplementation(async (...args: unknown[]) => { + const options = args[1] as DiagnosticFileLookupDialogOptions; + const preparation = options.prepareReport( + { path, documentId: selected._id, storageExists: true, source: "metadata" }, + diagnosticPath(path) + ); + await started; + const cancelled = expect(preparation).rejects.toMatchObject({ name: "AbortError" }); + options.cancel(); + releaseRead(); + await cancelled; + }); + + await showDiagnosticFileLookup(core as never); + + expect(core.localDatabase.localDatabase.get).toHaveBeenCalledTimes(1); + expect(core.localDatabase.allDocsRaw).toHaveBeenCalledTimes(pendingRead === "metadata" ? 0 : 1); + } + ); + + it("rejects an invalid path token before reading storage or the database", async () => { + const core = createCore(["note.md"], [{ _id: "f:note", path: "note.md", type: "plain" }]); + + await expect(findDiagnosticFiles(core as never, "not-a-path-token")).resolves.toEqual({ + valid: false, + status: "invalid", + processed: 0, + candidates: [], + }); + + expect(core.storageAccess.getFileNames).not.toHaveBeenCalled(); + expect(core.localDatabase.findAllDocs).not.toHaveBeenCalled(); + expect(core.localDatabase.localDatabase.get).not.toHaveBeenCalled(); + }); + + it("matches path spelling exactly without case folding or Unicode normalisation", async () => { + const decomposed = "Cafe\u0301.md"; + const core = createCore( + [decomposed, "CafÊ.md", "CAFÉ.md"], + [ + { _id: "f:decomposed", path: decomposed, type: "plain" }, + { _id: "f:composed", path: "CafÊ.md", type: "plain" }, + { _id: "f:case", path: "CAFÉ.md", type: "plain" }, + ] + ); + const progressCounts: number[] = []; + + const result = await findDiagnosticFiles(core as never, diagnosticPath(decomposed), { + onProgress: (processed) => progressCounts.push(processed), + }); + + expect(result).toMatchObject({ valid: true, status: "complete" }); + expect(result.candidates).toEqual([ + expect.objectContaining({ + path: decomposed, + documentId: "f:decomposed", + source: "metadata", + storageExists: true, + }), + ]); + expect(core.localDatabase.localDatabase.get).not.toHaveBeenCalled(); + expect(progressCounts).toEqual([0, 6]); + }); + + it("keeps distinct Metadata documents with the same exact path as separate candidates", async () => { + const path = "duplicate.md"; + const core = createCore( + [path], + [ + { _id: "f:duplicate-one", path, type: "plain" }, + { _id: "f:duplicate-two", path, type: "plain" }, + ] + ); + + const result = await findDiagnosticFiles(core as never, diagnosticPath(path)); + + expect(result.candidates).toEqual([ + expect.objectContaining({ documentId: "f:duplicate-one", path, storageExists: true, source: "metadata" }), + expect.objectContaining({ documentId: "f:duplicate-two", path, storageExists: true, source: "metadata" }), + ]); + expect(result.candidates).toHaveLength(2); + }); + + it("keeps logically deleted current file Metadata as an exact candidate", async () => { + const path = "vault-only.md"; + const core = createCore( + [path], + [ + { _id: "f:deleted", path, type: "plain", deleted: true }, + { _id: "f:chunk", path, type: "chunk" }, + { _id: "i:internal", path: "i:.obsidian/config.json", type: "plain" }, + ] + ); + + const result = await findDiagnosticFiles(core as never, diagnosticPath(path)); + expect(result.candidates).toEqual([ + expect.objectContaining({ path, documentId: "f:deleted", storageExists: true, source: "metadata" }), + ]); + }); + + it("returns a Vault-only candidate when there is no current normal-file Metadata", async () => { + const path = "vault-only.md"; + const core = createCore( + [path], + [ + { _id: "f:chunk", path, type: "chunk" }, + { _id: "f:legacy", path, type: "notes" }, + { _id: "i:internal", path: "i:.obsidian/config.json", type: "plain" }, + ] + ); + + const result = await findDiagnosticFiles(core as never, diagnosticPath(path)); + expect(result.candidates).toEqual([expect.objectContaining({ path, storageExists: true, source: "vault" })]); + expect(result.candidates[0]).not.toHaveProperty("documentId"); + + const report = await buildPseudonymousFileReport(core as never, result.candidates[0], "salt1234"); + const jsonText = report.artifacts.find(({ format }) => format === "json")!.content; + const json = JSON.parse(jsonText); + + expect(json.file.path).toBe(diagnosticPath(path)); + expect(jsonText).not.toContain(path); + expect(json.file.database).toEqual({ + source: "local database on this device", + remoteQueried: false, + inspected: false, + reason: "no-local-target", + }); + expect(json.scope).toEqual({ + source: "local database on this device", + remoteQueried: false, + vaultContentRead: false, + fileContents: "omitted", + revisions: "current revision, current conflict revisions, and selected merge-base revisions", + revisionHistory: "status metadata is listed for current and conflict revisions where available", + historicalFileBodiesScanned: false, + }); + expect(core.localDatabase.localDatabase.get).not.toHaveBeenCalled(); + }); + + it("returns partial exact matches when cancellation arrives during enumeration", async () => { + const path = "match.md"; + const documents = [ + { _id: "f:match", path, type: "plain" }, + ...Array.from({ length: 299 }, (_, index) => ({ + _id: `f:other-${index}`, + path: `other-${index}.md`, + type: "plain", + })), + ]; + const core = createCore([], documents); + const controller = new AbortController(); + const progressCounts: number[] = []; + + const result = await findDiagnosticFiles(core as never, diagnosticPath(path), { + signal: controller.signal, + onProgress: (processed) => { + progressCounts.push(processed); + if (processed >= 128) controller.abort(); + }, + }); + + expect(result).toMatchObject({ valid: true, status: "cancelled", processed: 128 }); + expect(result.candidates).toEqual([ + expect.objectContaining({ documentId: "f:match", path, storageExists: false, source: "metadata" }), + ]); + expect(progressCounts).toEqual([0, 128, 128]); + }); + + it("projects an exact target with the token salt, preserves revision generations, and omits raw values", async () => { + const path = "private-name.md"; + const salt = "salt1234"; + const token = diagnosticPath(path, salt); + const selected = { + _id: "f:selected-private-id", + _rev: "3-private-revision", + _conflicts: [], + _revs_info: [{ rev: "3-private-revision", status: "private-status-value" }], + path, + ctime: 11, + mtime: 22, + size: 33, + type: "plain", + data: "private file contents", + children: ["h:private-chunk-id"], + eden: {}, + unreported: "private-extra-value", + }; + const decoy = { ...selected, _id: "f:path-derived", path: "different.md", _rev: "9-decoy-revision" }; + const core = createCore([path], [selected, decoy]); + core.localDatabase.localDatabase.get.mockImplementation(async (id: string) => { + if (id === selected._id) return selected; + if (id === decoy._id) return decoy; + throw Object.assign(new Error("missing"), { status: 404 }); + }); + const candidate = { + path, + databasePath: path as never, + documentId: selected._id, + storageExists: true, + source: "metadata" as const, + }; + + const output = await buildPseudonymousFileReport(core as never, candidate, salt); + const jsonText = output.artifacts.find(({ format }) => format === "json")!.content; + const json = JSON.parse(jsonText); + const revision = json.file.database.revisions[0]; + + expect(output.nameMode).toBe("pseudonym"); + expect(json.kind).toBe("file"); + expect(json.nameMode).toBe("pseudonym"); + expect(json.salt).toBe(salt); + expect(json.file.path).toBe(token); + expect(json.scope).toMatchObject({ vaultContentRead: false, fileContents: "omitted" }); + expect(revision.documentId).toMatch(/^document-[0-9a-f]{32}$/); + expect(revision.revision).toMatch(/^3-psu[0-9a-f]{32}$/); + expect(revision.chunks[0].id).toMatch(/^chunk-[0-9a-f]{32}$/); + expect(revision.revisionHistory[0].status).toBe("unknown"); + expect(json.file.database.inspected).toBe(true); + expect(output.artifacts.map(({ format }) => format)).toEqual(["json", "markdown"]); + expect(output.artifacts.find(({ format }) => format === "markdown")?.content).toContain("```json"); + expect(jsonText).not.toContain(path); + expect(jsonText).not.toContain(selected._id); + expect(jsonText).not.toContain("3-private-revision"); + expect(jsonText).not.toContain("h:private-chunk-id"); + expect(jsonText).not.toContain("private-status-value"); + expect(jsonText).not.toContain("private file contents"); + expect(jsonText).not.toContain("private-extra-value"); + expect(core.localDatabase.localDatabase.get.mock.calls.map(([id]) => id)).toEqual([selected._id]); + expect(core.services.path.path2id).not.toHaveBeenCalled(); + }); + + it("keeps an exact Metadata target visibly missing without substituting a path-derived document", async () => { + const path = "vanished.md"; + const salt = "salt1234"; + const selected = { _id: "f:vanished", path, type: "plain" }; + const decoy = { _id: "f:path-derived", path: "wrong.md", type: "plain", _rev: "1-decoy" }; + const core = createCore([path], [selected, decoy]); + const lookup = await findDiagnosticFiles(core as never, diagnosticPath(path, salt)); + core.localDatabase.localDatabase.get.mockImplementation(async (id: string) => { + if (id === decoy._id) return decoy; + throw Object.assign(new Error("missing"), { status: 404 }); + }); + + const output = await buildPseudonymousFileReport(core as never, lookup.candidates[0], salt); + const jsonText = output.artifacts.find(({ format }) => format === "json")!.content; + const json = JSON.parse(jsonText); + + expect(json.file.database).toMatchObject({ inspected: true, exists: false }); + expect(json.file.database.documentId).toMatch(/^document-[0-9a-f]{32}$/); + expect(jsonText).not.toContain("f:vanished"); + expect(jsonText).not.toContain("f:path-derived"); + expect(core.localDatabase.localDatabase.get.mock.calls.map(([id]) => id)).toEqual([selected._id]); + expect(core.services.path.path2id).not.toHaveBeenCalled(); + }); +}); diff --git a/src/serviceFeatures/diagnosticFileLookupTypes.ts b/src/serviceFeatures/diagnosticFileLookupTypes.ts new file mode 100644 index 00000000..6a08cc2a --- /dev/null +++ b/src/serviceFeatures/diagnosticFileLookupTypes.ts @@ -0,0 +1,36 @@ +import type { FilePath, FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { DatabaseDiagnosticOutput } from "./databaseDiagnostics"; + +export type DiagnosticFileCandidate = { + path: string; + databasePath?: FilePathWithPrefix | FilePath; + documentId?: string; + storageExists: boolean; + source: "metadata" | "vault"; +}; + +export type DiagnosticFileLookupResult = + | { + valid: false; + status: "invalid"; + processed: 0; + candidates: []; + } + | { + valid: true; + status: "cancelled" | "complete"; + processed: number; + candidates: DiagnosticFileCandidate[]; + }; + +export interface FindDiagnosticFilesOptions { + signal?: AbortSignal; + onProgress?(processed: number): void; +} + +export interface DiagnosticFileLookupDialogOptions { + find(token: string, onProgress: (processed: number) => void): Promise; + prepareReport(candidate: DiagnosticFileCandidate, token: string): Promise; + openFile?(path: string): Promise | void; + cancel(): void; +} diff --git a/src/serviceFeatures/fileDatabaseInfo.ts b/src/serviceFeatures/fileDatabaseInfo.ts index 300ab4e3..8fbf06e3 100644 --- a/src/serviceFeatures/fileDatabaseInfo.ts +++ b/src/serviceFeatures/fileDatabaseInfo.ts @@ -91,6 +91,7 @@ export type FileDatabaseInfo = { source: "local database on this device"; remoteQueried: false; exists: boolean; + documentId?: string; currentRevision: string | null; conflictCount: number; conflictRevisions: string[]; @@ -100,6 +101,12 @@ export type FileDatabaseInfo = { }; }; +export type InspectFileDatabaseInfoOptions = { + documentId?: string; + databasePath?: FilePathWithPrefix | FilePath; + signal?: AbortSignal; +}; + const REPORT_WARNING = "All revisions and chunk availability below are a snapshot of this device's local database; the remote is not queried. Review the Vault-relative path, document identifier, content-derived chunk identifiers, and metadata before sharing this report. File contents are omitted."; @@ -125,25 +132,35 @@ type RawDatabaseDocument = { data?: string | string[]; eden?: Record; mtime?: number; + path?: FilePathWithPrefix | FilePath; size?: number; type?: string; }; +function checkInspectionCancelled(signal?: AbortSignal): void { + if (signal?.aborted) throw new DOMException("File inspection cancelled", "AbortError"); +} + async function getLocalDatabaseMeta( core: FileDatabaseInfoCore, path: FilePathWithPrefix | FilePath, - options: PouchDB.Core.GetOptions + options: PouchDB.Core.GetOptions, + target?: InspectFileDatabaseInfoOptions ): Promise { - const documentId = await core.services.path.path2id(path); + checkInspectionCancelled(target?.signal); + const documentId = target?.documentId ?? (await core.services.path.path2id(path)); + checkInspectionCancelled(target?.signal); let raw: RawDatabaseDocument; try { raw = await core.localDatabase.localDatabase.get(documentId, options); } catch (error) { + checkInspectionCancelled(target?.signal); if (isNotFoundError(error)) { return false; } throw error; } + checkInspectionCancelled(target?.signal); if (raw.type === "leaf") { return false; @@ -156,13 +173,15 @@ async function getLocalDatabaseMeta( const legacy = rawStorageType === null || rawStorageType === "notes"; const type = legacy ? "notes" : rawStorageType; const legacyBodyPresent = - legacy && (typeof raw.data === "string" || (Array.isArray(raw.data) && raw.data.every((item) => typeof item === "string"))); + legacy && + (typeof raw.data === "string" || + (Array.isArray(raw.data) && raw.data.every((item) => typeof item === "string"))); return { _id: raw._id, _rev: raw._rev, _conflicts: raw._conflicts, _revs_info: raw._revs_info, - path, + path: target ? (raw.path ?? path) : path, data: legacyBodyPresent ? raw.data : "", ctime: raw.ctime ?? 0, mtime: raw.mtime ?? 0, @@ -180,8 +199,10 @@ async function getLocalDatabaseMeta( async function collectRevisionDatabaseInfo( core: FileDatabaseInfoCore, meta: DatabaseMeta, - current: boolean + current: boolean, + signal?: AbortSignal ): Promise { + checkInspectionCancelled(signal); const legacy = meta._rawStorageType === null || meta._rawStorageType === "notes"; const children = legacy ? [] : "children" in meta ? meta.children : []; const uniqueChildren = [...new Set(children)]; @@ -201,6 +222,7 @@ async function collectRevisionDatabaseInfo( include_docs: false, }) ).rows; + checkInspectionCancelled(signal); const localChunkStates = new Map( localRows .filter((row) => "value" in row) @@ -236,9 +258,7 @@ async function collectRevisionDatabaseInfo( locallyStoredChunkReferences: children.filter((id) => localChunkStates.get(id)?.state === "available").length, contentAvailableLocally: legacy ? meta._legacyBodyPresent - : uniqueChildren.every( - (id) => embeddedChildren.has(id) || localChunkStates.get(id)?.state === "available" - ), + : uniqueChildren.every((id) => embeddedChildren.has(id) || localChunkStates.get(id)?.state === "available"), chunks: uniqueChildren.map((id) => { const localState = localChunkStates.get(id); return { @@ -273,15 +293,27 @@ function missingChunkIds(info: RevisionDatabaseInfo): string[] { .map(({ id }) => id); } -export async function inspectFileDatabaseInfo(core: FileDatabaseInfoCore, path: string): Promise { +export async function inspectFileDatabaseInfo( + core: FileDatabaseInfoCore, + path: string, + options?: InspectFileDatabaseInfoOptions +): Promise { + checkInspectionCancelled(options?.signal); const storageExists = await core.storageAccess.isExistsIncludeHidden(path); + checkInspectionCancelled(options?.signal); const storageStat = storageExists ? await core.storageAccess.statHidden(path) : null; - const databasePath = toDatabasePath(path); - const currentMeta = await getLocalDatabaseMeta(core, databasePath, { - conflicts: true, - revs: true, - revs_info: true, - }); + const requestedDatabasePath = options?.databasePath ?? toDatabasePath(path); + const currentMeta = await getLocalDatabaseMeta( + core, + requestedDatabasePath, + { + conflicts: true, + revs: true, + revs_info: true, + }, + options + ); + const databasePath = options && currentMeta ? currentMeta.path : requestedDatabasePath; const revisions: RevisionDatabaseInfo[] = []; const conflictRevisions = currentMeta === false ? [] : (currentMeta._conflicts ?? []); @@ -292,25 +324,31 @@ export async function inspectFileDatabaseInfo(core: FileDatabaseInfoCore, path: metadataByRevision.set(currentMeta._rev, currentMeta); } const getRevisionMeta = async (revision: string): Promise => { + checkInspectionCancelled(options?.signal); const cached = metadataByRevision.get(revision); if (cached !== undefined) { return cached; } - const meta = await getLocalDatabaseMeta(core, databasePath, { - rev: revision, - revs: true, - revs_info: true, - }); + const meta = await getLocalDatabaseMeta( + core, + requestedDatabasePath, + { + rev: revision, + revs: true, + revs_info: true, + }, + options + ); metadataByRevision.set(revision, meta); return meta; }; if (currentMeta) { - revisions.push(await collectRevisionDatabaseInfo(core, currentMeta, true)); + revisions.push(await collectRevisionDatabaseInfo(core, currentMeta, true, options?.signal)); for (const revision of conflictRevisions) { const conflictMeta = await getRevisionMeta(revision); if (conflictMeta) { - revisions.push(await collectRevisionDatabaseInfo(core, conflictMeta, false)); + revisions.push(await collectRevisionDatabaseInfo(core, conflictMeta, false, options?.signal)); const winnerHistory = revisionHistory(currentMeta); const conflictHistory = revisionHistory(conflictMeta); const conflictHistoryByRevision = new Map( @@ -323,13 +361,12 @@ export async function inspectFileDatabaseInfo(core: FileDatabaseInfoCore, path: const unavailableSharedRevisions = sharedHistory .filter( ({ revision: historyRevision, status }) => - status !== "available" || - conflictHistoryByRevision.get(historyRevision) !== "available" + status !== "available" || conflictHistoryByRevision.get(historyRevision) !== "available" ) .map(({ revision: historyRevision }) => historyRevision); const sharedMeta = sharedRevision ? await getRevisionMeta(sharedRevision) : false; const sharedInfo = sharedMeta - ? await collectRevisionDatabaseInfo(core, sharedMeta, false) + ? await collectRevisionDatabaseInfo(core, sharedMeta, false, options?.signal) : undefined; mergeBases.push({ winnerRevision: currentMeta._rev ?? "", @@ -346,6 +383,7 @@ export async function inspectFileDatabaseInfo(core: FileDatabaseInfoCore, path: } } + checkInspectionCancelled(options?.signal); const report: FileDatabaseInfo = { path, databasePath, @@ -363,6 +401,7 @@ export async function inspectFileDatabaseInfo(core: FileDatabaseInfoCore, path: source: "local database on this device", remoteQueried: false, exists: currentMeta !== false, + ...(options?.documentId !== undefined ? { documentId: options.documentId } : {}), currentRevision: currentMeta ? (currentMeta._rev ?? null) : null, conflictCount: conflictRevisions.length, conflictRevisions, diff --git a/src/serviceFeatures/fileDatabaseInfo.unit.spec.ts b/src/serviceFeatures/fileDatabaseInfo.unit.spec.ts index 1d1b6207..a9c8dcb1 100644 --- a/src/serviceFeatures/fileDatabaseInfo.unit.spec.ts +++ b/src/serviceFeatures/fileDatabaseInfo.unit.spec.ts @@ -198,8 +198,10 @@ describe("file database information", () => { it("reports the exact shared ancestor and its missing chunks for each conflict", async () => { const { conflict, core, current } = createCore(); + const selectedCurrent = { ...current, _id: "f:selected" }; + const selectedConflict = { ...conflict, _id: "f:selected" }; const parent = { - ...current, + ...selectedCurrent, _rev: "2-parent", _conflicts: undefined, _revs_info: [ @@ -212,7 +214,7 @@ describe("file database information", () => { core.localDatabase.localDatabase.get.mockImplementation(async (_id: string, options?: { rev?: string }) => { if (options?.rev === "2-conflict") { return { - ...conflict, + ...selectedConflict, _revs_info: [ { rev: "2-conflict", status: "available" }, { rev: "2-parent", status: "available" }, @@ -224,7 +226,7 @@ describe("file database information", () => { return parent; } return { - ...current, + ...selectedCurrent, _revs_info: [ { rev: "3-current", status: "available" }, { rev: "2-parent", status: "available" }, @@ -233,7 +235,10 @@ describe("file database information", () => { }; }); - const info = await inspectFileDatabaseInfo(core as never, "note.md"); + const info = await inspectFileDatabaseInfo(core as never, "note.md", { + documentId: "f:selected", + databasePath: "note.md" as never, + }); expect(info.database.mergeBases).toEqual([ { @@ -246,6 +251,64 @@ describe("file database information", () => { unavailableSharedRevisions: ["1-root"], }, ]); + expect(core.localDatabase.localDatabase.get.mock.calls.map(([id]) => id)).toEqual([ + "f:selected", + "f:selected", + "f:selected", + ]); + }); + + it("inspects the exact selected document when its ID differs from the path-derived ID", async () => { + const { core, current } = createCore(); + const selected = { + ...current, + _id: "f:selected", + _rev: "4-selected", + _conflicts: [], + path: "stored-name.md", + children: [], + }; + core.localDatabase.localDatabase.get.mockImplementation(async (id: string) => + id === "f:selected" ? selected : current + ); + + const info = await inspectFileDatabaseInfo(core as never, "note.md", { + documentId: "f:selected", + databasePath: "listed-name.md" as never, + }); + + expect(info.database.exists).toBe(true); + expect(info.database.revisions).toEqual([ + expect.objectContaining({ + documentId: "f:selected", + revision: "4-selected", + }), + ]); + expect(info.databasePath).toBe("stored-name.md"); + expect(info.database.documentId).toBe("f:selected"); + expect(core.services.path.path2id).not.toHaveBeenCalled(); + }); + + it("reports a missing exact document without falling back to the path-derived ID", async () => { + const { core, current } = createCore(); + core.localDatabase.localDatabase.get.mockImplementation(async (id: string) => { + if (id === "f:missing") { + throw Object.assign(new Error("missing"), { status: 404 }); + } + return current; + }); + + const info = await inspectFileDatabaseInfo(core as never, "note.md", { + documentId: "f:missing", + databasePath: "requested-name.md" as never, + }); + + expect(info.database.exists).toBe(false); + expect(info.database.revisions).toEqual([]); + expect(info.databasePath).toBe("requested-name.md"); + expect(info.database.documentId).toBe("f:missing"); + expect(core.localDatabase.localDatabase.get.mock.calls.map(([id]) => id)).toEqual(["f:missing"]); + expect(core.services.path.path2id).not.toHaveBeenCalled(); }); it("does not decode a revision whose chunks are not all available locally", async () => { @@ -293,13 +356,7 @@ describe("file database information", () => { await retryReadFileDatabaseRevision(core as never, "note.md", "2-conflict"); - expect(core.localDatabase.getDBEntry).toHaveBeenCalledWith( - "note.md", - { rev: "2-conflict" }, - false, - true, - true - ); + expect(core.localDatabase.getDBEntry).toHaveBeenCalledWith("note.md", { rev: "2-conflict" }, false, true, true); }); it("reports the exact revision as locally available after retry recovers its missing chunk", async () => { @@ -322,14 +379,10 @@ describe("file database information", () => { : [], })); - await expect( - retryReadFileDatabaseRevision(core as never, "note.md", "2-conflict") - ).resolves.not.toBe(false); + await expect(retryReadFileDatabaseRevision(core as never, "note.md", "2-conflict")).resolves.not.toBe(false); const information = await inspectFileDatabaseInfo(core as never, "note.md"); - expect( - information.database.revisions.find(({ revision }) => revision === "2-conflict") - ).toEqual( + expect(information.database.revisions.find(({ revision }) => revision === "2-conflict")).toEqual( expect.objectContaining({ contentAvailableLocally: true, chunks: [ diff --git a/src/serviceFeatures/useDatabaseDiagnostics.ts b/src/serviceFeatures/useDatabaseDiagnostics.ts index 71021b1c..2fa974c8 100644 --- a/src/serviceFeatures/useDatabaseDiagnostics.ts +++ b/src/serviceFeatures/useDatabaseDiagnostics.ts @@ -23,11 +23,12 @@ export const useDatabaseDiagnosticsFeature = createServiceFeature((host: Databas try { await services.UI.dialogManager.open<"close", DatabaseDiagnosticDialogOptions>(DatabaseDiagnosticDialog, { kind, - run: async (nameMode, onProgress) => { + run: async (nameMode, onProgress, includeRemote = true) => { if (!services.database.isDatabaseReady()) throw new Error("local-database-unavailable"); return await runDatabaseDiagnostic(host, kind, nameMode, { signal: controller.signal, onProgress, + includeRemote, }); }, cancel: () => controller.abort(), diff --git a/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts index 0e9669e0..8372b5ef 100644 --- a/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts +++ b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts @@ -119,4 +119,29 @@ describe("database diagnostics serviceFeature", () => { data.close(); await opened; }); + + it("forwards the per-run remote inspection choice and defaults existing callers to include the remote database", async () => { + vi.mocked(runDatabaseDiagnostic).mockClear(); + const data = fixture(); + const opened = data.view.open("integrity"); + const options = data.openDialog.mock.calls[0][1]; + + await options.run("pseudonym", () => {}, false); + expect(runDatabaseDiagnostic).toHaveBeenLastCalledWith( + data.host, + "integrity", + "pseudonym", + expect.objectContaining({ includeRemote: false, signal: expect.any(AbortSignal) }) + ); + + await options.run("pseudonym", () => {}); + expect(runDatabaseDiagnostic).toHaveBeenLastCalledWith( + data.host, + "integrity", + "pseudonym", + expect.objectContaining({ includeRemote: true, signal: expect.any(AbortSignal) }) + ); + data.close(); + await opened; + }); }); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 0a31cae4..05aadf06 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -87,9 +87,11 @@ npm run test:e2e:obsidian:focused -- couchdb-http-cache The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. -`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. +`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, the default-on integrity-only remote option, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout at 800×600 and 375×667, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. -`database-diagnostics-couchdb` needs the CouchDB fixture with CORS enabled. It checks unencrypted and encrypted databases, both browser fetch and Obsidian's native Request API, and both name policies through the real integrity dialogue. A remote-only document with a mismatched ID must produce a finding with readable Chunks and equal content. JSON and Markdown must omit file contents and credentials. The scenario observes real requests and requires GET-only access, unchanged local settings, revisions, and Vault contents, and unchanged remote documents and Security Seed. Commonlib's separate CouchDB integration tests additionally cover a missing Security Seed without creating one. +The same scenario checks the five actions in Reporting Tools, the separate Checks and Recovery panel, Metadata progress with its document total and gauge, and collapsible Markdown. It also observes Usage revision progress with a count and discovery note during a gated read. Local lookup checks cover multiple document IDs for one path, selection of an exact ID, and a follow-up report using the original salt. The compact Markdown must retain every JSON observation reference and pseudonym, and its copy and save payloads must match the full output. Results and export controls are checked at 800×600 on desktop and 375×667 in mobile emulation, including after copy and save feedback appears. + +`database-diagnostics-couchdb` needs the CouchDB fixture with CORS enabled. It checks that clearing the remote option completes a local database and Vault inspection with remote coverage marked not checked and makes no HTTP requests, then confirms that each new run includes the remote database by default. It also checks unencrypted and encrypted databases, both browser fetch and Obsidian's native Request API, and both name policies through the real integrity dialogue. A remote-only document with a mismatched ID must produce a finding with readable Chunks and equal content. JSON and Markdown must omit file contents and credentials. Included inspections observe real requests and require GET-only access, unchanged local settings, revisions, and Vault contents, and unchanged remote documents and Security Seed. Commonlib's separate CouchDB integration tests additionally cover a missing Security Seed without creating one. `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. @@ -326,6 +328,8 @@ E2E_OBSIDIAN_ONLY_INDEPENDENT_IDS=true npm run test:e2e:obsidian:two-vault-sync The first scenario needs no service; the other two need CouchDB. The last checks an encrypted note round trip between two real Obsidian Vaults, with Path Obfuscation and independent IDs, and rejection of incompatible keys. These checks complement Commonlib's classification, conflict, privacy, and cancellation tests. +For a real-Obsidian follow-up check, copy a path token from a pseudonym Usage or integrity report and use Hatch → Reporting Tools → **Find a file from a report pseudonym**. Confirm that **Find file** searches only the current Vault and local normal-file Metadata, returns every matching local document ID, and shows real paths and IDs only in the local dialogue. Include cases with several IDs for one path, a Vault-only candidate whose Metadata is marked not inspected, and a path which no longer matches locally. Choose a result to open an existing Vault file or prepare a shareable local report; check that it reuses the report salt and exact document ID, retains the selected pseudonym mode, and contains no file contents or credentials. Check that the lookup makes no remote requests. Its history status listing is not a scan of every historical body. + Set `E2E_OBSIDIAN_ARTIFACT_ROOT` to a directory containing the exact release `main.js`, `manifest.json`, and `styles.css` when testing downloaded artefacts. These three scenarios accept that directory. Invoke their direct commands above, because the focused wrapper rebuilds local artefacts. Record the commit, Obsidian version, artefact hashes, and results. If testing a local build, compare its hashes with the published release before relying on that evidence for BRAT acceptance. When these checks pass for the release artefacts, BRAT acceptance can be limited to installation or update, enabling the plug-in, and a cold start. Check any other changes included in that release using their own regression scenarios. The diagnostic export checks verify report payloads; they do not exercise the operating system's clipboard or download implementation on iOS or Android. diff --git a/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts b/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts index d51279bf..c7b39756 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts @@ -36,11 +36,16 @@ const timeout = 20000; interface IntegrityReport { status: string; + scope: { remote: "included" | "excluded" }; coverage: { local: { status: string }; remote: { status: string; contentReads: number } }; observations: { source: string; identity: string; - content: { status: string; chunksRead: number }; + path: string; + revision?: string; + metadata?: { recordedSize?: number; chunkReferences: number; uniqueChunkReferences: number }; + vault: { status: string; size?: number; contentSizeDifference?: number; mtimeDifferenceMs?: number }; + content: { status: string; chunksRead: number; size?: number; recordedSizeDifference?: number }; comparisons: { relation: string }[]; }[]; errors: unknown[]; @@ -165,6 +170,73 @@ async function runCase(config: CouchDbConfig, encrypted: boolean) { let calls: { transport: string; method: string }[] = []; try { await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 800, height: 600 }); + const opened = await page.evaluate( + "app.commands.executeCommandById('obsidian-livesync:inspect-database-integrity')" + ); + assert.equal(opened, true); + const dialogue = page.locator(".database-diagnostic"); + await dialogue.waitFor({ state: "visible", timeout }); + const remoteChoice = dialogue.getByRole("checkbox", { + name: "Include remote database", + exact: true, + }); + assert.equal(await remoteChoice.isChecked(), true, "Remote inspection must be selected by default"); + const modal = page.locator(".modal-container").filter({ has: dialogue }).locator(".modal"); + const layout = await modal.evaluate((element) => { + const modalBounds = element.getBoundingClientRect(); + const choice = element.querySelector(".diagnostic-remote-choice"); + if (!choice) + return { contained: false, outerScroll: element.scrollHeight - element.clientHeight }; + const choiceBounds = choice.getBoundingClientRect(); + return { + contained: + choiceBounds.top >= modalBounds.top - 1 && + choiceBounds.bottom <= modalBounds.bottom + 1, + outerScroll: Math.max(0, element.scrollHeight - element.clientHeight), + }; + }); + assert.equal(layout.contained, true, "The remote option must remain inside the desktop modal"); + assert.ok( + layout.outerScroll <= 1, + `The choice modal must not need outer scrolling; got ${layout.outerScroll}px` + ); + await remoteChoice.uncheck(); + assert.equal(await remoteChoice.isChecked(), false); + await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await dialogue.getByRole("radio", { name: "JSON", exact: true }).check(); + const report = JSON.parse(await dialogue.locator("textarea").inputValue()) as IntegrityReport; + assert.equal(report.status, "complete"); + assert.equal(report.scope.remote, "excluded"); + assert.equal(report.coverage.local.status, "complete"); + assert.equal(report.coverage.remote.status, "not-checked"); + assert.deepEqual(report.errors, []); + assert.equal( + report.observations.some((row) => row.source === "remote"), + false + ); + const localObservations = report.observations.filter((row) => row.source === "local"); + assert.ok(localObservations.length > 0); + assert.ok(localObservations.every((row) => row.identity === "consistent")); + assert.ok( + localObservations.every( + (row) => row.content.status === "not-checked" && row.vault.status === "not-checked" + ), + "Only the remote has a discrepancy in this fixture; excluding it leaves no suspect content group" + ); + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); + }); + } finally { + calls = await finishRequests(cli, session); + } + assert.equal(calls.length, 0, "Excluding the remote database must make no HTTP requests"); + + await observeRequests(cli, session); + try { + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 800, height: 600 }); for (const names of ["pseudonym", "original"]) { const opened = await page.evaluate( "app.commands.executeCommandById('obsidian-livesync:inspect-database-integrity')" @@ -172,6 +244,15 @@ async function runCase(config: CouchDbConfig, encrypted: boolean) { assert.equal(opened, true); const dialogue = page.locator(".database-diagnostic"); await dialogue.waitFor({ state: "visible", timeout }); + const remoteChoice = dialogue.getByRole("checkbox", { + name: "Include remote database", + exact: true, + }); + assert.equal( + await remoteChoice.isChecked(), + true, + "Each run must include the remote database by default" + ); const nameOptions = dialogue.getByRole("group", { name: "Names in this report", exact: true }); assert.equal(await nameOptions.getByRole("radio").count(), 2); const nameLabel = names === "pseudonym" ? "Pseudonyms (recommended)" : "Original names"; @@ -187,6 +268,7 @@ async function runCase(config: CouchDbConfig, encrypted: boolean) { const text = await dialogue.locator("textarea").inputValue(); const report = JSON.parse(text) as IntegrityReport; assert.equal(report.status, "complete"); + assert.equal(report.scope.remote, "included"); assert.equal(report.coverage.local.status, "complete"); assert.equal(report.coverage.remote.status, "complete"); assert.ok(report.coverage.remote.contentReads > 0); @@ -197,6 +279,22 @@ async function runCase(config: CouchDbConfig, encrypted: boolean) { assert.ok(suspect); assert.equal(suspect.content.status, "available"); assert.ok(suspect.content.chunksRead > 0); + const byteLength = Buffer.byteLength(fixtureContent); + assert.equal( + suspect.metadata?.recordedSize, + byteLength, + "Metadata properties must be decoded before reporting" + ); + assert.equal(suspect.metadata?.chunkReferences, entry.children.length); + assert.equal(suspect.content.size, byteLength); + assert.equal(suspect.content.recordedSizeDifference, 0); + assert.equal(suspect.vault.size, byteLength); + assert.equal(suspect.vault.contentSizeDifference, 0); + assert.ok(Number.isFinite(suspect.vault.mtimeDifferenceMs)); + if (names === "pseudonym") { + assert.match(suspect.path, /^[A-Za-z0-9_-]{8}\$[0-9a-f]{32}$/); + assert.match(suspect.revision ?? "", /^1-psu[0-9a-f]{32}$/); + } assert.ok(suspect.comparisons.some((comparison) => comparison.relation === "equal")); await formats.getByRole("radio", { name: "MARKDOWN", exact: true }).check(); assert.equal(await formats.getByRole("radio", { checked: true }).count(), 1); diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts index d4f44ae9..49080c5c 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -6,6 +6,7 @@ import { assertNoHorizontalOverflow, } from "@vrtmrz/obsidian-test-session"; import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc"; +import { createDiagnosticNameMapper, parseDiagnosticPathToken } from "@vrtmrz/livesync-commonlib/diagnostics"; import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; import { evalObsidianJson } from "../runner/cli.ts"; import { @@ -49,11 +50,35 @@ async function openCommand(page: Page, kind: "usage" | "integrity") { assert.equal(await names.getByRole("radio", { name, exact: true }).isVisible(), true); assert.equal(await names.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).isChecked(), true); assert.equal(await names.getByRole("radio", { checked: true }).count(), 1); + const remoteChoice = dialogue.getByRole("checkbox", { name: "Include remote database", exact: true }); + if (kind === "integrity") { + assert.equal(await remoteChoice.count(), 1); + assert.equal(await remoteChoice.isChecked(), true, "The remote database must be included by default"); + } else assert.equal(await remoteChoice.count(), 0, "Usage analysis has no remote database option"); assert.equal(await dialogue.getAttribute("data-diagnostic-phase"), "choice"); assert.equal(await dialogue.locator("textarea").count(), 0); return dialogue; } +async function assertDesktopDiagnosticChoiceLayout(page: Page, dialogue: Locator): Promise { + const modal = page.locator(".modal-container").filter({ has: dialogue }).locator(".modal"); + const layout = await modal.evaluate((element) => { + const modalBounds = element.getBoundingClientRect(); + const choice = element.querySelector(".diagnostic-remote-choice"); + if (!choice) return { contained: false, outerScroll: Math.max(0, element.scrollHeight - element.clientHeight) }; + const choiceBounds = choice.getBoundingClientRect(); + return { + contained: choiceBounds.top >= modalBounds.top - 1 && choiceBounds.bottom <= modalBounds.bottom + 1, + outerScroll: Math.max(0, element.scrollHeight - element.clientHeight), + }; + }); + assert.equal(layout.contained, true, "The remote database option must remain inside the desktop modal"); + assert.ok( + layout.outerScroll <= 1, + `The diagnostic choice must not need outer scrolling; got ${layout.outerScroll}px` + ); +} + async function assertFormatChoices(dialogue: Locator, formats: string[]): Promise { const group = dialogue.getByRole("group", { name: "Report format", exact: true }); assert.equal(await group.getByRole("radio").count(), formats.length); @@ -69,6 +94,31 @@ async function chooseFormat(dialogue: Locator, format: string): Promise { assert.equal(await group.getByRole("radio", { checked: true }).count(), 1); } +async function assertDiagnosticExportActionsWithinModal(page: Page, dialogue: Locator, action: string): Promise { + for (const name of ["Copy report", "Save report"]) { + const button = dialogue.getByRole("button", { name, exact: true }); + assert.equal(await button.isVisible(), true, `${name} must remain visible after ${action}`); + const contained = await button.evaluate((element) => { + const modalElement = element.closest(".modal"); + if (!modalElement || !modalElement.contains(element)) return false; + const buttonBounds = element.getBoundingClientRect(); + const modalBounds = modalElement.getBoundingClientRect(); + return buttonBounds.top >= modalBounds.top - 1 && buttonBounds.bottom <= modalBounds.bottom + 1; + }); + assert.equal(contained, true, `${name} must remain inside the modal after ${action}`); + } +} + +async function assertDesktopDiagnosticResultLayout(page: Page, dialogue: Locator, action: string): Promise { + const modal = page.locator(".modal-container").filter({ has: dialogue }).locator(".modal"); + const outerScroll = await modal.evaluate((element) => Math.max(0, element.scrollHeight - element.clientHeight)); + assert.ok( + outerScroll <= 1, + `The diagnostic modal must not need outer scrolling after ${action}; got ${outerScroll}px` + ); + await assertDiagnosticExportActionsWithinModal(page, dialogue, action); +} + async function assertMobileRadioTargets(page: Page, dialogue: Locator, name: string): Promise { const radios = dialogue.getByRole("group", { name, exact: true }).getByRole("radio"); for (let index = 0; index < (await radios.count()); index++) { @@ -113,11 +163,174 @@ async function installDiagnosticExportInterception(page: Page): Promise { async function restoreDiagnosticReadGate(cli: string, session: ObsidianLiveSyncSession): Promise { await evalObsidianJson( cli, - `(async()=>{const gate=globalThis.databaseDiagnosticReadGate;if(gate){gate.release();const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase.localDatabase;db.get=gate.originalGet;delete globalThis.databaseDiagnosticReadGate;}return JSON.stringify(true);})()`, + `(async()=>{const gate=globalThis.databaseDiagnosticReadGate;if(gate){gate.release();if(gate.reached&&gate.drainPromise)await gate.drainPromise;const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase.localDatabase;db.get=gate.originalGet;delete globalThis.databaseDiagnosticReadGate;}return JSON.stringify(true);})()`, session.cliEnv ); } +async function installDiagnosticReadGate( + cli: string, + session: ObsidianLiveSyncSession, + documentId: string, + readKind: "metadata" | "revision" +): Promise { + const condition = readKind === "metadata" ? "options?.conflicts" : "options?.rev"; + await evalObsidianJson( + cli, + `(async()=>{const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase.localDatabase;const gate={reached:false,release:null,releasePromise:null,drain:null,drainPromise:null,originalGet:db.get};gate.releasePromise=new Promise(resolve=>{gate.release=resolve;});gate.drainPromise=new Promise(resolve=>{gate.drain=resolve;});globalThis.databaseDiagnosticReadGate=gate;db.get=async function(id,options){if(id===${JSON.stringify(documentId)}&&${condition}&&!gate.reached){gate.reached=true;try{await gate.releasePromise;return await gate.originalGet.call(this,id,options);}finally{gate.drain();}}return await gate.originalGet.call(this,id,options);};return JSON.stringify(true);})()`, + session.cliEnv + ); +} + +async function assertIntegrityProgress(dialogue: Locator): Promise { + const progress = dialogue.locator(".diagnostic-progress"); + await progress.getByText("Metadata ¡ local", { exact: true }).waitFor({ state: "visible", timeout }); + const progressBar = progress.getByRole("progressbar"); + assert.equal(await progressBar.count(), 1, "Metadata inspection must expose a progressbar"); + assert.equal(await progressBar.isVisible(), true); + const numericText = progress.getByText(/^[1-9]\d* \/ \d+ documents?$/); + await numericText.waitFor({ state: "visible", timeout }); + const numeric = await numericText.innerText(); + const match = /^(\d+) \/ (\d+) documents?$/.exec(numeric); + assert.ok(match, `Metadata progress must report its document count; got ${numeric}`); + const [, processedText, totalText] = match; + const processed = Number(processedText); + const total = Number(totalText); + assert.ok(processed > 0, `Metadata progress must report a positive processed count; got ${numeric}`); + assert.ok(total >= processed, `Metadata progress must report its total; got ${numeric}`); +} + +async function assertHatchReportingGroups(hatch: Locator): Promise { + const reportingHeading = hatch.getByRole("heading", { name: "Reporting Tools", exact: true }); + const checksHeading = hatch.getByRole("heading", { name: "Checks and Recovery", exact: true }); + assert.equal(await reportingHeading.count(), 1); + assert.equal(await checksHeading.count(), 1); + const reportingPanel = reportingHeading.locator(".."); + const checksPanel = checksHeading.locator(".."); + const reportingNames = [ + "Prepare the 'report' to create an issue", + "Copy database information for a file", + "Find a file from a report pseudonym", + "Inspect database integrity", + "Analyse database usage", + ]; + assert.equal(await reportingPanel.locator(".setting-item").count(), reportingNames.length); + for (const name of reportingNames) { + assert.equal( + await reportingPanel.getByText(name, { exact: true }).count(), + 1, + `${name} belongs in Reporting Tools` + ); + assert.equal( + await checksPanel.getByText(name, { exact: true }).count(), + 0, + `${name} must remain outside Checks and Recovery` + ); + } + assert.equal(await checksPanel.locator(".setting-item").count(), 2); + for (const name of ["Run Doctor", "Scan for Broken files"]) { + const button = checksPanel.getByRole("button", { name, exact: true }); + assert.equal(await button.count(), 1, `${name} belongs in Checks and Recovery`); + assert.equal(await reportingPanel.getByRole("button", { name, exact: true }).count(), 0); + } +} + +async function assertDiagnosticFileLookup( + page: Page, + hatch: Locator, + token: string, + shadowRevision: string, + session: ObsidianLiveSyncSession, + mobile = false +): Promise { + const setting = hatch.locator(".setting-item").filter({ + has: page.getByText("Find a file from a report pseudonym", { exact: true }), + }); + await setting.getByRole("button", { name: "Find file", exact: true }).click(); + const dialogue = page.locator(".diagnostic-file-lookup"); + await dialogue.waitFor({ state: "visible", timeout }); + const input = dialogue.getByLabel("Report path pseudonym", { exact: true }); + await input.fill("file-001"); + await dialogue.getByRole("button", { name: "Find file", exact: true }).click(); + await dialogue.getByText("Enter a valid report path pseudonym.", { exact: true }).waitFor({ timeout }); + await input.fill(token); + await dialogue.getByRole("button", { name: "Find file", exact: true }).click(); + const candidates = dialogue.locator("[data-diagnostic-candidate]"); + await candidates.first().waitFor({ timeout }); + assert.equal(await candidates.count(), 2, "Distinct Metadata IDs claiming the same path must remain selectable"); + if (mobile) { + await assertNoHorizontalOverflow(page, dialogue, { label: "Pseudonym file lookup matches" }); + await assertNoHorizontalOverflow(page, dialogue.locator(".lookup-candidates"), { + label: "Pseudonym file lookup candidate list", + }); + assert.ok( + (await dialogue.locator(".lookup-candidates").evaluate((element) => element.clientHeight)) >= 160, + "The mobile lookup must leave enough room to read the matching file details" + ); + await captureObsidianElement( + session.remoteDebuggingPort, + "database-diagnostics-file-matches-mobile.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".diagnostic-file-lookup") }) + ); + } + const shadow = candidates.filter({ hasText: shadowId }); + assert.equal(await shadow.count(), 1); + assert.ok((await shadow.textContent())?.includes(fixturePath)); + await shadow.getByRole("button", { name: "Open file", exact: true }).click(); + await page.waitForFunction(`app.workspace.getActiveFile()?.path === ${JSON.stringify(fixturePath)}`, undefined, { + timeout, + }); + await shadow.getByRole("button", { name: "Prepare follow-up report", exact: true }).click(); + await dialogue.locator("textarea").waitFor({ timeout }); + const json = await dialogue.locator("textarea").inputValue(); + const report = JSON.parse(json); + const parsed = parseDiagnosticPathToken(token); + assert.ok(parsed); + const mapper = createDiagnosticNameMapper("pseudonym", parsed.salt); + assert.equal(report.file.path, token); + assert.equal(report.file.database.documentId, mapper.id(shadowId)); + assert.equal(report.file.database.currentRevision, mapper.revision(shadowRevision)); + assert.equal(report.file.database.remoteQueried, false); + for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) { + assert.ok(!json.includes(privateValue), "A follow-up report must preserve the original report's pseudonyms"); + } + await installDiagnosticExportInterception(page); + await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); + assert.equal(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), json); + await dialogue.getByRole("button", { name: "Save report", exact: true }).click(); + await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { timeout }); + assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { + name: "diagnostic-file-report.json", + content: json, + }); + await assertDesktopDiagnosticResultLayout(page, dialogue, "saving a follow-up report"); + if (mobile) { + const modal = page.locator(".modal-container").filter({ has: dialogue }); + await assertMobileDialogueLayout(page, modal, "Pseudonym file follow-up report"); + await assertNoHorizontalOverflow(page, dialogue, { label: "Pseudonym file follow-up content" }); + await assertNoHorizontalOverflow(page, dialogue.locator(".report-token"), { + label: "Pseudonym file follow-up token", + }); + } + await captureObsidianElement( + session.remoteDebuggingPort, + mobile ? "database-diagnostics-file-follow-up-mobile.png" : "database-diagnostics-file-follow-up.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".diagnostic-file-lookup") }) + ); + await dialogue.getByRole("radio", { name: "MARKDOWN", exact: true }).check(); + const markdown = await dialogue.locator("textarea").inputValue(); + assert.ok(markdown.startsWith("
\n")); + assert.ok(markdown.includes(json)); + await dialogue.getByRole("button", { name: "Back to matches", exact: true }).click(); + await candidates.first().waitFor({ timeout }); + await input.fill(createDiagnosticNameMapper("pseudonym", parsed.salt).path("not-present.md")); + await dialogue.getByRole("button", { name: "Find file", exact: true }).click(); + await dialogue.getByText(/No matches were found in current Vault paths/).waitFor({ timeout }); + assert.equal(await candidates.count(), 0); + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); +} + async function main(): Promise { const binary = requireObsidianBinary(); const cli = requireObsidianCli(); @@ -161,15 +374,17 @@ async function main(): Promise { session.cliEnv ); await waitForLocalDatabaseEntry(cli, session.cliEnv, fixturePath); - await evalObsidianJson( + const rawFixture = await evalObsidianJson<{ id: string; revision: string; shadowRevision: string }>( cli, - `(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;await core.services.fileProcessing.commitPendingFileEvents();const db=core.localDatabase.localDatabase;const id=await core.services.path.path2id(${JSON.stringify(fixturePath)});const original=await db.get(id);const shadow={...original,_id:${JSON.stringify(shadowId)}};delete shadow._rev;await db.put(shadow);return JSON.stringify(true);})()`, + `(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;await core.services.fileProcessing.commitPendingFileEvents();const db=core.localDatabase.localDatabase;const id=await core.services.path.path2id(${JSON.stringify(fixturePath)});const original=await db.get(id);const shadow={...original,_id:${JSON.stringify(shadowId)}};delete shadow._rev;const stored=await db.put(shadow);return JSON.stringify({id,revision:original._rev,shadowRevision:stored.rev});})()`, session.cliEnv ); const before = await readDatabaseState(cli, session); + const activeSession = session; let usageJSON = ""; let usageFixtureRow: { path: string; id: string; revision: string } | undefined; await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 800, height: 600 }); const choice = await openCommand(page, "usage"); await choice.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).focus(); await page.keyboard.press("ArrowRight"); @@ -188,6 +403,7 @@ async function main(): Promise { ); await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await assertDesktopDiagnosticResultLayout(page, dialogue, "the result appearing"); await assertFormatChoices(dialogue, ["TSV", "JSON"]); assert.equal(await dialogue.getByRole("radio", { name: "TSV", exact: true }).isChecked(), true); const tsv = await dialogue.locator("textarea").inputValue(); @@ -202,17 +418,24 @@ async function main(): Promise { const report = JSON.parse(usageJSON); assert.equal(report.status, "complete"); assert.ok(report.versions.commonlib); - assert.ok(report.rows.some((row: { path: string }) => row.path.startsWith("file-"))); - const fixtureRow = report.rows.find((row: { path: string; id: string; revision: string }) => - row.path.startsWith("file-") + const token = report.rows.find((row: { path: string }) => parseDiagnosticPathToken(row.path))?.path; + const parsedToken = parseDiagnosticPathToken(token ?? ""); + assert.ok(parsedToken, "The report must contain searchable salted path pseudonyms"); + const mapper = createDiagnosticNameMapper("pseudonym", parsedToken.salt); + const fixtureRow = report.rows.find( + (row: { path: string; id: string; revision: string }) => row.id === mapper.id(rawFixture.id) ); assert.ok(fixtureRow); + assert.equal(fixtureRow.path, mapper.path(fixturePath)); + assert.equal(fixtureRow.revision, mapper.revision(rawFixture.revision)); + assert.equal(fixtureRow.revision, `${fixtureRow.rev}-${fixtureRow.revHash}`); usageFixtureRow = { path: fixtureRow.path, id: fixtureRow.id, revision: fixtureRow.revision }; // Capture exports inside the isolated renderer without changing the // host clipboard or leaving downloads in a user's download directory. await installDiagnosticExportInterception(page); await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); + await assertDesktopDiagnosticResultLayout(page, dialogue, "copying the report"); assert.equal( await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), usageJSON @@ -221,6 +444,7 @@ async function main(): Promise { await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { timeout, }); + await assertDesktopDiagnosticResultLayout(page, dialogue, "saving the report"); assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { name: "database-usage-complete.json", content: usageJSON, @@ -240,10 +464,69 @@ async function main(): Promise { await original.waitFor({ state: "hidden", timeout }); const settingsNavigator = await openLiveSyncSettings(page, timeout); - const hatch = await settingsNavigator.openPage("Hatch"); + let hatch = await settingsNavigator.openPage("Hatch"); + await assertHatchReportingGroups(hatch); + assert.ok(usageFixtureRow); + await assertDiagnosticFileLookup( + page, + hatch, + usageFixtureRow.path, + rawFixture.shadowRevision, + activeSession + ); + hatch = await (await openLiveSyncSettings(page, timeout)).openPage("Hatch"); + await captureObsidianElement(activeSession.remoteDebuggingPort, "database-diagnostics-hatch.png", (page) => + page.locator(".modal.mod-settings") + ); const setting = hatch .locator(".setting-item") .filter({ has: page.getByText("Inspect database integrity", { exact: true }) }); + await setting.getByRole("button", { name: "Inspect", exact: true }).click(); + const progressDialogue = page.locator(".database-diagnostic"); + assert.equal( + await progressDialogue + .getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }) + .isChecked(), + true + ); + assert.equal( + await progressDialogue + .getByRole("checkbox", { name: "Include remote database", exact: true }) + .isChecked(), + true + ); + await assertDesktopDiagnosticChoiceLayout(page, progressDialogue); + await installDiagnosticReadGate(cli, activeSession, shadowId, "metadata"); + try { + await progressDialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.waitForFunction( + () => (globalThis as DiagnosticPageGlobal).databaseDiagnosticReadGate?.reached === true, + undefined, + { timeout } + ); + assert.equal(await progressDialogue.getAttribute("data-diagnostic-phase"), "running"); + await assertIntegrityProgress(progressDialogue); + await captureObsidianElement( + activeSession.remoteDebuggingPort, + "database-diagnostics-progress.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + ); + await progressDialogue.getByRole("button", { name: "Cancel inspection", exact: true }).click(); + await evalObsidianJson( + cli, + `(async()=>{globalThis.databaseDiagnosticReadGate?.release();return JSON.stringify(true);})()`, + activeSession.cliEnv + ); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await assertFormatChoices(progressDialogue, ["MARKDOWN", "JSON"]); + await chooseFormat(progressDialogue, "json"); + assert.equal(JSON.parse(await progressDialogue.locator("textarea").inputValue()).status, "cancelled"); + await progressDialogue.getByRole("button", { name: "Close", exact: true }).click(); + await progressDialogue.waitFor({ state: "hidden", timeout }); + } finally { + await restoreDiagnosticReadGate(cli, activeSession); + } + await setting.getByRole("button", { name: "Inspect", exact: true }).click(); const integrity = page.locator(".database-diagnostic"); assert.equal( @@ -254,10 +537,40 @@ async function main(): Promise { await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); await assertFormatChoices(integrity, ["MARKDOWN", "JSON"]); assert.equal(await integrity.getByRole("radio", { name: "MARKDOWN", exact: true }).isChecked(), true); + const markdownReport = await integrity.locator("textarea").inputValue(); + const markdownDetailsWithCodeBlock = + /
\s*Database integrity report \(partial\)<\/summary>\s*(`{3,})text\r?\n[\s\S]*?\1\s*<\/details>/; + assert.match(markdownReport, markdownDetailsWithCodeBlock); + await installDiagnosticExportInterception(page); + await integrity.getByRole("button", { name: "Copy report", exact: true }).click(); + assert.equal( + await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), + markdownReport + ); + await integrity.getByRole("button", { name: "Save report", exact: true }).click(); + await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { + timeout, + }); + assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { + name: "database-integrity-partial.md", + content: markdownReport, + }); await chooseFormat(integrity, "json"); const reportText = await integrity.locator("textarea").inputValue(); const integrityReport = JSON.parse(reportText); assert.equal(integrityReport.status, "partial"); + assert.ok(markdownReport.includes("## Reading the findings")); + assert.ok(markdownReport.includes("## State counts")); + assert.ok(markdownReport.includes("## Review index")); + assert.ok(markdownReport.includes("chunks=total/unique/embedded")); + assert.equal(markdownReport.match(/^### /gm)?.length, integrityReport.observations.length); + for (const observation of integrityReport.observations) { + assert.ok(markdownReport.includes(`### \`${observation.reference}\` source=${observation.source}`)); + assert.ok(markdownReport.includes(`path: \`${observation.path}\``)); + assert.ok(markdownReport.includes(`doc: \`${observation.id}\``)); + if (observation.revision !== undefined) + assert.ok(markdownReport.includes(`rev=\`${observation.revision}\``)); + } assert.ok( integrityReport.errors.some((error: { reason: string }) => error.reason === "source-unavailable") ); @@ -282,6 +595,27 @@ async function main(): Promise { await withObsidianPage(session.remoteDebuggingPort, async (page) => { await page.locator(".database-diagnostic").getByRole("button", { name: "Close", exact: true }).click(); await page.keyboard.press("Escape"); + const localOnly = await openCommand(page, "integrity"); + await localOnly.getByRole("checkbox", { name: "Include remote database", exact: true }).uncheck(); + await localOnly.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await chooseFormat(localOnly, "json"); + const report = JSON.parse(await localOnly.locator("textarea").inputValue()); + assert.equal(report.status, "complete"); + assert.equal(report.scope.remote, "excluded"); + assert.equal(report.coverage.remote.status, "not-checked"); + assert.deepEqual(report.errors, []); + assert.ok( + report.observations.some( + (row: { identity: string; vault: { status: string }; comparisons: { relation: string }[] }) => + row.identity === "document-id-mismatch" && + row.vault.status === "available" && + row.comparisons.some((comparison) => comparison.relation === "equal") + ), + "Local discrepancies must retain Vault comparisons when the remote is excluded" + ); + await localOnly.getByRole("button", { name: "Close", exact: true }).click(); + await localOnly.waitFor({ state: "hidden", timeout }); }); const cancellationOrder = await evalObsidianJson<{ fixtureId: string; orderedIds: string[] }>( cli, @@ -294,15 +628,10 @@ async function main(): Promise { assert.ok(gatePosition >= 0, "The cancellation read gate must exist in the local database"); assert.ok(fixturePosition < gatePosition, "The cancellation read gate must follow the report fixture"); assert.ok(usageFixtureRow, "The complete report must contain the cancellation fixture"); - const expectedFixtureRow = usageFixtureRow; const cancellationEnv = session.cliEnv; try { - await evalObsidianJson( - cli, - `(async()=>{const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase.localDatabase;const gate={reached:false,release:null,releasePromise:null,originalGet:db.get};gate.releasePromise=new Promise(resolve=>{gate.release=resolve;});globalThis.databaseDiagnosticReadGate=gate;db.get=async function(id,options){if(id===${JSON.stringify(shadowId)}&&options?.rev&&!gate.reached){gate.reached=true;await gate.releasePromise;}return await gate.originalGet.call(this,id,options);};return JSON.stringify(true);})()`, - session.cliEnv - ); + await installDiagnosticReadGate(cli, session, shadowId, "revision"); await withObsidianPage(session.remoteDebuggingPort, async (page) => { const dialogue = await openCommand(page, "usage"); await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); @@ -312,6 +641,10 @@ async function main(): Promise { { timeout } ); assert.equal(await dialogue.getAttribute("data-diagnostic-phase"), "running"); + const usageProgress = dialogue.locator(".diagnostic-progress"); + await usageProgress.getByText("Reading revisions", { exact: true }).waitFor({ timeout }); + await usageProgress.getByText(/^\d+ \/ \d+ revisions$/).waitFor({ timeout }); + await usageProgress.getByText(/Discovering revisions/).waitFor({ timeout }); await dialogue.getByRole("button", { name: "Cancel inspection", exact: true }).click(); await dialogue.getByText("Cancelling inspectionâ€Ļ", { exact: true }).waitFor({ timeout }); await evalObsidianJson( @@ -324,12 +657,18 @@ async function main(): Promise { const cancelledReport = JSON.parse(cancelledText); assert.equal(cancelledReport.status, "cancelled"); await assertFormatChoices(dialogue, ["JSON"]); + const cancelledToken = cancelledReport.rows.find((row: { path: string }) => + parseDiagnosticPathToken(row.path) + )?.path; + const parsedToken = parseDiagnosticPathToken(cancelledToken ?? ""); + assert.ok(parsedToken); + const mapper = createDiagnosticNameMapper("pseudonym", parsedToken.salt); assert.ok( cancelledReport.rows.some( (row: { path: string; id: string; revision: string }) => - row.path === expectedFixtureRow.path && - row.id === expectedFixtureRow.id && - row.revision === expectedFixtureRow.revision + row.path === mapper.path(fixturePath) && + row.id === mapper.id(rawFixture.id) && + row.revision === mapper.revision(rawFixture.revision) ), "Cancellation must retain report rows collected before the gated database read" ); @@ -351,6 +690,7 @@ async function main(): Promise { await setObsidianMobileTestMode(session.remoteDebuggingPort, true, timeout); await waitForLiveSyncCoreReady(cli, session.cliEnv); await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 375, height: 667 }); await openCommand(page, "integrity"); }); const mobileScreenshot = await captureObsidianElement( @@ -359,18 +699,31 @@ async function main(): Promise { (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 375, height: 667 }); const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); await assertMobileDialogueLayout(page, modal, "Database diagnostic choice"); await assertMobileRadioTargets(page, modal, "Names in this report"); + const remoteChoice = modal.getByRole("checkbox", { name: "Include remote database", exact: true }); + assert.equal(await remoteChoice.isChecked(), true); + await assertLocatorHasMinimumTouchTarget(page, remoteChoice.locator(".."), { + label: "Include remote database", + }); + await assertLocatorWithinSafeArea(page, remoteChoice.locator(".."), { + label: "Include remote database", + safeAreaInsets: iPhoneSafeArea, + }); + await assertNoHorizontalOverflow(page, remoteChoice.locator(".."), { label: "Include remote database" }); await modal.getByRole("button", { name: "Cancel", exact: true }).click(); await modal.waitFor({ state: "hidden", timeout }); }); await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 375, height: 667 }); const dialogue = await openCommand(page, "usage"); await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); await assertMobileDialogueLayout(page, modal, "Database diagnostic result"); + await assertDiagnosticExportActionsWithinModal(page, dialogue, "the mobile result appearing"); await assertFormatChoices(dialogue, ["TSV", "JSON"]); await assertMobileRadioTargets(page, dialogue, "Report format"); @@ -394,6 +747,7 @@ async function main(): Promise { (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 375, height: 667 }); const dialogue = page.locator(".database-diagnostic"); await installDiagnosticExportInterception(page); const reportText = await dialogue.locator("textarea").inputValue(); @@ -401,6 +755,7 @@ async function main(): Promise { await dialogue.getByText("Report copied.", { exact: true }).waitFor({ timeout }); const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); await assertMobileDialogueLayout(page, modal, "Database diagnostic copied result"); + await assertDiagnosticExportActionsWithinModal(page, dialogue, "copying the mobile report"); assert.equal( await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), reportText @@ -414,9 +769,35 @@ async function main(): Promise { content: reportText, }); await assertMobileDialogueLayout(page, modal, "Database diagnostic saved result"); + await assertDiagnosticExportActionsWithinModal(page, dialogue, "saving the mobile report"); + await captureObsidianElement( + activeSession.remoteDebuggingPort, + "database-diagnostics-mobile-saved.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + ); await dialogue.getByRole("button", { name: "Close", exact: true }).click(); await dialogue.waitFor({ state: "hidden", timeout }); }); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.setViewportSize({ width: 375, height: 667 }); + const settings = await openLiveSyncSettings(page, timeout); + const hatch = await settings.openPage("Hatch"); + assert.ok(usageFixtureRow); + await assertDiagnosticFileLookup( + page, + hatch, + usageFixtureRow.path, + rawFixture.shadowRevision, + activeSession, + true + ); + await settings.close(); + }); + assert.deepEqual( + await readDatabaseState(cli, session), + before, + "File lookup and follow-up reports remain read-only" + ); console.log( JSON.stringify({ status: "passed", diff --git a/updates.md b/updates.md index dcdc8352..394b9323 100644 --- a/updates.md +++ b/updates.md @@ -21,8 +21,12 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Diagnostics -- We can inspect normal-file Metadata and current conflicts in the local database and CouchDB together, with byte comparisons for suspect groups. Partial results retain the available evidence when a source is unavailable. The inspection leaves databases and settings unchanged. -- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Both name modes and the available report formats stay visible as radio buttons. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons. +- We can inspect normal-file Metadata and current conflicts in the local database and CouchDB together, with byte comparisons for suspect groups. A per-run option excludes the remote database when a local database and Vault inspection is enough; reports identify that scope. Partial results retain the available evidence when a source is unavailable. The inspection leaves databases and settings unchanged. +- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Both name modes and the available report formats stay visible as radio buttons. Pseudonym reports can be followed up with the same public salt; the local Hatch lookup searches the current Vault and local normal-file Metadata and can prepare a report for a selected document. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons. +- Revision pseudonyms retain their generation number. Integrity reports include existing size, modification-time, Chunk-reference, and case/NFC relationship evidence without additional database or Vault reads. +- We can review an integrity report more quickly in Markdown: a Review index groups references by identity, conflict, path, availability, and measured-difference findings before the observations. The compact display keeps each observation and its full follow-up references; the JSON report is unchanged. +- Usage progress shows the growing number of available revisions during reading, then a separate cancellable accounting stage. Integrity inspection reduces remote requests with bulk reads and bounded concurrency, shows progress counts and a gauge for each stage, and records document-read and HTTP-request timing and counts in the log. Markdown reports include a collapsible code block for sharing in issues. +- Hatch groups the five reporting tools separately from checks, recovery, and reset operations. Diagnostic dialogues resize their report preview to keep the controls visible in smaller windows. ## 1.0.38 From 070fef76698baca76b478fd0f023df245592f725 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 20 Oct 2026 08:24:05 +0000 Subject: [PATCH 22/68] Defer Pages validation until draft pull requests are ready --- .github/workflows/deploy-pages.yml | 2 ++ utils/ci-pages.unit.spec.ts | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 7683c7da..ab0538c0 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -14,6 +14,7 @@ on: - 'package-lock.json' - '.github/workflows/deploy-pages.yml' pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths: - 'aggregator.html' - 'src/**' @@ -32,6 +33,7 @@ concurrency: jobs: build: + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} name: Validate and package Pages site runs-on: ubuntu-latest steps: diff --git a/utils/ci-pages.unit.spec.ts b/utils/ci-pages.unit.spec.ts index 5130c2bb..fbece795 100644 --- a/utils/ci-pages.unit.spec.ts +++ b/utils/ci-pages.unit.spec.ts @@ -9,9 +9,9 @@ type Concurrency = { "cancel-in-progress"?: boolean | string; }; type Workflow = { - on?: Partial>; + on?: Partial>; concurrency?: Concurrency; - jobs?: Record; + jobs?: Record; }; function readWorkflow(filename: string): Workflow { @@ -26,6 +26,18 @@ function matchesAnyPath(patterns: string[], filename: string): boolean { const pagesWorkflow = readWorkflow("deploy-pages.yml"); const unitWorkflow = readWorkflow("unit-ci.yml"); +describe("Pages pull request lifecycle", () => { + it("defers draft validation and starts checks when the pull request is ready", () => { + expect(pagesWorkflow.jobs?.build?.if).toBe( + "${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}" + ); + expect(pagesWorkflow.on?.pull_request?.types).toEqual( + expect.arrayContaining(["opened", "synchronize", "reopened", "ready_for_review"]) + ); + expect(pagesWorkflow.jobs?.deploy?.if).toBe("github.event_name != 'pull_request'"); + }); +}); + describe("CI workflow path filters", () => { it.each(["push", "pull_request"] as const)("runs Pages checks for the manifest on %s", (trigger) => { const patterns = pagesWorkflow.on?.[trigger]?.paths ?? []; From 579a2ff373b92f92fbed5a95ee01591175881c17 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 20 Oct 2026 09:47:29 +0000 Subject: [PATCH 23/68] Use published Commonlib 0.1.44 --- package-lock.json | 476 +--------------------------------------------- package.json | 2 +- 2 files changed, 7 insertions(+), 471 deletions(-) diff --git a/package-lock.json b/package-lock.json index fcfe4617..36f32d6f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.43", + "@vrtmrz/livesync-commonlib": "0.1.44", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -945,214 +945,6 @@ "node": ">=18.0.0" } }, - "node_modules/@babel/code-frame": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", - "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.29.7", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/code-frame/node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, - "node_modules/@babel/compat-data": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", - "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", - "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/generator": "^7.29.7", - "@babel/helper-compilation-targets": "^7.29.7", - "@babel/helper-module-transforms": "^7.29.7", - "@babel/helpers": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/template": "^7.29.7", - "@babel/traverse": "^7.29.7", - "@babel/types": "^7.29.7", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/core/node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@babel/core/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/generator": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", - "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/parser": "^7.29.7", - "@babel/types": "^7.29.7", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", - "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/compat-data": "^7.29.7", - "@babel/helper-validator-option": "^7.29.7", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/@babel/helper-compilation-targets/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/helper-globals": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", - "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", - "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/traverse": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", - "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/helper-module-imports": "^7.29.7", - "@babel/helper-validator-identifier": "^7.29.7", - "@babel/traverse": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, "node_modules/@babel/helper-string-parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", @@ -1173,34 +965,6 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/helper-validator-option": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", - "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", - "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/template": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", @@ -1217,44 +981,6 @@ "node": ">=6.0.0" } }, - "node_modules/@babel/template": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", - "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", - "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/generator": "^7.29.7", - "@babel/helper-globals": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/template": "^7.29.7", - "@babel/types": "^7.29.7", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/types": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", @@ -4142,7 +3868,7 @@ "version": "8.69.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.69.0.tgz", "integrity": "sha512-K3VrubUPhlo9VDBS6QdI8YB5j7ClpqLRdefcz6PFrhnwicehBweqQ9Evhl4l+FYz0HdDmMqIiSX0aldGRYtDCA==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4389,11 +4115,12 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.43", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.43.tgz", - "integrity": "sha512-WM2k545hT0ctLv64WdKZ0PkHwyw9Fnlf9xssH7Uv9C0HEMm8CDtj6dfJsPOHs0+Z41Kfco1U/y6R5f5+xWZ00g==", + "version": "0.1.44", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.44.tgz", + "integrity": "sha512-GjXNSLTnWLZzdgrhW3sIOsLqGLP4iMgypPMtcBc2Ju2vMq64TRA1Llfoya1KRIeu6m0G7Xu3ZtkWARTRcHkYYg==", "license": "MIT", "dependencies": { + "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/client-s3": "^3.808.0", "@smithy/fetch-http-handler": "^5.3.10", "@smithy/md5-js": "^4.2.9", @@ -4833,21 +4560,6 @@ ], "license": "MIT" }, - "node_modules/baseline-browser-mapping": { - "version": "2.11.20", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.20.tgz", - "integrity": "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw==", - "dev": true, - "license": "Apache-2.0", - "optional": true, - "peer": true, - "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", @@ -4866,42 +4578,6 @@ "node": "20 || >=22" } }, - "node_modules/browserslist": { - "version": "4.28.8", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz", - "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "baseline-browser-mapping": "^2.11.12", - "caniuse-lite": "^1.0.30001809", - "electron-to-chromium": "^1.5.402", - "node-releases": "^2.0.53", - "update-browserslist-db": "^1.3.0" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -4993,29 +4669,6 @@ "node": ">=6" } }, - "node_modules/caniuse-lite": { - "version": "1.0.30001810", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", - "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0", - "optional": true, - "peer": true - }, "node_modules/catering": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.1.tgz", @@ -5432,15 +5085,6 @@ "node": ">= 0.4" } }, - "node_modules/electron-to-chromium": { - "version": "1.5.419", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.419.tgz", - "integrity": "sha512-nHMPn8x4yCxCI0iSnL+LlHL5sUoUfjLXkcRIagZ4GBdrfFLFaiLNvzJWbJqZhFT9IAhw5tUSNlhggWN+otvp/A==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true - }, "node_modules/empathic": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/empathic/-/empathic-2.0.1.tgz", @@ -5777,18 +5421,6 @@ "node": ">=18" } }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=6" - } - }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -6970,18 +6602,6 @@ "node": ">= 0.4" } }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -7853,21 +7473,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -8895,18 +8500,6 @@ "node-gyp-build-test": "build-test.js" } }, - "node_modules/node-releases": { - "version": "2.0.54", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", - "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=18" - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -10644,21 +10237,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", - "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-1.8.0.tgz", @@ -11219,39 +10797,6 @@ "dev": true, "license": "MIT" }, - "node_modules/update-browserslist-db": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", - "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -11798,15 +11343,6 @@ "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", "license": "MIT" }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true - }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", diff --git a/package.json b/package.json index 983ca49f..b98c52f8 100644 --- a/package.json +++ b/package.json @@ -193,7 +193,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.43", + "@vrtmrz/livesync-commonlib": "0.1.44", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", From 7b5dc1243ca2994e1f95277a15cab18d557c4008 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 20 Oct 2026 09:47:29 +0000 Subject: [PATCH 24/68] Verify diagnostic follow-up cancellation in Obsidian --- test/e2e-obsidian/README.md | 2 +- .../scripts/database-diagnostics.ts | 85 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 05aadf06..b4e854a1 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -89,7 +89,7 @@ The wrapper accepts only maintained real-Obsidian scenario names; run it with `- `database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, the default-on integrity-only remote option, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout at 800×600 and 375×667, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. -The same scenario checks the five actions in Reporting Tools, the separate Checks and Recovery panel, Metadata progress with its document total and gauge, and collapsible Markdown. It also observes Usage revision progress with a count and discovery note during a gated read. Local lookup checks cover multiple document IDs for one path, selection of an exact ID, and a follow-up report using the original salt. The compact Markdown must retain every JSON observation reference and pseudonym, and its copy and save payloads must match the full output. Results and export controls are checked at 800×600 on desktop and 375×667 in mobile emulation, including after copy and save feedback appears. +The same scenario checks the five actions in Reporting Tools, the separate Checks and Recovery panel, Metadata progress with its document total and gauge, and collapsible Markdown. It also observes Usage revision progress with a count and discovery note during a gated read. Local lookup checks cover multiple document IDs for one path, selection of an exact ID, and a follow-up report using the original salt. Closing the follow-up dialogue during a pending Metadata read must abort the report after that read settles, without querying Chunk availability. The compact Markdown must retain every JSON observation reference and pseudonym, and its copy and save payloads must match the full output. Results and export controls are checked at 800×600 on desktop and 375×667 in mobile emulation, including after copy and save feedback appears. `database-diagnostics-couchdb` needs the CouchDB fixture with CORS enabled. It checks that clearing the remote option completes a local database and Vault inspection with remote coverage marked not checked and makes no HTTP requests, then confirms that each new run includes the remote database by default. It also checks unencrypted and encrypted databases, both browser fetch and Obsidian's native Request API, and both name policies through the real integrity dialogue. A remote-only document with a mismatched ID must produce a finding with readable Chunks and equal content. JSON and Markdown must omit file contents and credentials. Included inspections observe real requests and require GET-only access, unchanged local settings, revisions, and Vault contents, and unchanged remote documents and Security Seed. Commonlib's separate CouchDB integration tests additionally cover a missing Security Seed without creating one. diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts index 49080c5c..773c60c2 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -331,6 +331,89 @@ async function assertDiagnosticFileLookup( await dialogue.waitFor({ state: "hidden", timeout }); } +async function assertFollowUpStopsAfterClose(page: Page, hatch: Locator, token: string): Promise { + await page.evaluate(`(() => { + const core = app.plugins.plugins['obsidian-livesync'].core; + const db = core.localDatabase.localDatabase; + const manager = core.services.UI.dialogManager; + const gate = { + reached: false, metadataReads: 0, chunkQueries: 0, outcome: null, + originalGet: db.get, originalAllDocsRaw: core.localDatabase.allDocsRaw, + originalOpen: manager.open, + }; + gate.releasePromise = new Promise(resolve => { gate.release = resolve; }); + gate.settledPromise = new Promise(resolve => { gate.settle = resolve; }); + db.get = async function(id, options) { + if (id === ${JSON.stringify(shadowId)} && options?.conflicts) { + gate.metadataReads++; + if (!gate.reached) { + gate.reached = true; + await gate.releasePromise; + } + } + return await gate.originalGet.call(this, id, options); + }; + core.localDatabase.allDocsRaw = async function(...args) { + if (gate.reached) gate.chunkQueries++; + return await gate.originalAllDocsRaw.apply(this, args); + }; + manager.open = function(component, options, ...args) { + if (typeof options?.prepareReport === 'function') { + const prepare = options.prepareReport; + options.prepareReport = async (...reportArgs) => { + try { + const result = await prepare(...reportArgs); + gate.outcome = 'completed'; + return result; + } catch (error) { + gate.outcome = error.name; + throw error; + } finally { + gate.settle(); + } + }; + } + return gate.originalOpen.call(this, component, options, ...args); + }; + globalThis.diagnosticFollowUpCloseGate = gate; + })()`); + try { + const setting = hatch.locator(".setting-item").filter({ + has: page.getByText("Find a file from a report pseudonym", { exact: true }), + }); + await setting.getByRole("button", { name: "Find file", exact: true }).click(); + const dialogue = page.locator(".diagnostic-file-lookup"); + await dialogue.getByLabel("Report path pseudonym", { exact: true }).fill(token); + await dialogue.getByRole("button", { name: "Find file", exact: true }).click(); + const candidate = dialogue.locator("[data-diagnostic-candidate]").filter({ hasText: shadowId }); + await candidate.getByRole("button", { name: "Prepare follow-up report", exact: true }).click(); + await page.waitForFunction("globalThis.diagnosticFollowUpCloseGate.reached", undefined, { timeout }); + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); + const result = await page.evaluate(`(async () => { + const gate = globalThis.diagnosticFollowUpCloseGate; + gate.release(); + await gate.settledPromise; + return { outcome: gate.outcome, metadataReads: gate.metadataReads, chunkQueries: gate.chunkQueries }; + })()`); + assert.deepEqual( + result, + { outcome: "AbortError", metadataReads: 1, chunkQueries: 0 }, + "Closing a pending follow-up must stop inspection after the active Metadata read settles" + ); + } finally { + await page.evaluate(`(() => { + const gate = globalThis.diagnosticFollowUpCloseGate; + const core = app.plugins.plugins['obsidian-livesync'].core; + gate.release(); + core.localDatabase.localDatabase.get = gate.originalGet; + core.localDatabase.allDocsRaw = gate.originalAllDocsRaw; + core.services.UI.dialogManager.open = gate.originalOpen; + delete globalThis.diagnosticFollowUpCloseGate; + })()`); + } +} + async function main(): Promise { const binary = requireObsidianBinary(); const cli = requireObsidianCli(); @@ -475,6 +558,7 @@ async function main(): Promise { activeSession ); hatch = await (await openLiveSyncSettings(page, timeout)).openPage("Hatch"); + await assertFollowUpStopsAfterClose(page, hatch, usageFixtureRow.path); await captureObsidianElement(activeSession.remoteDebuggingPort, "database-diagnostics-hatch.png", (page) => page.locator(".modal.mod-settings") ); @@ -805,6 +889,7 @@ async function main(): Promise { mobileScreenshot, mobileResultScreenshot, readOnly: true, + followUpCloseCancellation: true, }) ); } catch (error) { From d139e01c4885c434d1ebbaaa505e2be52bdccef3 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 20 Oct 2026 10:08:45 +0000 Subject: [PATCH 25/68] fix: package.json to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-MARKDOWNIT-21277720 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b98c52f8..2e95db5a 100644 --- a/package.json +++ b/package.json @@ -198,7 +198,7 @@ "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", "idb": "^8.0.3", - "markdown-it": "^14.3.2", + "markdown-it": "^14.3.3", "minimatch": "^10.2.5", "obsidian": "^1.13.1", "octagonal-wheels": "^0.1.55", From 6842b9c0136ce11162648ebde2adc1796396e006 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:25:53 +0000 Subject: [PATCH 26/68] Releasing 1.0.39 --- manifest.json | 2 +- package-lock.json | 10 +++++----- package.json | 2 +- src/apps/cli/package.json | 2 +- src/apps/webapp/package.json | 2 +- src/apps/webpeer/package.json | 2 +- updates.md | 4 ++++ versions.json | 3 ++- 8 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest.json b/manifest.json index f1633e1a..f12a57e2 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.38", + "version": "1.0.39", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index 36f32d6f..1d8d4dfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.38", + "version": "1.0.39", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.38", + "version": "1.0.39", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -11398,7 +11398,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.38-cli", + "version": "1.0.39-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11423,7 +11423,7 @@ }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.38-webapp", + "version": "1.0.39-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11435,7 +11435,7 @@ } }, "src/apps/webpeer": { - "version": "1.0.38-webpeer", + "version": "1.0.39-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, diff --git a/package.json b/package.json index b98c52f8..a6ff13d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.38", + "version": "1.0.39", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 67daece6..538c0d99 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.38-cli", + "version": "1.0.39-cli", "main": "dist/index.cjs", "type": "module", "scripts": { diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index e08ebf91..43800d07 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.38-webapp", + "version": "1.0.39-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index de5b3f32..dbe5c64e 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.38-webpeer", + "version": "1.0.39-webpeer", "type": "module", "scripts": { "dev": "vite", diff --git a/updates.md b/updates.md index 394b9323..6a2e856d 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,10 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +## 1.0.39 + +20th October, 2026 + ### Synchronisation #### Fixed diff --git a/versions.json b/versions.json index 34ccf31b..1ae45d74 100644 --- a/versions.json +++ b/versions.json @@ -49,5 +49,6 @@ "1.0.35": "1.7.2", "1.0.36": "1.7.2", "1.0.37": "1.7.2", - "1.0.38": "1.7.2" + "1.0.38": "1.7.2", + "1.0.39": "1.7.2" } From 11edc6edef6291dd90ad10e33cf726ee5f7f82e0 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 20 Oct 2026 10:27:22 +0000 Subject: [PATCH 27/68] Polish 1.0.39 release notes --- docs/releases/1.0.md | 15 +++++++++++++++ updates.md | 38 +++++++++++++++++--------------------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/docs/releases/1.0.md b/docs/releases/1.0.md index cc8731f8..96ce8a6b 100644 --- a/docs/releases/1.0.md +++ b/docs/releases/1.0.md @@ -2,6 +2,21 @@ This document contains earlier published releases from the 1.0 line of the [current Self-hosted LiveSync release history](../../updates.md). Beta and release-candidate builds published before 1.0.0 are recorded in the [1.0 preview history](1.0-previews.md). Earlier release lines continue in the [0.25 history](0.25.md) and the [legacy history](legacy.md). +## 1.0.34 + +2nd October, 2026 + +### Security + +- Fixed an issue where sensitive information could be included in exported configuration data under certain conditions. Updating is recommended. (refs #1218) +- Improved configuration persistence and JSON conflict handling. + +### Acknowledgements + +Thank you for your contributions! + +- @kimjansheden (#1218) + ## 1.0.33 1st October, 2026 diff --git a/updates.md b/updates.md index 6a2e856d..ca2e3497 100644 --- a/updates.md +++ b/updates.md @@ -16,6 +16,8 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi 20th October, 2026 +I have been finding troubleshooting difficult, so I decided to go back to basics and prepare the tools we need. + ### Synchronisation #### Fixed @@ -25,12 +27,21 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Diagnostics -- We can inspect normal-file Metadata and current conflicts in the local database and CouchDB together, with byte comparisons for suspect groups. A per-run option excludes the remote database when a local database and Vault inspection is enough; reports identify that scope. Partial results retain the available evidence when a source is unavailable. The inspection leaves databases and settings unchanged. -- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Both name modes and the available report formats stay visible as radio buttons. Pseudonym reports can be followed up with the same public salt; the local Hatch lookup searches the current Vault and local normal-file Metadata and can prepare a report for a selected document. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons. -- Revision pseudonyms retain their generation number. Integrity reports include existing size, modification-time, Chunk-reference, and case/NFC relationship evidence without additional database or Vault reads. -- We can review an integrity report more quickly in Markdown: a Review index groups references by identity, conflict, path, availability, and measured-difference findings before the observations. The compact display keeps each observation and its full follow-up references; the JSON report is unchanged. -- Usage progress shows the growing number of available revisions during reading, then a separate cancellable accounting stage. Integrity inspection reduces remote requests with bulk reads and bounded concurrency, shows progress counts and a gauge for each stage, and records document-read and HTTP-request timing and counts in the log. Markdown reports include a collapsible code block for sharing in issues. -- Hatch groups the five reporting tools separately from checks, recovery, and reset operations. Diagnostic dialogues resize their report preview to keep the controls visible in smaller windows. +- We can inspect file differences and current conflicts across the local database, CouchDB, and the Vault without changing databases or settings. + - Integrity inspection compares Metadata for normal files and checks content for suspect groups. **Include remote database** is enabled by default and can be cleared for one run. Reports distinguish an excluded remote from an unavailable source and retain the evidence collected so far. +- We can share reports using pseudonyms and find the affected files locally for follow-up. + - Integrity and Usage offer original names or pseudonyms before each run, and omit file contents and credentials in both modes. Path pseudonyms use a random public salt for each report; revision pseudonyms retain their generation number. + - The Hatch lookup searches current Vault paths and local Metadata for normal files. It preserves separate document IDs for the same path and can prepare a report for the selected document using the original salt. Public salts permit guessed-path checks. +- We can follow progress in large inspections and stop work when enough evidence has been collected. + - Integrity shows counts and a gauge for each stage, reduces remote requests with bulk reads and bounded concurrency, and logs read and request timings. Usage shows revision discovery with a growing total, followed by cancellable accounting. + - Closing a follow-up dialogue stops further reads after the active read settles. Complete Usage reports retain the existing TSV columns; partial reports use JSON to retain their status and reasons. +- We can read and share large Integrity reports more easily. + - Compact Markdown retains every observation and its follow-up references, with state counts, a review index, and a field guide. It includes available size, modification-time, Chunk-reference, and case/NFC evidence without additional database or Vault reads. JSON retains the full structure. + - Markdown reports use a collapsible code block for sharing in issues. Review candidates indicate where to investigate; they do not establish corruption, and unchecked content remains distinct from healthy content. +- We can find reporting actions more easily in Hatch and keep report controls visible in smaller windows. + - The five reporting tools are grouped separately from checks, recovery, and reset operations. Name and format choices remain visible, and the report preview resizes around the controls. +- We can request and provide targeted troubleshooting evidence through the issue-reporting guide. + - Direct links cover Integrity, Usage, pseudonym lookup, and follow-up reports, with request examples for issue discussions. ## 1.0.38 @@ -139,18 +150,3 @@ Thank you for your contributions! Thank you for your contributions! - @kimjansheden (#1220) - -## 1.0.34 - -2nd October, 2026 - -### Security - -- Fixed an issue where sensitive information could be included in exported configuration data under certain conditions. Updating is recommended. (refs #1218) -- Improved configuration persistence and JSON conflict handling. - -### Acknowledgements - -Thank you for your contributions! - -- @kimjansheden (#1218) From b35ac635d09a44cb00c0ee5e442be1fd2e89aabe Mon Sep 17 00:00:00 2001 From: Junker der Provinz <133605895+junkerderprovinz@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:25:44 +0200 Subject: [PATCH 28/68] Read Setup URI generator values from LIVESYNC_ variables hostname, username, and the other unprefixed names collide with shell and system variables (fish makes hostname read-only, Windows already defines USERNAME). The generator now reads LIVESYNC_HOST, LIVESYNC_USER, LIVESYNC_PASS, LIVESYNC_DB, and LIVESYNC_PASSPHRASE first and falls back to the old names, so existing instructions keep working. Co-authored-by: rwbru <59871009+rwbru@users.noreply.github.com> --- docs/setup_object_storage.md | 2 +- docs/setup_own_server.md | 17 ++++--- docs/setup_p2p.md | 2 +- updates.md | 7 +++ utils/readme.md | 18 ++++--- utils/setup/generate_setup_uri.test.ts | 70 ++++++++++++++++++++++++++ utils/setup/generate_setup_uri.ts | 22 +++++--- 7 files changed, 114 insertions(+), 24 deletions(-) diff --git a/docs/setup_object_storage.md b/docs/setup_object_storage.md index cad31264..c2a5237b 100644 --- a/docs/setup_object_storage.md +++ b/docs/setup_object_storage.md @@ -24,7 +24,7 @@ export secret_key= export bucket=vault-data export region=auto export bucket_prefix=my-vault -export passphrase= +export LIVESYNC_PASSPHRASE= export uri_passphrase= deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/docs/setup_own_server.md b/docs/setup_own_server.md index 1b7d0e9c..6924ec55 100644 --- a/docs/setup_own_server.md +++ b/docs/setup_own_server.md @@ -128,6 +128,9 @@ export database=obsidiannotes curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | bash ``` +> [!TIP] +> In fish, `hostname` is read-only and cannot be exported. Pass the values through `env` instead: `curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | env hostname=http://localhost:5984 username=<...> password=<...> database=obsidiannotes bash`. + If it results like the following: ``` CouchDB provisioning completed. @@ -173,20 +176,20 @@ Now `https://tiles-photograph-routine-groundwater.trycloudflare.com` is our serv ### 1. Generate the setup URI on a desktop device or server -The `username` and `password` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. +The `LIVESYNC_USER` and `LIVESYNC_PASS` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. ```bash -export hostname=https://tiles-photograph-routine-groundwater.trycloudflare.com -export database=obsidiannotes -export username= -export password= -export passphrase= +export LIVESYNC_HOST=https://tiles-photograph-routine-groundwater.trycloudflare.com +export LIVESYNC_DB=obsidiannotes +export LIVESYNC_USER= +export LIVESYNC_PASS= +export LIVESYNC_PASSPHRASE= export uri_passphrase= # Optional deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` > [!TIP] -> `passphrase` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel. +> `LIVESYNC_PASSPHRASE` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel. > > If `uri_passphrase` is omitted, the generator creates a cryptographically random value and prints it once. diff --git a/docs/setup_p2p.md b/docs/setup_p2p.md index 4797f420..c755ce0a 100644 --- a/docs/setup_p2p.md +++ b/docs/setup_p2p.md @@ -130,7 +130,7 @@ export remote_type=p2p export p2p_relays=wss://relay.example.com export p2p_room_id= # Optional; generated when omitted export p2p_passphrase= # Optional; generated when omitted -export passphrase= +export LIVESYNC_PASSPHRASE= export uri_passphrase= deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/updates.md b/updates.md index ca2e3497..d2575c27 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Setup tools + +#### Fixed + +- We can generate a Setup URI from fish, PowerShell, or cmd without the connection details colliding with `hostname` or `USERNAME`. (#817) + - The generator reads `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE`. The previous names are still read when these are not set. + ## 1.0.39 20th October, 2026 diff --git a/utils/readme.md b/utils/readme.md index 6ee10742..22439215 100644 --- a/utils/readme.md +++ b/utils/readme.md @@ -31,21 +31,23 @@ Authentication and other non-retryable HTTP failures stop immediately. Network a The existing `flyio/generate_setupuri.ts` path remains a CouchDB-only compatibility wrapper for the Fly.io deployment script. +The connection and Vault encryption values use a `LIVESYNC_` prefix so that they do not collide with shell or system variables such as `hostname` and `USERNAME`. The unprefixed names `hostname`, `username`, `password`, `database`, and `passphrase` are still read when the prefixed ones are not set. + The generator creates a fresh random ID key by default and includes it in the encrypted Setup URI. It prints a tagged `sls-id-v1:` recovery code. Set `id_recovery_code` to that code when generating another URI for the same Vault; a new run without it creates a different key. This restores only the ID key: reuse the original connection details too. For P2P, provide the original `p2p_room_id` and `p2p_passphrase` because omitted values are generated afresh. Set `id_mode=legacy` to generate a URI with the previous ID behaviour. `id_mode=legacy` and `id_recovery_code` cannot be combined. Keep the recovery code private and retain it if every device might be lost. ### CouchDB ```sh -export hostname=https://couch.example.com -export username=couchdb-admin-username -export password=couchdb-admin-password -export database=obsidiannotes -export passphrase=a-strong-vault-encryption-passphrase +export LIVESYNC_HOST=https://couch.example.com +export LIVESYNC_USER=couchdb-admin-username +export LIVESYNC_PASS=couchdb-admin-password +export LIVESYNC_DB=obsidiannotes +export LIVESYNC_PASSPHRASE=a-strong-vault-encryption-passphrase export uri_passphrase=a-separate-setup-uri-passphrase # Optional deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` -If `uri_passphrase` is omitted, the tool generates and prints a cryptographically random one. Store the Setup URI and its passphrase separately. The `passphrase` value protects synchronised Vault data and must also be stored safely. +If `uri_passphrase` is omitted, the tool generates and prints a cryptographically random one. Store the Setup URI and its passphrase separately. The `LIVESYNC_PASSPHRASE` value protects synchronised Vault data and must also be stored safely. The generator defaults to `uri_mode=ephemeral`. The URI opens only during the current fixed seven-day UTC window, and the tool prints its exact end time. It may have less than seven days remaining when generated. For an indefinitely reusable URI, set `uri_mode=persistent` before running the command. Persistent uses the existing encrypted format, which older clients that already support that format can read. The time condition controls opening the URI; it does not revoke settings or credentials after import. @@ -59,7 +61,7 @@ export secret_key= export bucket=vault-data export region=auto export bucket_prefix=team-a # Optional -export passphrase= +export LIVESYNC_PASSPHRASE= deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` @@ -69,7 +71,7 @@ Optional Object Storage variables are `use_custom_request_handler`, `force_path_ ```sh export remote_type=p2p -export passphrase= +export LIVESYNC_PASSPHRASE= deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` diff --git a/utils/setup/generate_setup_uri.test.ts b/utils/setup/generate_setup_uri.test.ts index 2bf802ff..66ced671 100644 --- a/utils/setup/generate_setup_uri.test.ts +++ b/utils/setup/generate_setup_uri.test.ts @@ -8,6 +8,76 @@ function assert(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message); } +Deno.test("reads LIVESYNC_ variables before the unprefixed names", async () => { + const generated = await generateSetupURI({ + LIVESYNC_HOST: "https://couch.example.test", + LIVESYNC_USER: "alice", + LIVESYNC_PASS: "couch-secret", + LIVESYNC_DB: "notes", + LIVESYNC_PASSPHRASE: "vault-secret", + hostname: "https://shell.example.test", + username: "shell-user", + password: "shell-secret", + database: "shell-db", + passphrase: "shell-passphrase", + uri_passphrase: "setup-secret", + }); + const decoded = await decodeSettingsFromSetupURI( + generated.setupURI, + generated.setupPassphrase, + ); + assert(decoded, "Commonlib could not decode the CouchDB Setup URI"); + const effective = { ...DEFAULT_SETTINGS, ...decoded }; + assert( + effective.couchDB_URI === "https://couch.example.test" && + effective.couchDB_USER === "alice" && + effective.couchDB_PASSWORD === "couch-secret" && + effective.couchDB_DBNAME === "notes", + "the CouchDB connection did not come from the LIVESYNC_ variables", + ); + assert( + effective.passphrase === "vault-secret", + "the Vault passphrase did not come from LIVESYNC_PASSPHRASE", + ); +}); + +Deno.test("falls back to the unprefixed names when LIVESYNC_ variables are blank", async () => { + const generated = await generateSetupURI({ + LIVESYNC_HOST: " ", + hostname: "https://couch.example.test", + username: "alice", + password: "couch-secret", + database: "notes", + passphrase: "vault-secret", + uri_passphrase: "setup-secret", + }); + const decoded = await decodeSettingsFromSetupURI( + generated.setupURI, + generated.setupPassphrase, + ); + assert(decoded, "Commonlib could not decode the CouchDB Setup URI"); + const effective = { ...DEFAULT_SETTINGS, ...decoded }; + assert( + effective.couchDB_URI === "https://couch.example.test" && + effective.couchDB_USER === "alice" && + effective.couchDB_PASSWORD === "couch-secret" && + effective.couchDB_DBNAME === "notes" && + effective.passphrase === "vault-secret", + "the generator did not fall back to the unprefixed names", + ); +}); + +Deno.test("names both variables when a CouchDB value is missing", async () => { + let rejected = false; + try { + await generateSetupURI({ uri_passphrase: "setup-secret" }); + } catch (error) { + rejected = error instanceof Error && + error.message === "LIVESYNC_HOST (or hostname) is required"; + } + assert(rejected, "the generator did not name both host variables"); +}); + Deno.test("generates an Object Storage Setup URI with a selected S3 profile", async () => { const generated = await generateSetupURI({ remote_type: "s3", diff --git a/utils/setup/generate_setup_uri.ts b/utils/setup/generate_setup_uri.ts index 6bf1ebdd..14c67724 100644 --- a/utils/setup/generate_setup_uri.ts +++ b/utils/setup/generate_setup_uri.ts @@ -64,9 +64,17 @@ function configureIdDerivation( function requireValue( environment: SetupGeneratorEnvironment, name: string, + legacyName?: string, ): string { - const value = environment[name]?.trim(); - if (!value) throw new Error(`${name} is required`); + const value = environment[name]?.trim() || + (legacyName && environment[legacyName]?.trim()); + if (!value) { + throw new Error( + legacyName + ? `${name} (or ${legacyName}) is required` + : `${name} is required`, + ); + } return value; } @@ -97,7 +105,7 @@ function applyEncryptedVaultSettings( Object.assign(settings, { isConfigured: true, encrypt: true, - passphrase: requireValue(environment, "passphrase"), + passphrase: requireValue(environment, "LIVESYNC_PASSPHRASE", "passphrase"), usePathObfuscation: true, }); } @@ -107,10 +115,10 @@ function createCouchDBSettings( ): ObsidianLiveSyncSettings { const settings = createNewVaultSettings(); Object.assign(settings, PREFERRED_SETTING_SELF_HOSTED, { - couchDB_URI: requireValue(environment, "hostname"), - couchDB_USER: requireValue(environment, "username"), - couchDB_PASSWORD: requireValue(environment, "password"), - couchDB_DBNAME: requireValue(environment, "database"), + couchDB_URI: requireValue(environment, "LIVESYNC_HOST", "hostname"), + couchDB_USER: requireValue(environment, "LIVESYNC_USER", "username"), + couchDB_PASSWORD: requireValue(environment, "LIVESYNC_PASS", "password"), + couchDB_DBNAME: requireValue(environment, "LIVESYNC_DB", "database"), batchSave: true, periodicReplication: true, syncOnStart: true, From e83833d4e8663d16ab95d626376c8a7ee45ee188 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 01:00:22 +0000 Subject: [PATCH 29/68] Fix Automatic mode prerequisites and plug-in initialisation targets --- docs/settings.md | 2 + docs/tips/hidden-file-sync.md | 6 + .../messages/LiveSyncProvisionalMessages.ts | 3 + src/features/ConfigSync/PluginPane.svelte | 17 +- .../ConfigSync/getAutomaticSyncTargetFiles.ts | 13 + .../getAutomaticSyncTargetFiles.unit.spec.ts | 58 ++++ .../HiddenFileSync/CmdHiddenFileSync.ts | 14 +- .../CmdHiddenFileSync.unit.spec.ts | 190 ++++++++++- test/e2e-obsidian/README.md | 4 + .../scripts/customisation-sync.ts | 316 +++++++++++++++++- .../scripts/hidden-file-snippet-sync.ts | 124 +++++++ updates.md | 7 + 12 files changed, 735 insertions(+), 19 deletions(-) create mode 100644 src/features/ConfigSync/getAutomaticSyncTargetFiles.ts create mode 100644 src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts diff --git a/docs/settings.md b/docs/settings.md index 0885589c..bdad0692 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -783,6 +783,8 @@ When a saved setting changes whether a normal file can be reflected, LiveSync re Customisation Sync is a supported, advanced opt-in feature. Its current per-file implementation is covered by a two-Vault real-Obsidian workflow for snippets, configuration files, and plug-in files. Hidden File Sync is a separate feature with different setup, selection, and conflict behaviour; do not use both features to manage the same files. +**Automatic** delegates an item's files to Hidden File Sync. Enable and initialise Hidden File Sync in **Sync Settings** on this device first, after reviewing its [target and ignore patterns](./tips/hidden-file-sync.md#review-the-file-selection). Choosing Automatic while Hidden File Sync is disabled displays a Notice and leaves the current mode unchanged. Existing Automatic assignments, including those imported from another device, remain saved while Hidden File Sync is disabled. + ### 1. Customisation Sync #### Device name diff --git a/docs/tips/hidden-file-sync.md b/docs/tips/hidden-file-sync.md index 6b270da2..c8dccf1c 100644 --- a/docs/tips/hidden-file-sync.md +++ b/docs/tips/hidden-file-sync.md @@ -68,6 +68,12 @@ Hidden File Sync must be enabled independently after ordinary setup on each devi If you later run Rebuild or Fetch as a recovery operation, expect optional features to be disabled again. Complete ordinary recovery first, then repeat this guide deliberately. +## Automatic in Customisation Sync + +**Automatic** delegates an item's files to Hidden File Sync. Enable Hidden File Sync on this device through the steps above before choosing Automatic. If Hidden File Sync is disabled, the choice displays a Notice without changing the item's mode or starting initialisation. Previously saved Automatic assignments remain saved, so this prerequisite also applies after importing settings from another device. + +Once Hidden File Sync is enabled, choose the item's initial action in Customisation Sync. Selecting a whole plug-in applies that action to both its **MAIN** and **DATA** selections. + ## Related settings The [settings reference](../settings.md#7-hidden-files-advanced) describes periodic scanning, scanning before replication, notification suppression, selectors, and overwrite patterns. Change those controls only after the basic Hidden File Sync path is working. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 34ae100b..650dea93 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,9 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files.": + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files.", + "Hidden File Sync initialisation was not started.": "Hidden File Sync initialisation was not started.", "Preparing follow-up reportâ€Ļ": "Preparing follow-up reportâ€Ļ", "Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.", "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.": diff --git a/src/features/ConfigSync/PluginPane.svelte b/src/features/ConfigSync/PluginPane.svelte index 18a65109..5be97ac9 100644 --- a/src/features/ConfigSync/PluginPane.svelte +++ b/src/features/ConfigSync/PluginPane.svelte @@ -20,10 +20,14 @@ MODE_SHINY, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { normalizePath } from "@/deps"; - import { HiddenFileSync } from "@/features/HiddenFileSync/CmdHiddenFileSync.ts"; + import { + HiddenFileSync, + HIDDEN_FILE_SYNC_DISABLED_NOTICE, + } from "@/features/HiddenFileSync/CmdHiddenFileSync.ts"; import { LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger"; import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore.ts"; import { $msg as translateMessage } from "@/common/translation"; + import { getAutomaticSyncTargetFiles } from "./getAutomaticSyncTargetFiles.ts"; import { REPLICATION_PROGRESS_PRESENTATIONS, USER_INITIATED_REPLICATION_AUTHORITY, @@ -139,6 +143,11 @@ async function deleteData(data: IPluginDataExDisplay): Promise { return await addOn.deleteData(data); } + function checkAutomaticSyncAvailable() { + if (addOnHiddenFileSync.isThisModuleEnabled()) return true; + Logger(translateMessage(HIDDEN_FILE_SYNC_DISABLED_NOTICE), LOG_LEVEL_NOTICE); + return false; + } function askMode(evt: MouseEvent, title: string, key: string) { const menu = new Menu(); menu.addItem((item) => item.setTitle(title).setIsLabel(true)); @@ -149,6 +158,7 @@ item.setTitle(`${getIcon(mode as SYNC_MODE)}:${TITLES[mode]}`) .onClick((e) => { if (mode === MODE_AUTOMATIC) { + if (!checkAutomaticSyncAvailable()) return; askOverwriteModeForAutomatic(evt, key); } else { setMode(key, mode as SYNC_MODE); @@ -161,10 +171,11 @@ menu.showAtMouseEvent(evt); } function applyAutomaticSync(key: string, direction: "pushForce" | "pullForce" | "safe") { + if (!checkAutomaticSyncAvailable()) return; setMode(key, MODE_AUTOMATIC); const configDir = normalizePath(plugin.core.services.API.getSystemConfigDir()); - const files = (plugin.core.settings.pluginSyncExtendedSetting[key]?.files ?? []).map((e) => `${configDir}/${e}`); - addOnHiddenFileSync.initialiseInternalFileSync(direction, true, files); + const files = getAutomaticSyncTargetFiles(key, configDir, plugin.core.settings.pluginSyncExtendedSetting); + void addOnHiddenFileSync.initialiseInternalFileSync(direction, true, files); } function askOverwriteModeForAutomatic(evt: MouseEvent, key: string) { const menu = new Menu(); diff --git a/src/features/ConfigSync/getAutomaticSyncTargetFiles.ts b/src/features/ConfigSync/getAutomaticSyncTargetFiles.ts new file mode 100644 index 00000000..181e3669 --- /dev/null +++ b/src/features/ConfigSync/getAutomaticSyncTargetFiles.ts @@ -0,0 +1,13 @@ +import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; + +export function getAutomaticSyncTargetFiles( + key: string, + configDir: string, + entries: ObsidianLiveSyncSettings["pluginSyncExtendedSetting"] +): string[] { + const keys = key.startsWith("PLUGIN_ALL/") + ? [`PLUGIN_DATA/${key.substring("PLUGIN_ALL/".length)}`, `PLUGIN_MAIN/${key.substring("PLUGIN_ALL/".length)}`] + : [key]; + const files = new Set(keys.flatMap((entryKey) => entries[entryKey]?.files ?? [])); + return [...files].map((file) => `${configDir}/${file}`); +} diff --git a/src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts b/src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts new file mode 100644 index 00000000..a1c8265c --- /dev/null +++ b/src/features/ConfigSync/getAutomaticSyncTargetFiles.unit.spec.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { MODE_AUTOMATIC, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { getAutomaticSyncTargetFiles } from "./getAutomaticSyncTargetFiles.ts"; + +function createEntries(): ObsidianLiveSyncSettings["pluginSyncExtendedSetting"] { + return { + "CONFIG/app.json": { key: "CONFIG/app.json", mode: MODE_AUTOMATIC, files: ["app.json"] }, + "PLUGIN_MAIN/example": { + key: "PLUGIN_MAIN/example", + mode: MODE_AUTOMATIC, + files: ["plugins/example/main.js", "plugins/example/manifest.json"], + }, + "PLUGIN_DATA/example": { + key: "PLUGIN_DATA/example", + mode: MODE_AUTOMATIC, + files: ["plugins/example/data.json"], + }, + "PLUGIN_ETC/example": { + key: "PLUGIN_ETC/example", + mode: MODE_AUTOMATIC, + files: ["plugins/example/extra.json"], + }, + "PLUGIN_MAIN/other": { + key: "PLUGIN_MAIN/other", + mode: MODE_AUTOMATIC, + files: ["plugins/other/main.js"], + }, + }; +} + +describe("getAutomaticSyncTargetFiles", () => { + it("initialises a single selected item in the current configuration directory", () => { + expect(getAutomaticSyncTargetFiles("CONFIG/app.json", ".custom-config", createEntries())).toEqual([ + ".custom-config/app.json", + ]); + }); + + it("initialises both saved child selections when the whole plug-in is selected", () => { + expect(getAutomaticSyncTargetFiles("PLUGIN_ALL/example", ".obsidian", createEntries())).toEqual([ + ".obsidian/plugins/example/data.json", + ".obsidian/plugins/example/main.js", + ".obsidian/plugins/example/manifest.json", + ]); + }); + + it("retains the available child files when the other child has no saved entry", () => { + const entries = createEntries(); + delete entries["PLUGIN_DATA/example"]; + expect(getAutomaticSyncTargetFiles("PLUGIN_ALL/example", ".obsidian", entries)).toEqual([ + ".obsidian/plugins/example/main.js", + ".obsidian/plugins/example/manifest.json", + ]); + }); + + it("keeps an empty selection empty rather than including unrelated plug-ins", () => { + expect(getAutomaticSyncTargetFiles("PLUGIN_ALL/absent", ".obsidian", createEntries())).toEqual([]); + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.ts index 5b945c69..48564566 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.ts @@ -69,6 +69,9 @@ type HiddenFileInitialisationProgress = { const HIDDEN_FILE_NOTICE_GROUP = "hidden-file-changes"; const HIDDEN_FILE_NOTICE_DURATION_MS = 20_000; +export const HIDDEN_FILE_SYNC_DISABLED_NOTICE = + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files."; + function getComparingMTime( doc: (MetaEntry | LoadedEntry | false) | UXFileInfo | UXStat | null | undefined, includeDeleted = false @@ -1444,6 +1447,11 @@ Offline Changed files: ${files.length}`; initialisationProgress?: HiddenFileInitialisationProgress ) { const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; + if (!this.isThisModuleEnabled()) { + this._log($msg(HIDDEN_FILE_SYNC_DISABLED_NOTICE), logLevel); + initialisationProgress?.done($msg("Hidden File Sync initialisation was not started.")); + return false; + } const p = initialisationProgress ?? this._progress("[⚙ Initialise]\n", logLevel); // p.log("Resolving conflicts before starting..."); // await this.resolveConflictOnInternalFiles(); @@ -1494,6 +1502,7 @@ Offline Changed files: ${files.length}`; await this.scanAllDatabaseChanges(showChildNotices, true, false); } p.done(); + return true; } // <-- Initialization functions @@ -1925,6 +1934,7 @@ Offline Changed files: ${files.length}`; async configureHiddenFileSync(mode: OptionalSyncFeatureMode) { let initialisationProgress: HiddenFileInitialisationProgress | undefined; + let initialised = false; let result: ConfigureHiddenFileSyncResult; try { result = await configureHiddenFileSyncMode(mode, { @@ -1954,7 +1964,7 @@ Offline Changed files: ${files.length}`; ); }, initialise: async (direction) => { - await this.initialiseInternalFileSync(direction, true, false, initialisationProgress); + initialised = await this.initialiseInternalFileSync(direction, true, false, initialisationProgress); initialisationProgress = undefined; }, }); @@ -1962,7 +1972,7 @@ Offline Changed files: ${files.length}`; initialisationProgress?.done("Failed"); throw error; } - if (result == "ignored" || result == "disabled") { + if (result == "ignored" || result == "disabled" || !initialised) { return; } // this.plugin.settings.useAdvancedMode = true; diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts index ad664a0f..9018b222 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts @@ -99,6 +99,115 @@ function createHiddenRevisionOperation() { } describe("HiddenFileSync configuration-change notices", () => { + it.each(["pushForce", "pullForce", "safe"] as const)( + "refuses %s initialisation before processing files while Hidden File Sync is disabled", + async (direction) => { + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const path = ".obsidian/example.json" as FilePath; + const operations = { + rebuildFromStorage: vi.fn(async () => [path]), + rebuildFromDatabase: vi.fn(async () => []), + rebuildMerging: vi.fn(async () => [path]), + adoptCurrentStorageFilesAsProcessed: vi.fn(async () => undefined), + adoptCurrentDatabaseFilesAsProcessed: vi.fn(async () => undefined), + scanAllStorageChanges: vi.fn(async () => undefined), + scanAllDatabaseChanges: vi.fn(async () => undefined), + }; + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + const settings = { syncInternalFiles: false }; + const createProgress = vi.fn(() => progress); + const log = vi.fn(); + Object.assign(hiddenFileSync, { + core: { settings }, + _progress: createProgress, + _log: log, + ...operations, + }); + + const result = await hiddenFileSync.initialiseInternalFileSync(direction, true, [path]); + + for (const operation of Object.values(operations)) { + expect(operation).not.toHaveBeenCalled(); + } + expect(result).toBe(false); + expect(settings.syncInternalFiles).toBe(false); + expect(createProgress).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith(expect.stringContaining("Hidden File Sync is disabled"), LOG_LEVEL_NOTICE); + expect(progress.done).not.toHaveBeenCalled(); + } + ); + + it("finishes an existing preparation Notice as not started when initialisation is disabled", async () => { + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const createProgress = vi.fn(); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { settings: { syncInternalFiles: false } }, + _progress: createProgress, + _log: vi.fn(), + }); + + const result = await hiddenFileSync.initialiseInternalFileSync("safe", true, false, progress); + + expect(result).toBe(false); + expect(createProgress).not.toHaveBeenCalled(); + expect(progress.done).toHaveBeenCalledExactlyOnceWith("Hidden File Sync initialisation was not started."); + }); + + it("does not apply a stored deletion when initialisation is requested while disabled", async () => { + const path = ".obsidian/example.json" as FilePath; + const deleted: MetaEntry = { + _id: "i:example" as DocumentID, + _rev: "2-deleted", + path: `i:${path}` as FilePathWithPrefix, + type: "plain", + ctime: 1, + mtime: 10, + size: 0, + deleted: true, + children: [], + eden: {}, + }; + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const deleteFile = vi.fn(async () => "OK" as const); + const updateDeletion = vi.fn(); + const getFiles = vi.fn(async () => [deleted]); + const processedStorage = new Map([[path, "storage-processed"]]); + const processedDatabase = new Map([[path, "database-processed"]]); + const resetProcessed = vi.fn(() => processedDatabase.clear()); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { + settings: { syncInternalFiles: false }, + services: { vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) } }, + }, + _progress: vi.fn(() => progress), + _log: vi.fn(), + _verbose: vi.fn(), + _fileInfoLastProcessed: processedStorage, + _databaseInfoLastProcessed: processedDatabase, + getAllDatabaseFiles: getFiles, + getPath: (doc: MetaEntry) => doc.path, + resetLastProcessedDatabase: resetProcessed, + serializedForEvent: async (_path: unknown, operation: () => Promise) => await operation(), + __deleteFile: deleteFile, + updateLastProcessedDeletion: updateDeletion, + adoptCurrentStorageFilesAsProcessed: vi.fn(), + adoptCurrentDatabaseFilesAsProcessed: vi.fn(), + scanAllStorageChanges: vi.fn(), + scanAllDatabaseChanges: vi.fn(), + }); + + await hiddenFileSync.initialiseInternalFileSync("pullForce", true); + + expect(deleteFile).not.toHaveBeenCalled(); + expect(updateDeletion).not.toHaveBeenCalled(); + expect(getFiles).not.toHaveBeenCalled(); + expect(resetProcessed).not.toHaveBeenCalled(); + expect(processedStorage).toEqual(new Map([[path, "storage-processed"]])); + expect(processedDatabase).toEqual(new Map([[path, "database-processed"]])); + }); + it("shows manual Hidden File Sync commands only when the feature, Advanced mode, and runtime are ready", () => { const commands: Array<{ id: string; @@ -136,6 +245,13 @@ describe("HiddenFileSync configuration-change notices", () => { expect(command?.checkCallback?.(true)).toBe(false); } + settings.useAdvancedMode = true; + for (const commandId of commandIds) { + const command = commands.find(({ id }) => id === commandId); + expect(command?.checkCallback?.(true)).toBe(false); + expect(command?.checkCallback?.(false)).toBe(false); + } + settings.syncInternalFiles = true; settings.useAdvancedMode = true; for (const commandId of commandIds) { @@ -252,6 +368,7 @@ describe("HiddenFileSync configuration-change notices", () => { const scanAllDatabaseChanges = vi.fn(async () => undefined); const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; Object.assign(hiddenFileSync, { + core: { settings: { syncInternalFiles: true } }, _progress: vi.fn(() => progress), rebuildMerging, adoptCurrentStorageFilesAsProcessed, @@ -260,14 +377,83 @@ describe("HiddenFileSync configuration-change notices", () => { scanAllDatabaseChanges, }); - await hiddenFileSync.initialiseInternalFileSync("safe", true); + const result = await hiddenFileSync.initialiseInternalFileSync("safe", true); + expect(result).toBe(true); expect(rebuildMerging).toHaveBeenCalledWith(false, false); expect(scanAllStorageChanges).toHaveBeenCalledWith(false, true, false); expect(scanAllDatabaseChanges).toHaveBeenCalledWith(false, true, false); expect(progress.done).toHaveBeenCalledOnce(); }); + it.each(["pushForce", "pullForce", "safe"] as const)( + "retains the selected files and completes %s initialisation when enabled", + async (direction) => { + const files = [".obsidian/plugins/example/main.js", ".obsidian/plugins/example/data.json"] as FilePath[]; + const entries = files.map((path) => ({ path: `i:${path}` as FilePathWithPrefix })); + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const rebuildFromStorage = vi.fn(async () => files); + const rebuildFromDatabase = vi.fn(async () => entries); + const rebuildMerging = vi.fn(async () => files); + const adoptStorage = vi.fn(); + const adoptDatabase = vi.fn(); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { settings: { syncInternalFiles: true } }, + _progress: vi.fn(() => progress), + getPath: (entry: { path: FilePathWithPrefix }) => entry.path, + rebuildFromStorage, + rebuildFromDatabase, + rebuildMerging, + adoptCurrentStorageFilesAsProcessed: adoptStorage, + adoptCurrentDatabaseFilesAsProcessed: adoptDatabase, + scanAllStorageChanges: vi.fn(), + scanAllDatabaseChanges: vi.fn(), + }); + + const result = await hiddenFileSync.initialiseInternalFileSync(direction, true, files); + + expect(result).toBe(true); + if (direction === "pushForce") { + expect(rebuildFromStorage).toHaveBeenCalledExactlyOnceWith(false, files, false); + expect(rebuildFromDatabase).not.toHaveBeenCalled(); + expect(rebuildMerging).not.toHaveBeenCalled(); + } else if (direction === "pullForce") { + expect(rebuildFromDatabase).toHaveBeenCalledExactlyOnceWith(false, files, false); + expect(rebuildFromStorage).not.toHaveBeenCalled(); + expect(rebuildMerging).not.toHaveBeenCalled(); + } else { + expect(rebuildMerging).toHaveBeenCalledExactlyOnceWith(false, files); + expect(rebuildFromStorage).not.toHaveBeenCalled(); + expect(rebuildFromDatabase).not.toHaveBeenCalled(); + } + expect(adoptStorage).toHaveBeenCalledExactlyOnceWith(files); + expect(adoptDatabase).toHaveBeenCalledExactlyOnceWith(files); + expect(progress.done).toHaveBeenCalledOnce(); + } + ); + + it("does not report completion when the initialisation entry point refuses to start", async () => { + vi.mocked(configureHiddenFileSyncMode).mockImplementation(async (_mode, handlers) => { + await handlers.enable(); + await handlers.initialise("safe"); + return "enabled"; + }); + const progress = { log: vi.fn(), once: vi.fn(), done: vi.fn() }; + const log = vi.fn(); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { services: { setting: { applyPartial: vi.fn() } } }, + _progress: vi.fn(() => progress), + _log: log, + initialiseInternalFileSync: vi.fn(async () => false), + }); + + await hiddenFileSync.configureHiddenFileSync("MERGE"); + + expect(log).not.toHaveBeenCalledWith("Hidden File Sync initialisation completed.", expect.any(Number)); + }); + it("retirement guard: does not restore separate gathering and restart Notices", async () => { vi.mocked(configureHiddenFileSyncMode).mockImplementation(async (_mode, handlers) => { await handlers.enable(); @@ -286,7 +472,7 @@ describe("HiddenFileSync configuration-change notices", () => { const applyPartial = vi.fn(async () => { events.push("apply-settings"); }); - const initialiseInternalFileSync = vi.fn(async () => undefined); + const initialiseInternalFileSync = vi.fn(async () => true); const log = vi.fn(); const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; Object.assign(hiddenFileSync, { diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..77358281 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -248,10 +248,14 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. +Before enablement, the scenario directly requests initialisation while Hidden File Sync is disabled. It requires a refused result, unchanged local files, Metadata revisions, and processed markers, and no initialisation progress Notice. + `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. +`--case=automatic` separately checks that choosing **Automatic** while Hidden File Sync is disabled displays a Notice without changing modes, files, Metadata revisions, or processed markers. The fixture starts with a saved Automatic selection and requires it to survive start-up and both rejected actions. It also checks disabling the feature after opening **Initial Action**. After enablement, selecting a whole plug-in must initialise both **MAIN** and **DATA**, and transfer their files to the other Vault. + `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`. `test:e2e:obsidian:remote-feature-change` starts real Obsidian with continuous CouchDB replication, then changes the remote version document from generation 12 to generation 13 with an unknown feature. It waits for the control document to reach the local database and the active Replicator to retire, checks that another replication is refused, and verifies that an already accepted Vault note remains intact. After restarting the same Vault, the current remote declaration still blocks finite replication and the actual continuous connection attempt. No KV feature history is involved. It is a focused test outside `test:e2e:obsidian:local-suite`; recovery with a future compatible client remains a separate validation boundary. diff --git a/test/e2e-obsidian/scripts/customisation-sync.ts b/test/e2e-obsidian/scripts/customisation-sync.ts index 78fd7a96..f20391e2 100644 --- a/test/e2e-obsidian/scripts/customisation-sync.ts +++ b/test/e2e-obsidian/scripts/customisation-sync.ts @@ -1,6 +1,10 @@ import { mkdir, readFile, rm, utimes, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; -import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + MODE_AUTOMATIC, + VERSIONING_DOCID, + type ObsidianLiveSyncSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication"; import { evalObsidianJson } from "../runner/cli.ts"; import { @@ -23,11 +27,12 @@ import { prepareRemote, pushLocalChanges, waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, } from "../runner/liveSyncWorkflow.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { waitForVisibleObsidianDialogue, withObsidianPage } from "../runner/ui.ts"; import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; -import type { Locator } from "playwright"; +import type { Locator, Page } from "playwright"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000"; process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "20000"; @@ -59,6 +64,18 @@ const pluginDir = ".obsidian/plugins/livesync-e2e-sample"; const pluginManifestPath = `${pluginDir}/manifest.json`; const pluginMainPath = `${pluginDir}/main.js`; const pluginStylesPath = `${pluginDir}/styles.css`; +const pluginDisplayName = "LiveSync E2E Sample"; +const pluginDataPath = `${pluginDir}/data.json`; +const pluginDataContent = JSON.stringify({ source: "customisation-sync-a", enabled: true }, null, 4) + "\n"; +const targetPluginDataContent = JSON.stringify({ source: "customisation-sync-b", enabled: true }, null, 4) + "\n"; +const existingAutomaticKey = "SNIPPET/livesync-e2e-existing-automatic.css"; +const existingAutomaticSelections: ObsidianLiveSyncSettings["pluginSyncExtendedSetting"] = { + [existingAutomaticKey]: { + key: existingAutomaticKey, + mode: MODE_AUTOMATIC, + files: ["snippets/livesync-e2e-existing-automatic.css"], + }, +}; const identicalSnippetPath = ".obsidian/snippets/livesync-customisation-e2e-identical.css"; const identicalSnippetContent = ".livesync-customisation-identical { color: #3d6f54; }\n"; const differentSnippetPath = ".obsidian/snippets/livesync-customisation-e2e-different.css"; @@ -104,7 +121,7 @@ const pluginFixtures = [ { path: pluginStylesPath, source: pluginStylesContent, target: targetPluginStylesContent }, ] as const; -type CustomisationSyncCase = "all" | "visibility" | "mtime"; +type CustomisationSyncCase = "all" | "visibility" | "mtime" | "automatic"; type CustomisationSyncTestGlobal = typeof globalThis & { app?: { commands?: { executeCommandById(commandId: string): boolean } }; @@ -131,6 +148,14 @@ type CustomisationScanResult = { files: string[]; }; +type AutomaticSyncSnapshot = { + syncInternalFiles: boolean; + pluginSyncExtendedSetting: Record; + hiddenFiles: { path: string; rev: string; deleted: boolean }[]; + processedFiles: [string, string | undefined][]; + processedDatabaseFiles: [string, string | undefined][]; +}; + async function writeVaultFile(vaultPath: string, path: string, content: string): Promise { const fullPath = join(vaultPath, path); await mkdir(dirname(fullPath), { recursive: true }); @@ -147,10 +172,11 @@ function selectedCase(): CustomisationSyncCase { if (args.length === 0) return "all"; const arg = args[0]; if (args.length !== 1 || !arg?.startsWith("--case=")) { - throw new Error("Usage: test:e2e:obsidian:customisation-sync [--case=all|visibility|mtime]"); + throw new Error("Usage: test:e2e:obsidian:customisation-sync [--case=all|visibility|mtime|automatic]"); } const selected = arg.slice("--case=".length); - if (selected === "all" || selected === "visibility" || selected === "mtime") return selected; + if (selected === "all" || selected === "visibility" || selected === "mtime" || selected === "automatic") + return selected; throw new Error(`Unknown Customisation Sync E2E case: ${selected}`); } @@ -244,6 +270,196 @@ async function inspectPluginFreshnessAndNewestSelection(session: ObsidianLiveSyn }); } +async function openAutomaticModeForPlugin(page: Page, pluginName: string): Promise { + const opened = await page.evaluate(() => + (globalThis as CustomisationSyncTestGlobal).app?.commands?.executeCommandById( + "obsidian-livesync:livesync-plugin-dialog-ex" + ) + ); + if (opened !== true) throw new Error("Could not open the Customisation Sync dialogue command."); + const dialogue = await waitForVisibleObsidianDialogue(page, "Customization Sync (Beta3)"); + const pluginRow = dialogue.locator(".labelrow").filter({ hasText: pluginName }).first(); + await pluginRow.waitFor({ state: "visible", timeout: 10000 }); + await pluginRow.locator("button.status").click(); + const automaticOption = page.locator(".menu-item").filter({ hasText: "Automatic" }).last(); + await automaticOption.waitFor({ state: "visible", timeout: 10000 }); + await automaticOption.click(); + return dialogue; +} + +async function selectAutomaticInitialAction(page: Page, label: "Overwrite Remote" | "Overwrite Local"): Promise { + const action = page.locator(".menu-item").filter({ hasText: label }).last(); + await action.waitFor({ state: "visible", timeout: 10000 }); + await action.click(); +} + +async function readAutomaticSyncSnapshot(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const hidden=core.getAddOn('HiddenFileSync');", + "const files=await hidden.getAllDatabaseFiles();", + "const processed=(cache)=>Array.from(cache.keys()).sort().map((path)=>[path,cache.get(path)]);", + "return JSON.stringify({", + "syncInternalFiles:core.settings.syncInternalFiles,", + "pluginSyncExtendedSetting:core.settings.pluginSyncExtendedSetting,", + "hiddenFiles:files.map((entry)=>{const path=hidden.getPath(entry);return {path:path.startsWith('i:')?path.slice(2):path,rev:entry._rev??'',deleted:!!(entry.deleted||entry._deleted)};}).sort((a,b)=>a.path.localeCompare(b.path)),", + "processedFiles:processed(hidden._fileInfoLastProcessed),", + "processedDatabaseFiles:processed(hidden._databaseInfoLastProcessed),", + "});", + "})()", + ].join(""), + env + ); +} + +async function setHiddenSyncEnabledForFixture( + cliBinary: string, + env: NodeJS.ProcessEnv, + enabled: boolean +): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + `core.settings.syncInternalFiles=${JSON.stringify(enabled)};`, + "return JSON.stringify({syncInternalFiles:core.settings.syncInternalFiles});", + "})()", + ].join(""), + env + ); +} + +async function inspectAutomaticSyncSelection( + context: RunnerContext, + session: ObsidianLiveSyncSession, + vault: TemporaryVault, + pluginName: string +): Promise { + const noticeText = + "Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files."; + const trackedFiles = [pluginStylesPath, pluginDataPath]; + const contentsBefore = await Promise.all(trackedFiles.map((path) => readVaultFile(vault.path, path))); + const disabledBefore = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + assertEqual( + JSON.stringify(disabledBefore.pluginSyncExtendedSetting[existingAutomaticKey]), + JSON.stringify(existingAutomaticSelections[existingAutomaticKey]), + "The fixture did not retain the saved Automatic selection while Hidden File Sync was disabled." + ); + + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await openAutomaticModeForPlugin(page, pluginDisplayName); + const notice = page.locator(".notice:visible").filter({ hasText: noticeText }).last(); + await notice.waitFor({ state: "visible", timeout: 10000 }); + const initialActionMenu = page.locator(".menu:visible").filter({ hasText: "Initial Action" }); + if ((await initialActionMenu.count()) !== 0) { + throw new Error("Automatic displayed an Initial Action menu while Hidden File Sync was disabled."); + } + await page.keyboard.press("Escape"); + await dialogue.waitFor({ state: "hidden", timeout: 10000 }); + await notice.click({ force: true }); + await notice.waitFor({ state: "hidden", timeout: 10000 }); + }); + + const disabledAfter = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + const contentsAfter = await Promise.all(trackedFiles.map((path) => readVaultFile(vault.path, path))); + if (JSON.stringify(disabledAfter) !== JSON.stringify(disabledBefore)) { + throw new Error( + `Automatic changed Hidden File Sync state while it was disabled: ${JSON.stringify({ + before: disabledBefore, + after: disabledAfter, + })}` + ); + } + assertEqual( + JSON.stringify(contentsAfter), + JSON.stringify(contentsBefore), + "Automatic changed plug-in files while Hidden File Sync was disabled." + ); + + // Exercise the second guard: the user may open Initial Action while enabled, + // then disable Hidden File Sync before choosing a direction. + await setHiddenSyncEnabledForFixture(context.cliBinary, session.cliEnv, true); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await openAutomaticModeForPlugin(page, pluginDisplayName); + const initialActionMenu = page.locator(".menu:visible").filter({ hasText: "Initial Action" }); + await initialActionMenu.waitFor({ state: "visible", timeout: 10000 }); + }); + await setHiddenSyncEnabledForFixture(context.cliBinary, session.cliEnv, false); + const beforeDisabledAction = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await selectAutomaticInitialAction(page, "Overwrite Remote"); + const notice = page.locator(".notice:visible").filter({ hasText: noticeText }).last(); + await notice.waitFor({ state: "visible", timeout: 10000 }); + await notice.click({ force: true }); + await notice.waitFor({ state: "hidden", timeout: 10000 }); + }); + const afterDisabledAction = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + if (JSON.stringify(afterDisabledAction) !== JSON.stringify(beforeDisabledAction)) { + throw new Error( + `Automatic changed Hidden File Sync state after it was disabled from Initial Action: ${JSON.stringify({ + before: beforeDisabledAction, + after: afterDisabledAction, + })}` + ); + } + assertEqual( + JSON.stringify(await Promise.all(trackedFiles.map((path) => readVaultFile(vault.path, path)))), + JSON.stringify(contentsBefore), + "Automatic changed plug-in files after Hidden File Sync was disabled from Initial Action." + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.keyboard.press("Escape"); + }); + + await setHiddenSyncEnabledForFixture(context.cliBinary, session.cliEnv, true); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await openAutomaticModeForPlugin(page, pluginDisplayName); + await selectAutomaticInitialAction(page, "Overwrite Remote"); + for (const label of ["MAIN", "DATA"]) { + const row = dialogue.locator(".filerow").filter({ hasText: label }).first(); + await row.locator(".statusnote").filter({ hasText: "Automatic" }).waitFor({ state: "visible" }); + } + await page.keyboard.press("Escape"); + await dialogue.waitFor({ state: "hidden", timeout: 10000 }); + }); + + for (const path of [pluginManifestPath, pluginMainPath, pluginStylesPath, pluginDataPath]) { + await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, path, { hidden: true }); + } + const enabled = await readAutomaticSyncSnapshot(context.cliBinary, session.cliEnv); + const mainMode = enabled.pluginSyncExtendedSetting[`PLUGIN_MAIN/${pluginName}`]; + const dataMode = enabled.pluginSyncExtendedSetting[`PLUGIN_DATA/${pluginName}`]; + if (mainMode?.mode !== MODE_AUTOMATIC || dataMode?.mode !== MODE_AUTOMATIC) { + throw new Error( + `Automatic did not save both plug-in group child settings: ${JSON.stringify({ mainMode, dataMode })}` + ); + } + assertEqual( + JSON.stringify([...mainMode.files].sort()), + JSON.stringify([ + `plugins/${pluginName}/main.js`, + `plugins/${pluginName}/manifest.json`, + `plugins/${pluginName}/styles.css`, + ]), + "Automatic did not select the PLUGIN_MAIN files." + ); + assertEqual( + JSON.stringify([...dataMode.files].sort()), + JSON.stringify([`plugins/${pluginName}/data.json`]), + "Automatic did not select the PLUGIN_DATA file." + ); + assertEqual( + enabled.hiddenFiles.filter((entry) => entry.path.startsWith(pluginDir + "/")).length, + 4, + "Automatic did not initialise all files in the PLUGIN_ALL group." + ); + await pushLocalChanges(context.cliBinary, session.cliEnv); +} + async function removeVaultFile(vaultPath: string, path: string): Promise { await rm(join(vaultPath, path), { force: true }); } @@ -287,7 +503,8 @@ async function waitForPathContent( async function startConfiguredSession( context: RunnerContext, vault: TemporaryVault, - deviceName: string + deviceName: string, + settings: Partial> = {} ): Promise { const couchDbSettings = { uri: context.couchDb.uri, @@ -306,6 +523,7 @@ async function startConfiguredSession( autoSweepPlugins: false, autoSweepPluginsPeriodic: false, syncInternalFiles: false, + ...settings, }; const session = await startObsidianLiveSyncSession({ binary: context.binary, @@ -319,6 +537,18 @@ async function startConfiguredSession( }); await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); await assertE2eCompatibilityMarker(context.cliBinary, session.cliEnv); + if (Object.keys(settings.pluginSyncExtendedSetting ?? {}).length > 0) { + const restoredSelections = await evalObsidianJson( + context.cliBinary, + "JSON.stringify(app.plugins.plugins['obsidian-livesync'].core.settings.pluginSyncExtendedSetting)", + session.cliEnv + ); + assertEqual( + JSON.stringify(restoredSelections), + JSON.stringify(settings.pluginSyncExtendedSetting), + "The saved Automatic selection changed during start-up." + ); + } await configureCouchDb(context.cliBinary, session.cliEnv, couchDbSettings, customisationSettings); await evalObsidianJson( context.cliBinary, @@ -389,7 +619,7 @@ async function storeCustomisationFile(cliBinary: string, env: NodeJS.ProcessEnv, async function deleteCustomisationSyncEntry( cliBinary: string, env: NodeJS.ProcessEnv, - category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN", + category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN" | "PLUGIN_DATA", name: string, term?: string ): Promise { @@ -418,7 +648,7 @@ async function deleteCustomisationSyncEntry( async function waitForCustomisationEntry( cliBinary: string, env: NodeJS.ProcessEnv, - category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN", + category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN" | "PLUGIN_DATA", name: string, term?: string, timeoutMs = Number(process.env.E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS ?? 15000) @@ -430,7 +660,7 @@ async function waitForCustomisationEntry( async function waitForCustomisationEntries( cliBinary: string, env: NodeJS.ProcessEnv, - category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN", + category: "CONFIG" | "SNIPPET" | "PLUGIN_MAIN" | "PLUGIN_DATA", name: string, count: number, term?: string, @@ -574,6 +804,7 @@ async function main(): Promise { const testCase = selectedCase(); const testVisibility = testCase === "all" || testCase === "visibility"; const testMtime = testCase === "all" || testCase === "mtime"; + const testAutomatic = testCase === "automatic"; const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); if (!cli.binary) { @@ -604,6 +835,7 @@ async function main(): Promise { await writeVaultFile(vaultA.path, pluginManifestPath, pluginManifestContent); await writeVaultFile(vaultA.path, pluginMainPath, pluginMainContent); await writeVaultFile(vaultA.path, pluginStylesPath, pluginStylesContent); + if (testAutomatic) await writeVaultFile(vaultA.path, pluginDataPath, pluginDataContent); if (testVisibility) { for (const fixture of visibilityFixtures) { await writeVaultFile(vaultA.path, fixture.path, fixture.source); @@ -627,6 +859,12 @@ async function main(): Promise { await setVaultFileMtime(vaultB.path, fixture.path, targetPluginMtime); } } + if (testAutomatic) { + for (const fixture of pluginFixtures) { + await writeVaultFile(vaultB.path, fixture.path, fixture.target); + } + await writeVaultFile(vaultB.path, pluginDataPath, targetPluginDataContent); + } let session = await startConfiguredSession(context, vaultA, sourceDeviceName); const scanResult = await scanCustomisations(context.cliBinary, session.cliEnv); @@ -636,6 +874,9 @@ async function main(): Promise { for (const fixture of pluginFixtures) { await storeCustomisationFile(context.cliBinary, session.cliEnv, fixture.path); } + if (testAutomatic) { + await storeCustomisationFile(context.cliBinary, session.cliEnv, pluginDataPath); + } if (testVisibility) { for (const fixture of visibilityFixtures) { await storeCustomisationFile(context.cliBinary, session.cliEnv, fixture.path); @@ -650,15 +891,18 @@ async function main(): Promise { pluginName, 3 ); + const pluginDataEntry = testAutomatic + ? await waitForCustomisationEntry(context.cliBinary, session.cliEnv, "PLUGIN_DATA", pluginName) + : undefined; await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForCouchDbDocs(context.couchDb, context.dbName, (docs) => { const ids = new Set(docs.map((doc) => doc._id)); - const entries = [entry, configEntry, ...pluginEntries]; + const entries = [entry, configEntry, ...pluginEntries, ...(pluginDataEntry ? [pluginDataEntry] : [])]; return entries.every( (target) => ids.has(target.id) && target.children.every((childId) => ids.has(childId)) ); }); - for (const target of [entry, configEntry, ...pluginEntries]) { + for (const target of [entry, configEntry, ...pluginEntries, ...(pluginDataEntry ? [pluginDataEntry] : [])]) { const remoteEntry = await fetchCouchDbDocument(context.couchDb, context.dbName, target.id); if ( !remoteEntry.path?.startsWith("/\\:") || @@ -678,8 +922,15 @@ async function main(): Promise { throw new Error("The remote feature list does not declare encrypted internal Metadata."); } await session.app.stop(); + if (testAutomatic) { + for (const path of [pluginManifestPath, pluginMainPath, pluginStylesPath, pluginDataPath]) { + await removeVaultFile(vaultA.path, path); + } + } - session = await startConfiguredSession(context, vaultB, targetDeviceName); + session = await startConfiguredSession(context, vaultB, targetDeviceName, { + pluginSyncExtendedSetting: testAutomatic ? existingAutomaticSelections : {}, + }); if (testVisibility) { for (const fixture of visibilityFixtures) { await storeCustomisationFile(context.cliBinary, session.cliEnv, fixture.path); @@ -692,6 +943,15 @@ async function main(): Promise { } await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForCustomisationEntry(context.cliBinary, session.cliEnv, "SNIPPET", snippetName, sourceDeviceName); + if (testAutomatic) { + await waitForCustomisationEntry( + context.cliBinary, + session.cliEnv, + "PLUGIN_DATA", + pluginName, + sourceDeviceName + ); + } if (testVisibility) { for (const fixture of visibilityFixtures) { const name = fixture.path.split("/").pop() ?? fixture.path; @@ -721,10 +981,42 @@ async function main(): Promise { try { if (testVisibility) await inspectCustomisationVisibility(session); if (testMtime) await inspectPluginFreshnessAndNewestSelection(session); + if (testAutomatic) await inspectAutomaticSyncSelection(context, session, vaultB, pluginName); } catch (error) { await session.app.stop().catch(() => undefined); throw error; } + if (testAutomatic) { + await session.app.stop(); + session = await startConfiguredSession(context, vaultA, sourceDeviceName, { syncInternalFiles: true }); + try { + await pushLocalChanges(context.cliBinary, session.cliEnv); + const receivedStyles = await waitForPathContent( + vaultA.path, + pluginStylesPath, + (content) => content === targetPluginStylesContent + ); + const receivedData = await waitForPathContent( + vaultA.path, + pluginDataPath, + (content) => content === targetPluginDataContent + ); + assertEqual( + receivedStyles, + targetPluginStylesContent, + "Automatic plug-in group lost PLUGIN_MAIN files." + ); + assertEqual( + receivedData, + targetPluginDataContent, + "Automatic plug-in group lost the PLUGIN_DATA file." + ); + } finally { + await session.app.stop(); + } + console.log("Automatic Customisation Sync initialised and transferred both plug-in group children."); + return; + } if (testCase !== "all") { await session.app.stop(); console.log(`Customisation Sync ${testCase} regression case passed.`); diff --git a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts index 5079c021..9f9dba08 100644 --- a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts +++ b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts @@ -63,6 +63,21 @@ const manualMergeJsonPath = ".obsidian/livesync-e2e-manual-merge.json"; const targetPath = ".obsidian/livesync-targeted/only-a.json"; const hiddenFileCliTimeoutMs = Number(process.env.E2E_OBSIDIAN_HIDDEN_FILE_CLI_TIMEOUT_MS ?? 90000); const hiddenFileInitialisationStateKey = "__livesyncE2EHiddenFileInitialisation"; +const disabledInitialisationProbeKey = "__livesyncE2EDisabledHiddenInitialisation"; + +type DisabledInitialisationSnapshot = { + syncInternalFiles: boolean; + hiddenFiles: { id: string; path: string; rev: string; deleted: boolean }[]; + processedFiles: [string, string | undefined][]; + processedDatabaseFiles: [string, string | undefined][]; +}; + +type DisabledInitialisationObservation = { + result: boolean | null; + before: DisabledInitialisationSnapshot; + after: DisabledInitialisationSnapshot; + progressNotices: string[]; +}; type RunnerContext = { binary: string; @@ -132,6 +147,114 @@ async function waitForPathDeleted( throw new Error(`Timed out waiting for deleted file: ${join(vaultPath, path)}`); } +async function assertDisabledInitialisationIsNoOp( + context: RunnerContext, + session: ObsidianLiveSyncSession, + vault: TemporaryVault +): Promise { + const snippetExisted = await pathExists(vault.path, snippetPath); + const originalSnippet = snippetExisted ? await readVaultFile(vault.path, snippetPath) : undefined; + const originalJson = await readVaultFile(vault.path, mergeJsonPath); + const preservedSnippet = "body { --disabled-hidden-initialisation: preserved; }\n"; + const preservedJson = JSON.stringify({ disabledInitialisation: "preserved" }, null, 4) + "\n"; + await writeVaultFile(vault.path, snippetPath, preservedSnippet); + await writeVaultFile(vault.path, mergeJsonPath, preservedJson); + + try { + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.evaluate((stateKey) => { + const state = { progressNotices: [] as string[] }; + const observer = new MutationObserver(() => { + for (const notice of document.querySelectorAll(".notice")) { + const text = notice.textContent?.trim() ?? ""; + if (text.includes("[⚙") && !state.progressNotices.includes(text)) { + state.progressNotices.push(text); + } + } + }); + observer.observe(document.body, { childList: true, subtree: true, characterData: true }); + (globalThis as unknown as Record)[ + stateKey + ] = { ...state, observer }; + }, disabledInitialisationProbeKey); + }); + + const observation = await evalObsidianJson( + context.cliBinary, + [ + "(async()=>{", + `const targetFiles=${JSON.stringify([snippetPath, mergeJsonPath])};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const addOn=core.getAddOn('HiddenFileSync');", + "const snapshot=async()=>{", + "const files=await addOn.getAllDatabaseFiles();", + "const markers=(cache)=>Array.from(cache.keys()).sort().map((path)=>[path,cache.get(path)]);", + "return {", + "syncInternalFiles:core.settings.syncInternalFiles,", + "hiddenFiles:files.map((entry)=>({id:entry._id,path:addOn.getPath(entry),rev:entry._rev??'',deleted:!!(entry.deleted||entry._deleted)})).sort((a,b)=>a.path.localeCompare(b.path)),", + "processedFiles:markers(addOn._fileInfoLastProcessed),", + "processedDatabaseFiles:markers(addOn._databaseInfoLastProcessed),", + "};};", + "const before=await snapshot();", + "const result=await addOn.initialiseInternalFileSync('pullForce',true,targetFiles);", + "const after=await snapshot();", + "return JSON.stringify({result:result===undefined?null:result,before,after});", + "})()", + ].join(""), + session.cliEnv, + hiddenFileCliTimeoutMs + ); + const progressNotices = await withObsidianPage(session.remoteDebuggingPort, async (page) => { + return await page.evaluate((stateKey) => { + const state = ( + globalThis as unknown as Record< + string, + { progressNotices: string[]; observer: MutationObserver } | undefined + > + )[stateKey]; + state?.observer.disconnect(); + return state?.progressNotices ?? []; + }, disabledInitialisationProbeKey); + }); + + const failures: string[] = []; + if (observation.result !== false) + failures.push(`expected false result, received ${String(observation.result)}`); + if (observation.before.syncInternalFiles !== false || observation.after.syncInternalFiles !== false) { + failures.push("Hidden File Sync setting changed while initialisation was disabled"); + } + if (JSON.stringify(observation.before) !== JSON.stringify(observation.after)) { + failures.push("local hidden database state or processed markers changed"); + } + if ((await readVaultFile(vault.path, snippetPath)) !== preservedSnippet) { + failures.push("the existing local snippet was deleted or replaced"); + } + if ((await readVaultFile(vault.path, mergeJsonPath)) !== preservedJson) { + failures.push("the existing local JSON file was replaced"); + } + if (progressNotices.length > 0) + failures.push(`initialisation created progress Notices: ${JSON.stringify(progressNotices)}`); + if (failures.length > 0) { + throw new Error( + `Disabled Hidden File Sync initialisation was not a no-op: ${failures.join("; ")}\n${JSON.stringify({ + observation, + progressNotices, + })}` + ); + } + console.log( + "Disabled Hidden File Sync initialisation returned false without changing hidden files or cached state." + ); + } finally { + await writeVaultFile(vault.path, mergeJsonPath, originalJson); + if (snippetExisted && originalSnippet !== undefined) { + await writeVaultFile(vault.path, snippetPath, originalSnippet); + } else { + await removeVaultFile(vault.path, snippetPath); + } + } +} + function hasJsonValues(content: string, values: Record): boolean { try { const parsed = JSON.parse(content) as Record; @@ -633,6 +756,7 @@ async function runInitialisationNoticeGrouping(context: RunnerContext, vault: Te "Transient start-up Notices remained before the Hidden File Sync initialisation check." ); }); + await assertDisabledInitialisationIsNoOp(context, session, vault); await withObsidianPage(port, async (page) => { await page.evaluate((stateKey) => { const obsidianApp = (globalThis as typeof globalThis & { app: any }).app; diff --git a/updates.md b/updates.md index ca2e3497..cd3c3836 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We receive a clear Notice when **Automatic** requires Hidden File Sync to be enabled on this device. Our existing mode remains unchanged, and hidden-file initialisation cannot start while the feature is disabled. (#1270) +- We can initialise a whole plug-in's **MAIN** and **DATA** selections using the chosen Automatic initial action. + ## 1.0.39 20th October, 2026 From 9d30bed52ddd707a7af3ac9ed867b22a195ebe67 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 01:11:41 +0000 Subject: [PATCH 30/68] Clarify Hidden File Sync selectors and test normal allow-lists --- docs/settings.md | 6 +- docs/tips/hidden-file-sync.md | 4 + package.json | 1 + .../messages/LiveSyncProvisionalMessages.ts | 4 + .../CmdHiddenFileSync.selectors.unit.spec.ts | 127 +++++++++++ .../features/SettingDialogue/PaneSelector.ts | 6 +- test/e2e-obsidian/README.md | 2 + .../scripts/hidden-file-selectors.ts | 202 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + test/fixtures/optionalFileSyncDatabase.ts | 78 +++++++ updates.md | 7 + 11 files changed, 432 insertions(+), 6 deletions(-) create mode 100644 src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/hidden-file-selectors.ts create mode 100644 test/fixtures/optionalFileSyncDatabase.ts diff --git a/docs/settings.md b/docs/settings.md index 0885589c..f328db64 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -750,11 +750,13 @@ If enabled, the notification of hidden files change will be suppressed. #### Synchronising files -(RegExp) Empty to sync all files. Set filter as a regular expression to limit synchronising files. +Setting key: syncOnlyRegEx +(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns. #### Non-Synchronising files -(RegExp) If this is set, any changes to local and remote files that match this will be skipped. +Setting key: syncIgnoreRegEx +(RegExp) Skip changes to matching normal and hidden files, locally and remotely. #### Maximum file size diff --git a/docs/tips/hidden-file-sync.md b/docs/tips/hidden-file-sync.md index 6b270da2..d761bb73 100644 --- a/docs/tips/hidden-file-sync.md +++ b/docs/tips/hidden-file-sync.md @@ -32,6 +32,10 @@ Enabling Hidden File Sync requires an initialisation direction: Prefer a narrow target list. Device-specific workspace state and another plug-in's credentials are usually poor candidates for cross-device synchronisation. +`Normal Files` → `Synchronising files` limits ordinary files only. Hidden File Sync uses its own `Target patterns`. The `Non-Synchronising files` patterns under `Normal Files` also exclude matching hidden files, so review them alongside the hidden-file selectors. + +If an earlier version skipped hidden files because of the normal-file allow-list, repeat Hidden File Sync initialisation after updating. A routine scan can skip unchanged files which were already checked. Back up the Vault and choose the intended source direction as described above. + Target patterns also control directory traversal. A pattern must therefore match each parent directory as well as the intended files. For example, the following pattern admits the `.obsidian` parent and only its `snippets` subtree: ```text diff --git a/package.json b/package.json index a6ff13d4..4edfd29d 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "test:e2e:obsidian:two-vault-sync": "tsx test/e2e-obsidian/scripts/two-vault-sync.ts", "test:e2e:obsidian:security-seed-reconnect": "tsx test/e2e-obsidian/scripts/security-seed-reconnect.ts", "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", + "test:e2e:obsidian:hidden-file-selectors": "tsx test/e2e-obsidian/scripts/hidden-file-selectors.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", "test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts", "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 34ae100b..17ebe033 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,10 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns.": + "(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns.", + "(RegExp) Skip changes to matching normal and hidden files, locally and remotely.": + "(RegExp) Skip changes to matching normal and hidden files, locally and remotely.", "Preparing follow-up reportâ€Ļ": "Preparing follow-up reportâ€Ļ", "Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.", "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.": diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts new file mode 100644 index 00000000..4ec0f752 --- /dev/null +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.selectors.unit.spec.ts @@ -0,0 +1,127 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("./configureHiddenFileSyncMode.ts", () => ({ configureHiddenFileSyncMode: vi.fn() })); + +import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { readContent } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase"; +import { HiddenFileSync } from "./CmdHiddenFileSync"; + +const hiddenPath = ".obsidian/plugins/example/main.js" as FilePath; +const prefixedPath = `i:${hiddenPath}`; +const body = "module.exports = class ExamplePlugin {};"; +const allowList = "\\.(?:md|png|pdf)$"; +const fixtureOwners: Awaited>[] = []; + +async function hiddenFiles() { + const fixture = await createOptionalFileSyncDatabase(); + fixtureOwners.push(fixture); + fixture.settings.usePluginSync = false; + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { + settings: fixture.settings, + services: { + ...fixture.services, + vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) }, + }, + }, + cacheFileRegExps: new Map(), + cacheCustomisationSyncIgnoredFiles: new Map(), + }); + return { ...fixture, hiddenFileSync }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + for (const fixture of fixtureOwners.splice(0)) await fixture.dispose(); +}); + +describe("issue #1272: Hidden File Sync with normal-file selectors", () => { + it("stores a selected hidden file while the normal-file allow-list is set", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(true); + + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(prefixedPath, body))).resolves.toMatchObject({ + ok: true, + }); + const stored = await fixture.localDatabase.getDBEntry(prefixedPath as FilePath); + expect(stored).not.toBe(false); + if (stored) expect(await readContent(stored)).toBe(body); + }); + + it("reads existing hidden-file content while the normal-file allow-list is set", async () => { + const fixture = await hiddenFiles(); + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(prefixedPath, body))).resolves.toMatchObject({ + ok: true, + }); + const meta = await fixture.localDatabase.getDBEntryMeta(prefixedPath as FilePath); + if (!meta) throw new Error("The seeded hidden-file Metadata is unavailable"); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(true); + + const entry = await fixture.localDatabase.getDBEntryFromMeta(meta); + expect(entry).not.toBe(false); + if (entry) expect(await readContent(entry)).toBe(body); + }); + + it("stores and reads the same hidden file when the normal-file allow-list is empty", async () => { + const fixture = await hiddenFiles(); + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(prefixedPath, body))).resolves.toMatchObject({ + ok: true, + }); + const entry = await fixture.localDatabase.getDBEntry(prefixedPath as FilePath); + expect(entry).not.toBe(false); + if (entry) expect(await readContent(entry)).toBe(body); + }); + + it("continues to apply the normal-file allow-list to ordinary files", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + expect(fixture.localDatabase.isTargetFile("notes/example.md" as FilePath)).toBe(true); + expect(fixture.localDatabase.isTargetFile("notes/example.js" as FilePath)).toBe(false); + expect(fixture.localDatabase.isTargetFile("i:.obsidian/example.md" as FilePath)).toBe(true); + }); + + it("keeps Customisation Sync documents usable with the same normal-file allow-list", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + const path = "ix:device/PLUGIN_MAIN/example%main.js" as FilePath; + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(path, body))).resolves.toMatchObject({ + ok: true, + }); + const entry = await fixture.localDatabase.getDBEntry(path); + expect(entry).not.toBe(false); + if (entry) expect(await readContent(entry)).toBe(body); + }); + + it("keeps Hidden File Sync target and ignore patterns effective", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncInternalFilesTargetPatterns = + "^\\.obsidian/plugins/" as typeof fixture.settings.syncInternalFilesTargetPatterns; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(true); + await expect(fixture.hiddenFileSync.isTargetFile(".obsidian/snippets/example.css" as FilePath)).resolves.toBe( + false + ); + fixture.settings.syncInternalFilesIgnorePatterns = + "main\\.js$" as typeof fixture.settings.syncInternalFilesIgnorePatterns; + await expect(fixture.hiddenFileSync.isTargetFile(hiddenPath)).resolves.toBe(false); + }); + + it("compatibility: applies the normal-file ignore-list to hidden documents", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncOnlyRegEx = allowList as typeof fixture.settings.syncOnlyRegEx; + fixture.settings.syncIgnoreRegEx = "^\\.obsidian/" as typeof fixture.settings.syncIgnoreRegEx; + expect(fixture.localDatabase.isTargetFile(prefixedPath as FilePath)).toBe(false); + }); + + it("rejects hidden documents when Hidden File Sync is disabled", async () => { + const fixture = await hiddenFiles(); + fixture.settings.syncInternalFiles = false; + expect(fixture.localDatabase.isTargetFile(prefixedPath as FilePath)).toBe(false); + }); +}); diff --git a/src/modules/features/SettingDialogue/PaneSelector.ts b/src/modules/features/SettingDialogue/PaneSelector.ts index bfd7c0b1..783d6e99 100644 --- a/src/modules/features/SettingDialogue/PaneSelector.ts +++ b/src/modules/features/SettingDialogue/PaneSelector.ts @@ -13,7 +13,7 @@ export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme .setName($msg("Synchronising files")) .setDesc( $msg( - "(RegExp) Empty to sync all files. Set filter as a regular expression to limit synchronising files." + "(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns." ) ); const syncFilesControl = mount(MultipleRegExpControl, { @@ -32,9 +32,7 @@ export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme const nonSyncFilesSetting = new Setting(paneEl) .setName($msg("Non-Synchronising files")) - .setDesc( - $msg("(RegExp) If this is set, any changes to local and remote files that match this will be skipped.") - ); + .setDesc($msg("(RegExp) Skip changes to matching normal and hidden files, locally and remotely.")); const nonSyncFilesControl = mount(MultipleRegExpControl, { target: nonSyncFilesSetting.controlEl, diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..3e3093e7 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -248,6 +248,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. +`test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path. + `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. diff --git a/test/e2e-obsidian/scripts/hidden-file-selectors.ts b/test/e2e-obsidian/scripts/hidden-file-selectors.ts new file mode 100644 index 00000000..8686daa7 --- /dev/null +++ b/test/e2e-obsidian/scripts/hidden-file-selectors.ts @@ -0,0 +1,202 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { fileURLToPath } from "node:url"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +const hiddenRoot = ".obsidian/livesync-selector-example"; +const hiddenFiles = { + [`${hiddenRoot}/main.js`]: "console.log('hidden fixture');\n", + [`${hiddenRoot}/styles.css`]: ".hidden-fixture { color: blue; }\n", + [`${hiddenRoot}/settings.json`]: '{"fixture":true}\n', +}; +const controlRoot = ".obsidian/livesync-selector-control"; +const controlFiles = { + [`${controlRoot}/main.js`]: "console.log('control fixture');\n", + [`${controlRoot}/styles.css`]: ".control-fixture { color: red; }\n", + [`${controlRoot}/settings.json`]: '{"control":true}\n', +}; +const allowList = "\\.(?:md|png|pdf)$"; +const coreCode = "const core=app.plugins.plugins['obsidian-livesync'].core;"; +const hiddenCode = `${coreCode}const addOn=core.getAddOn('HiddenFileSync');`; + +type Observation = { name: string; expected: unknown; actual: unknown; passed: boolean }; + +async function writeFixtures(vaultPath: string, files: Record) { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(vaultPath, path)), { recursive: true }); + await writeFile(join(vaultPath, path), content); + } +} + +async function removeFixtures(vaultPath: string, files: Record) { + for (const path of Object.keys(files)) await rm(join(vaultPath, path), { force: true }); +} + +async function matchingFiles(vaultPath: string, files: Record) { + const matches: string[] = []; + for (const [path, content] of Object.entries(files)) { + try { + if ((await readFile(join(vaultPath, path), "utf8")) === content) matches.push(path); + } catch (error) { + if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error; + } + } + return matches.sort(); +} + +async function main() { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error("The Obsidian CLI is unavailable"); + const cliBinary = cli.binary; + const output = + process.env.E2E_HIDDEN_FILE_SELECTORS_OUTPUT ?? join(tmpdir(), "livesync-hidden-file-selectors.json"); + const observations: Observation[] = []; + const report: Record = { + revision: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), + sourceDiffSha256: createHash("sha256") + .update(new Uint8Array(execFileSync("git", ["diff", "--binary", "HEAD"]))) + .digest("hex"), + scenarioSha256: createHash("sha256") + .update(new Uint8Array(await readFile(fileURLToPath(import.meta.url)))) + .digest("hex"), + commonlibEntrySha256: createHash("sha256") + .update( + new Uint8Array( + await readFile( + "node_modules/@vrtmrz/livesync-commonlib/dist/managers/EntryManager/EntryManagerImpls.js" + ) + ) + ) + .digest("hex"), + bundleSha256: createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"), + commonlibVersion: JSON.parse(await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8")) + .version, + observations, + }; + const vault = await createTemporaryVault("livesync-hidden-file-selectors-"); + let session: ObsidianLiveSyncSession | undefined; + const record = (name: string, actual: unknown, expected: unknown) => { + const observation = { name, expected, actual, passed: isDeepStrictEqual(actual, expected) }; + observations.push(observation); + console.log(JSON.stringify(observation)); + }; + const evaluate = async (code: string): Promise => { + if (!session) throw new Error("The isolated Obsidian session has not started"); + return await evalObsidianJson(cliBinary, `(async()=>{${code}})()`, session.cliEnv, 60_000); + }; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + pluginData: createE2eCouchDbPluginData( + { uri: "http://127.0.0.1:9", username: "", password: "", dbName: "selector-local-only" }, + { periodicReplication: false, syncOnFileOpen: false, useOnlyLocalChunk: true } + ), + }); + report.pluginVersion = session.readiness.pluginVersion; + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + report.runtime = await evaluate(`${coreCode} + core.services.setting.setDeviceAndVaultName('selector-local'); + Object.assign(core.settings,{ + isConfigured:true,usePluginSync:false,usePluginSyncV2:true,syncInternalFiles:false, + autoSweepPlugins:false,autoSweepPluginsPeriodic:false,watchInternalFileChanges:false, + syncOnlyRegEx:${JSON.stringify(allowList)},syncIgnoreRegEx:'' + }); + return JSON.stringify({userAgent:navigator.userAgent,databaseReady:core.localDatabase.isReady}); + `); + + await writeFixtures(vault.path, hiddenFiles); + const uploadCode = (normalAllowList: string, files: Record) => `${hiddenCode} + Object.assign(core.settings,{ + syncInternalFiles:true,syncOnlyRegEx:${JSON.stringify(normalAllowList)}, + syncInternalFilesTargetPatterns:'^\\\\.obsidian(?:$|/livesync-selector-(?:example|control)(?:/|$))',syncInternalFilesIgnorePatterns:'' + }); + const candidates=await addOn.scanInternalFileNames(); + const selected=candidates.filter(path=>${JSON.stringify(Object.keys(files))}.includes(path)); + if(selected.length!==3)throw new Error('Hidden-file fixture selection failed: '+JSON.stringify({selected,target:core.settings.syncInternalFilesTargetPatterns})); + for(const path of selected)await addOn.updateLastProcessedAsActualFile(path); + await addOn.rebuildFromStorage(false,${JSON.stringify(Object.keys(files))},false); + const paths=[]; + for await(const doc of core.localDatabase.findEntries('i:','i;')){ + if(${JSON.stringify(Object.keys(files).map((path) => `i:${path}`))}.includes(doc.path)&&!doc.deleted&&!doc._deleted)paths.push(doc.path.substring(2)); + } + return JSON.stringify(paths.sort()); + `; + const hiddenUpload = await evaluate(uploadCode(allowList, hiddenFiles)); + record( + "#1272: a normal allow-list permits selected hidden uploads", + hiddenUpload, + Object.keys(hiddenFiles).sort() + ); + await writeFixtures(vault.path, controlFiles); + const uploadControl = await evaluate(uploadCode("", controlFiles)); + record( + "#1272 control: clearing the normal allow-list permits hidden uploads", + uploadControl, + Object.keys(controlFiles).sort() + ); + + const pullCode = (normalAllowList: string, files: Record) => `${hiddenCode} + core.settings.syncOnlyRegEx=${JSON.stringify(normalAllowList)}; + const results=[]; + for await(const doc of core.localDatabase.findEntries('i:','i;')){ + if(${JSON.stringify(Object.keys(files).map((path) => `i:${path}`))}.includes(doc.path)&&!doc.deleted&&!doc._deleted){ + results.push(await addOn.extractInternalFileFromDatabase(doc.path.substring(2),true,doc,false)); + } + } + return JSON.stringify(results); + `; + await removeFixtures(vault.path, hiddenFiles); + report.hiddenPullWithAllowList = await evaluate(pullCode(allowList, hiddenFiles)); + record( + "#1272: a normal allow-list permits selected hidden downloads", + await matchingFiles(vault.path, hiddenFiles), + Object.keys(hiddenFiles).sort() + ); + await removeFixtures(vault.path, controlFiles); + report.hiddenPullWithoutAllowList = await evaluate(pullCode("", controlFiles)); + record( + "#1272 control: clearing the normal allow-list permits hidden downloads", + await matchingFiles(vault.path, controlFiles), + Object.keys(controlFiles).sort() + ); + } catch (error) { + report.runtimeError = error instanceof Error ? error.stack : String(error); + if (session) report.processOutput = JSON.stringify(session.app.output()).slice(-4_000); + throw error; + } finally { + try { + await writeFile(output, JSON.stringify(report, null, 2) + "\n"); + console.log(`Verification report: ${output}`); + } finally { + try { + if (session) await session.app.stop(); + } finally { + await vault.dispose(); + } + } + } + if (observations.some((observation) => !observation.passed)) process.exitCode = 1; +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 7f766c67..d7da07c2 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -39,6 +39,7 @@ const focusedScenarios = new Set([ "two-vault-sync", "security-seed-reconnect", "hidden-file-snippet-sync", + "hidden-file-selectors", "customisation-sync", "received-change-readiness", "internal-metadata-doctor", diff --git a/test/fixtures/optionalFileSyncDatabase.ts b/test/fixtures/optionalFileSyncDatabase.ts new file mode 100644 index 00000000..dd3c6590 --- /dev/null +++ b/test/fixtures/optionalFileSyncDatabase.ts @@ -0,0 +1,78 @@ +import { randomUUID } from "node:crypto"; +import PouchDB from "pouchdb-core"; +import MemoryAdapter from "pouchdb-adapter-memory"; +import { vi } from "vitest"; +import { + DEFAULT_SETTINGS, + type EntryDoc, + type SavingEntry, + type FilePathWithPrefix, + type DocumentID, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { createTextBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { createLiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; +import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; + +PouchDB.plugin(MemoryAdapter); + +export function optionalFileEntry(path: string, content: string): SavingEntry { + return { + _id: path as DocumentID, + path: path as FilePathWithPrefix, + ctime: 1_000, + mtime: 2_000, + size: content.length, + type: "plain", + datatype: "plain", + data: createTextBlob(content), + children: [], + eden: {}, + }; +} + +export async function createOptionalFileSyncDatabase() { + const name = `optional-file-sync-${randomUUID()}`; + const database = new PouchDB(name, { adapter: "memory" }); + const settings = { + ...DEFAULT_SETTINGS, + syncInternalFiles: true, + useOnlyLocalChunk: true, + usePluginSync: true, + usePluginSyncV2: true, + pluginSyncExtendedSetting: {}, + }; + const services = { + API: { addLog: vi.fn(), getSystemConfigDir: () => ".obsidian" }, + setting: { currentSettings: () => settings, getDeviceAndVaultName: () => "device" }, + path: { + path2id: async (path: string) => path, + id2path: (id: string, entry?: { path?: string }) => entry?.path ?? id, + getPath: (entry: { path: FilePathWithPrefix }) => entry.path, + }, + context: { events: createLiveSyncEventHub() }, + database: { createPouchDBInstance: () => database }, + databaseEvents: { + onDatabaseInitialisation: async () => true, + onDatabaseHasReady: async () => true, + onCloseDatabase: async () => true, + onUnloadDatabase: async () => true, + }, + replicator: { onCloseActiveReplication: async () => true }, + }; + const localDatabase = new LiveSyncLocalDB(name, { services } as unknown as LiveSyncLocalDBEnv); + if (!(await localDatabase.initializeDatabase())) { + await database.destroy(); + throw new Error("The optional-file test database could not initialise"); + } + return { + database, + localDatabase, + services, + settings, + async dispose() { + localDatabase.offRemoteChunkFetchedHandler?.(); + await localDatabase.managers.teardownManagers(); + await database.destroy(); + }, + }; +} diff --git a/updates.md b/updates.md index ca2e3497..1700a53f 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Hidden File Sync + +#### Fixed + +- We can synchronise selected hidden files while the normal-file allow-list is set. (#1272) + - Normal-file exclusion patterns still apply to hidden files, and the Selector descriptions now explain their scope. + ## 1.0.39 20th October, 2026 From b7101fe9366839885d8180aac1fc9c7358931f09 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 01:11:41 +0000 Subject: [PATCH 31/68] Fix concurrent Customisation Sync catalogue updates --- package.json | 1 + .../CmdConfigSync.catalogue.unit.spec.ts | 298 ++++++++++++++++++ src/features/ConfigSync/CmdConfigSync.ts | 58 ++-- test/e2e-obsidian/README.md | 2 + .../scripts/customisation-catalogue-reload.ts | 220 +++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + test/fixtures/optionalFileSyncDatabase.ts | 78 +++++ updates.md | 7 + 8 files changed, 642 insertions(+), 23 deletions(-) create mode 100644 src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/customisation-catalogue-reload.ts create mode 100644 test/fixtures/optionalFileSyncDatabase.ts diff --git a/package.json b/package.json index a6ff13d4..224d8ef6 100644 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "test:e2e:obsidian:security-seed-reconnect": "tsx test/e2e-obsidian/scripts/security-seed-reconnect.ts", "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", + "test:e2e:obsidian:customisation-catalogue-reload": "tsx test/e2e-obsidian/scripts/customisation-catalogue-reload.ts", "test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts", "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", "test:e2e:obsidian:internal-metadata-migration": "tsx test/e2e-obsidian/scripts/internal-metadata-migration.ts", diff --git a/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts new file mode 100644 index 00000000..06b04919 --- /dev/null +++ b/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts @@ -0,0 +1,298 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + diff_match_patch: class DiffMatchPatch {}, + normalizePath: (path: string) => path, + parseYaml: vi.fn(), + Platform: {}, +})); +vi.mock("./PluginDialogModal.ts", () => ({ PluginDialogModal: class PluginDialogModal {} })); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({ + ConflictResolveModal: class ConflictResolveModal {}, +})); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("@/common/utils.ts", () => ({ + cancelTask: vi.fn(), + EVEN: Symbol("even"), + isCustomisationSyncMetadata: vi.fn(), + isPluginMetadata: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ PeriodicProcessor: class PeriodicProcessor {} })); +vi.mock("@/common/events.ts", () => ({ + EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync", + eventHub: { onEvent: vi.fn() }, +})); +vi.mock("@/common/translation", () => ({ $msg: (message: string) => message })); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ getObsidianCommunityPluginManager: vi.fn() })); + +import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase"; +import { ConfigSync, PluginDataExDisplayV2, pluginManifests } from "./CmdConfigSync"; + +const filenames = ["main.js", "manifest.json", "styles.css"]; +const content: Record = { + "main.js": "module.exports = class ExamplePlugin {};", + "manifest.json": JSON.stringify({ id: "example", name: "Example", version: "1.0.0" }), + "styles.css": ".example { color: red; }", +}; +const fixtureOwners: Awaited>[] = []; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function catalogue() { + const fixture = await createOptionalFileSyncDatabase(); + fixtureOwners.push(fixture); + fixture.settings.syncOnlyRegEx = "\\.(?:md|png|pdf)$" as typeof fixture.settings.syncOnlyRegEx; + const paths = filenames.map((name) => `ix:device/PLUGIN_MAIN/example%${name}` as FilePathWithPrefix); + for (const [index, path] of paths.entries()) { + const wireContent = `fixture-header\u200b\n\u200c${btoa(content[filenames[index]])}`; + await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(path, wireContent))).resolves.toMatchObject({ + ok: true, + }); + } + const gates = new Map(paths.map((path) => [path, deferred()])); + const readEntry = fixture.localDatabase.getDBEntry.bind(fixture.localDatabase); + vi.spyOn(fixture.localDatabase, "getDBEntry").mockImplementation(async (path, ...args) => { + await gates.get(path)?.promise; + return readEntry(path, ...args); + }); + const written = new Map(); + const configSync = Object.create(ConfigSync.prototype) as ConfigSync; + Object.assign(configSync, { + core: { + localDatabase: fixture.localDatabase, + settings: fixture.settings, + services: fixture.services, + storageAccess: { + ensureDir: vi.fn(async () => true), + readHiddenFileText: vi.fn(async (path: string) => written.get(path) ?? ""), + writeHiddenFileAuto: vi.fn(async (path: string, data: string) => { + written.set(path, data); + }), + }, + }, + pluginList: [], + pluginListGeneration: 0, + loadedManifest_mTime: new Map(), + updatingV2Count: 0, + pluginScanProcessor: { clearQueue: vi.fn(), enqueue: vi.fn() }, + _log: vi.fn(), + storeCustomisationFileV2: vi.fn(async () => true), + }); + const updates: Promise[] = []; + const update = configSync.updatePluginListV2.bind(configSync); + vi.spyOn(configSync, "updatePluginListV2").mockImplementation((...args) => { + const promise = update(...args); + updates.push(promise); + return promise; + }); + const release = (order = filenames) => { + for (const name of order) gates.get(`ix:device/PLUGIN_MAIN/example%${name}` as FilePathWithPrefix)!.resolve(); + }; + return { ...fixture, configSync, updates, paths, release, written }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + pluginManifests.clear(); + for (const fixture of fixtureOwners.splice(0)) await fixture.dispose(); +}); + +describe("issue #1271: rebuilding a multi-file Customisation Sync item", () => { + it.each([ + { order: filenames }, + { order: [...filenames].reverse() }, + { order: ["manifest.json", "main.js", "styles.css"] }, + ])("retains all per-file documents when reads are released as $order", async ({ order }) => { + const fixture = await catalogue(); + await fixture.configSync.reloadPluginList(false); + fixture.release(order); + await Promise.all(fixture.updates); + + const documents = await fixture.database.allDocs({ keys: fixture.paths }); + expect(documents.rows).toHaveLength(3); + expect(fixture.configSync.pluginList).toHaveLength(1); + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual( + filenames.map((name) => `plugins/example/${name}`).sort() + ); + }); + + it("applies all three files after the first reload", async () => { + const fixture = await catalogue(); + await fixture.configSync.reloadPluginList(false); + fixture.release(); + await Promise.all(fixture.updates); + + await expect( + fixture.configSync.applyDataV2(fixture.configSync.pluginList[0] as PluginDataExDisplayV2) + ).resolves.toBe(true); + expect([...fixture.written.keys()].sort()).toEqual( + filenames.map((name) => `.obsidian/plugins/example/${name}`).sort() + ); + for (const name of filenames) + expect(fixture.written.get(`.obsidian/plugins/example/${name}`)).toBe(content[name]); + }); + + it("fills an existing row during a second enumeration", async () => { + const fixture = await catalogue(); + await fixture.configSync.reloadPluginList(false); + fixture.release(); + await Promise.all(fixture.updates); + await fixture.configSync.updatePluginList(false); + await Promise.all(fixture.updates); + + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual( + filenames.map((name) => `plugins/example/${name}`).sort() + ); + }); + + it("retains all files when item updates run sequentially", async () => { + const fixture = await catalogue(); + fixture.release(); + for (const path of fixture.paths) await fixture.configSync.updatePluginListV2(false, path); + + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual( + filenames.map((name) => `plugins/example/${name}`).sort() + ); + }); + + it("updates a different item while another item's reads are pending", async () => { + const fixture = await catalogue(); + const otherPath = "ix:device/PLUGIN_MAIN/other%main.js" as FilePathWithPrefix; + await fixture.localDatabase.putDBEntry( + optionalFileEntry(otherPath, `fixture-header\u200b\n\u200c${btoa(content["main.js"])}`) + ); + const pending = fixture.paths.map((path) => fixture.configSync.updatePluginListV2(false, path)); + try { + await fixture.configSync.updatePluginListV2(false, otherPath); + expect(fixture.configSync.pluginList.map((entry) => entry.name)).toEqual(["other"]); + expect(fixture.configSync.updatingV2Count).toBe(3); + } finally { + fixture.release(); + await Promise.all(pending); + } + expect(fixture.configSync.updatingV2Count).toBe(0); + }); + + it("continues updating an item after an earlier read fails", async () => { + const fixture = await catalogue(); + fixture.release(); + vi.spyOn(fixture.configSync, "createPluginDataExFileV2").mockRejectedValueOnce(new Error("Read failed")); + const failed = fixture.configSync.updatePluginListV2(false, fixture.paths[0]); + const next = fixture.configSync.updatePluginListV2(false, fixture.paths[1]); + + await expect(failed).rejects.toThrow("Read failed"); + await next; + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename)).toEqual([ + "plugins/example/manifest.json", + ]); + expect(fixture.configSync.updatingV2Count).toBe(0); + }); + + it.each(["read", "file update"])("reload discards cached files while an earlier %s is pending", async (phase) => { + const fixture = await catalogue(); + fixture.release(); + for (const path of fixture.paths) await fixture.configSync.updatePluginListV2(false, path); + const earlierRead = deferred(); + const freshRead = deferred(); + const entered = deferred(); + const createFile = fixture.configSync.createPluginDataExFileV2.bind(fixture.configSync); + let reads = 0; + vi.spyOn(fixture.configSync, "createPluginDataExFileV2").mockImplementation(async (path) => { + if (reads++ === 0) { + if (phase === "read") { + entered.resolve(); + await earlierRead.promise; + } + } else { + await freshRead.promise; + } + return createFile(path); + }); + if (phase === "file update") { + const entry = fixture.configSync.pluginList[0] as PluginDataExDisplayV2; + const setFile = entry.setFile.bind(entry); + vi.spyOn(entry, "setFile").mockImplementationOnce(async (file) => { + entered.resolve(); + await earlierRead.promise; + await setFile(file); + }); + } + const pending = fixture.configSync.updatePluginListV2(false, fixture.paths[0]); + try { + await entered.promise; + for (const path of fixture.paths.slice(1)) await fixture.database.remove(await fixture.database.get(path)); + await fixture.configSync.reloadPluginList(false); + earlierRead.resolve(); + await pending; + } finally { + earlierRead.resolve(); + freshRead.resolve(); + await Promise.all(fixture.updates); + } + expect(fixture.configSync.pluginList[0].files.map((file) => file.filename)).toEqual([ + "plugins/example/main.js", + ]); + }); + + it("clearing the disabled feature prevents pending updates from repopulating the list", async () => { + const fixture = await catalogue(); + const pending = fixture.paths.map((path) => fixture.configSync.updatePluginListV2(false, path)); + fixture.settings.usePluginSync = false; + await fixture.configSync.updatePluginList(false); + fixture.release(); + await Promise.all(pending); + + expect(fixture.configSync.pluginList).toEqual([]); + expect(fixture.configSync.updatingV2Count).toBe(0); + }); + + it("a superseded enumeration cannot reintroduce a deleted item", async () => { + const fixture = await catalogue(); + const obsoletePath = "ix:device/PLUGIN_MAIN/obsolete%main.js" as FilePathWithPrefix; + await fixture.localDatabase.putDBEntry( + optionalFileEntry(obsoletePath, `fixture-header\u200b\n\u200c${btoa(content["main.js"])}`) + ); + const enumerate = fixture.localDatabase.findEntries.bind(fixture.localDatabase); + type EnumeratedEntry = ReturnType extends AsyncGenerator ? Entry : never; + const obsoleteDocs: EnumeratedEntry[] = []; + for await (const doc of enumerate(obsoletePath, `${obsoletePath}\u{10ffff}`, { include_docs: true })) { + obsoleteDocs.push(doc); + } + const entered = deferred(); + const release = deferred(); + let scans = 0; + vi.spyOn(fixture.localDatabase, "findEntries").mockImplementation((...args) => { + if (scans++ !== 0) return enumerate(...args); + return (async function* () { + entered.resolve(); + await release.promise; + yield* obsoleteDocs; + })(); + }); + const pending = fixture.configSync.updatePluginList(false); + try { + await entered.promise; + await fixture.database.remove(await fixture.database.get(obsoletePath)); + await fixture.configSync.reloadPluginList(false); + } finally { + release.resolve(); + fixture.release(); + await pending; + await Promise.all(fixture.updates); + } + + expect(fixture.configSync.pluginList.map((entry) => entry.name)).toEqual(["example"]); + expect(fixture.configSync.pluginList[0].files).toHaveLength(3); + expect(fixture.configSync.updatingV2Count).toBe(0); + }); +}); diff --git a/src/features/ConfigSync/CmdConfigSync.ts b/src/features/ConfigSync/CmdConfigSync.ts index 0e1412ee..4b07c99f 100644 --- a/src/features/ConfigSync/CmdConfigSync.ts +++ b/src/features/ConfigSync/CmdConfigSync.ts @@ -412,6 +412,7 @@ export class ConfigSync extends LiveSyncCommands { periodicPluginSweepProcessor = new PeriodicProcessor(this.core, async () => await this.scanAllConfigFiles(false)); pluginList: IPluginDataExDisplay[] = []; + private pluginListGeneration = 0; showPluginSyncModal() { if (!this.isThisModuleEnabled()) { return; @@ -532,6 +533,7 @@ export class ConfigSync extends LiveSyncCommands { return Promise.resolve(true); } async reloadPluginList(showMessage: boolean) { + this.pluginListGeneration++; this.pluginList = []; this.loadedManifest_mTime.clear(); pluginList.set(this.pluginList); @@ -761,39 +763,46 @@ export class ConfigSync extends LiveSyncCommands { updatingV2Count = 0; async updatePluginListV2(showMessage: boolean, unifiedFilenameWithKey: FilePathWithPrefix): Promise { + const generation = this.pluginListGeneration; + const isCurrent = () => generation === this.pluginListGeneration && this.isThisModuleEnabled(); try { this.updatingV2Count++; pluginV2Progress.set(this.updatingV2Count); // const unifiedFilenameWithKey = this.filenameWithUnifiedKey(updatedDocumentPath); const { pathV1 } = this.parseUnifiedPath(unifiedFilenameWithKey); - const oldEntry = this.pluginList.find((e) => e.documentPath == pathV1); - let entry: PluginDataExDisplayV2 | undefined = undefined; + await serialized(`config:catalogue:${pathV1}`, async () => { + if (!isCurrent()) return; + const oldEntry = this.pluginList.find((e) => e.documentPath == pathV1); + let entry: PluginDataExDisplayV2 | undefined = undefined; - if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) { - const newEntry = this.createPluginDataFromV2(unifiedFilenameWithKey); - if (newEntry) { - entry = newEntry; + if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) { + const newEntry = this.createPluginDataFromV2(unifiedFilenameWithKey); + if (newEntry) { + entry = newEntry; + } + } else if (oldEntry instanceof PluginDataExDisplayV2) { + entry = oldEntry; } - } else if (oldEntry instanceof PluginDataExDisplayV2) { - entry = oldEntry; - } - if (!entry) return; - const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey); - if (file) { - await entry.setFile(file); - } else { - entry.deleteFile(unifiedFilenameWithKey); - if (entry.files.length == 0) { - this.pluginList = this.pluginList.filter((e) => e.documentPath != pathV1); + if (!entry) return; + const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey); + if (!isCurrent()) return; + if (file) { + await entry.setFile(file); + } else { + entry.deleteFile(unifiedFilenameWithKey); + if (entry.files.length == 0) { + this.pluginList = this.pluginList.filter((e) => e.documentPath != pathV1); + } } - } - const newList = this.pluginList.filter((e) => e.documentPath != entry.documentPath); - newList.push(entry); - this.pluginList = newList; + if (!isCurrent()) return; + const newList = this.pluginList.filter((e) => e.documentPath != entry.documentPath); + newList.push(entry); + this.pluginList = newList; - scheduleTask("updatePluginListV2", 100, () => { - pluginList.set(this.pluginList); + scheduleTask("updatePluginListV2", 100, () => { + pluginList.set(this.pluginList); + }); }); } finally { this.updatingV2Count--; @@ -871,11 +880,13 @@ export class ConfigSync extends LiveSyncCommands { async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise { if (!this.isThisModuleEnabled()) { + this.pluginListGeneration++; this.pluginScanProcessor.clearQueue(); this.pluginList = []; pluginList.set(this.pluginList); return; } + const generation = this.pluginListGeneration; try { this.updatingV2Count++; pluginV2Progress.set(this.updatingV2Count); @@ -888,6 +899,7 @@ export class ConfigSync extends LiveSyncCommands { }) : this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true }); for await (const v of plugins) { + if (generation !== this.pluginListGeneration || !this.isThisModuleEnabled()) return; if (v.deleted || v._deleted) continue; if (v.path.indexOf("%") !== -1) { fireAndForget(() => this.updatePluginListV2(showMessage, v.path)); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..ec4c0f25 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -250,6 +250,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. +`test:e2e:obsidian:customisation-catalogue-reload` verifies that the first Reload lists all three files of a sample plug-in and that Apply restores their contents. It also checks a second enumeration and Apply in one temporary Vault with a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_CUSTOMISATION_CATALOGUE_OUTPUT` to choose the JSON report path. + The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`. diff --git a/test/e2e-obsidian/scripts/customisation-catalogue-reload.ts b/test/e2e-obsidian/scripts/customisation-catalogue-reload.ts new file mode 100644 index 00000000..ec74438d --- /dev/null +++ b/test/e2e-obsidian/scripts/customisation-catalogue-reload.ts @@ -0,0 +1,220 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { fileURLToPath } from "node:url"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +const pluginName = "livesync-catalogue-example"; +const sourceTerm = "catalogue-source"; +const pluginRoot = `.obsidian/plugins/${pluginName}`; +const pluginFiles = { + [`${pluginRoot}/main.js`]: "module.exports = class ExamplePlugin {};\n", + [`${pluginRoot}/manifest.json`]: JSON.stringify({ + id: pluginName, + name: "Catalogue Example", + version: "1.0.0", + minAppVersion: "1.0.0", + description: "An isolated test fixture", + author: "Self-hosted LiveSync", + }), + [`${pluginRoot}/styles.css`]: ".livesync-catalogue-example { color: red; }\n", +}; +const allowList = "\\.(?:md|png|pdf)$"; +const coreCode = "const core=app.plugins.plugins['obsidian-livesync'].core;"; +const configCode = `${coreCode}const addOn=core.getAddOn('ConfigSync');`; +const settleCode = ` + const settle=async()=>{ + await new Promise(resolve=>setTimeout(resolve,100)); + for(let attempt=0;attempt<200;attempt++){ + if(addOn.updatingV2Count===0)return; + await new Promise(resolve=>setTimeout(resolve,25)); + } + throw new Error('Customisation Sync did not settle'); + }; +`; +const sourceRowCode = ` + const row=addOn.pluginList.find(entry=>entry.category==='PLUGIN_MAIN'&&entry.name===${JSON.stringify(pluginName)}&&entry.term===${JSON.stringify(sourceTerm)}); + if(!row)throw new Error('The source catalogue row is missing'); +`; + +type Observation = { name: string; expected: unknown; actual: unknown; passed: boolean }; + +async function writeFixtures(vaultPath: string, files: Record) { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(vaultPath, path)), { recursive: true }); + await writeFile(join(vaultPath, path), content); + } +} + +async function removeFixtures(vaultPath: string, files: Record) { + for (const path of Object.keys(files)) await rm(join(vaultPath, path), { force: true }); +} + +async function matchingFiles(vaultPath: string, files: Record) { + const matches: string[] = []; + for (const [path, content] of Object.entries(files)) { + try { + if ((await readFile(join(vaultPath, path), "utf8")) === content) matches.push(path); + } catch (error) { + if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error; + } + } + return matches.sort(); +} + +async function main() { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error("The Obsidian CLI is unavailable"); + const cliBinary = cli.binary; + const output = + process.env.E2E_CUSTOMISATION_CATALOGUE_OUTPUT ?? join(tmpdir(), "livesync-customisation-catalogue.json"); + const observations: Observation[] = []; + const report: Record = { + revision: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), + sourceDiffSha256: createHash("sha256") + .update(new Uint8Array(execFileSync("git", ["diff", "--binary", "HEAD"]))) + .digest("hex"), + scenarioSha256: createHash("sha256") + .update(new Uint8Array(await readFile(fileURLToPath(import.meta.url)))) + .digest("hex"), + commonlibEntrySha256: createHash("sha256") + .update( + new Uint8Array( + await readFile( + "node_modules/@vrtmrz/livesync-commonlib/dist/managers/EntryManager/EntryManagerImpls.js" + ) + ) + ) + .digest("hex"), + bundleSha256: createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"), + commonlibVersion: JSON.parse(await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8")) + .version, + observations, + }; + const vault = await createTemporaryVault("livesync-customisation-catalogue-"); + let session: ObsidianLiveSyncSession | undefined; + const record = (name: string, actual: unknown, expected: unknown) => { + const observation = { name, expected, actual, passed: isDeepStrictEqual(actual, expected) }; + observations.push(observation); + console.log(JSON.stringify(observation)); + }; + const evaluate = async (code: string): Promise => { + if (!session) throw new Error("The isolated Obsidian session has not started"); + return await evalObsidianJson(cliBinary, `(async()=>{${code}})()`, session.cliEnv, 60_000); + }; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + pluginData: createE2eCouchDbPluginData( + { uri: "http://127.0.0.1:9", username: "", password: "", dbName: "catalogue-local-only" }, + { periodicReplication: false, syncOnFileOpen: false, useOnlyLocalChunk: true } + ), + }); + report.pluginVersion = session.readiness.pluginVersion; + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + report.runtime = await evaluate(`${coreCode} + core.services.setting.setDeviceAndVaultName('catalogue-local'); + Object.assign(core.settings,{ + isConfigured:true,usePluginSync:true,usePluginSyncV2:true,syncInternalFiles:false, + autoSweepPlugins:false,autoSweepPluginsPeriodic:false,watchInternalFileChanges:false, + syncOnlyRegEx:${JSON.stringify(allowList)},syncIgnoreRegEx:'' + }); + return JSON.stringify({userAgent:navigator.userAgent,databaseReady:core.localDatabase.isReady}); + `); + + await writeFixtures(vault.path, pluginFiles); + await evaluate(`${configCode}${settleCode} + for(const path of ${JSON.stringify(Object.keys(pluginFiles))}){ + const stored=await addOn.storeCustomisationFileV2(path,${JSON.stringify(sourceTerm)}); + if(stored===false)throw new Error('Could not seed a Customisation Sync document'); + } + await settle(); + return JSON.stringify({seeded:true}); + `); + const catalogue = await evaluate<{ files: string[]; documents: number }>(`${configCode}${settleCode} + await addOn.reloadPluginList(false); + await settle(); + ${sourceRowCode} + let documents=0; + for await(const doc of core.localDatabase.findEntries('ix:','ix;')){ + if(doc.path.startsWith(${JSON.stringify(`ix:${sourceTerm}/PLUGIN_MAIN/${pluginName}%`)})&&!doc.deleted&&!doc._deleted)documents++; + } + return JSON.stringify({files:row.files.map(file=>file.filename).sort(),documents}); + `); + record("#1271: all source documents remain in the database", catalogue.documents, 3); + record( + "#1271: one reload lists all files", + catalogue.files, + Object.keys(pluginFiles) + .map((path) => path.substring(".obsidian/".length)) + .sort() + ); + await removeFixtures(vault.path, pluginFiles); + const firstApply = await evaluate( + `${configCode}${sourceRowCode}return JSON.stringify({applied:await addOn.applyDataV2(row)});` + ); + report.firstApply = firstApply; + record( + "#1271: first Apply restores all file contents", + await matchingFiles(vault.path, pluginFiles), + Object.keys(pluginFiles).sort() + ); + + const secondPass = await evaluate(`${configCode}${settleCode} + await addOn.updatePluginList(false);await settle();${sourceRowCode} + return JSON.stringify(row.files.map(file=>file.filename).sort()); + `); + record( + "#1271 control: a second pass lists all files", + secondPass, + Object.keys(pluginFiles) + .map((path) => path.substring(".obsidian/".length)) + .sort() + ); + await removeFixtures(vault.path, pluginFiles); + await evaluate(`${configCode}${sourceRowCode}return JSON.stringify({applied:await addOn.applyDataV2(row)});`); + record( + "#1271 control: Apply after a second pass restores all contents", + await matchingFiles(vault.path, pluginFiles), + Object.keys(pluginFiles).sort() + ); + } catch (error) { + report.runtimeError = error instanceof Error ? error.stack : String(error); + if (session) report.processOutput = JSON.stringify(session.app.output()).slice(-4_000); + throw error; + } finally { + try { + await writeFile(output, JSON.stringify(report, null, 2) + "\n"); + console.log(`Verification report: ${output}`); + } finally { + try { + if (session) await session.app.stop(); + } finally { + await vault.dispose(); + } + } + } + if (observations.some((observation) => !observation.passed)) process.exitCode = 1; +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 7f766c67..54534944 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -40,6 +40,7 @@ const focusedScenarios = new Set([ "security-seed-reconnect", "hidden-file-snippet-sync", "customisation-sync", + "customisation-catalogue-reload", "received-change-readiness", "internal-metadata-doctor", "setting-markdown-export", diff --git a/test/fixtures/optionalFileSyncDatabase.ts b/test/fixtures/optionalFileSyncDatabase.ts new file mode 100644 index 00000000..dd3c6590 --- /dev/null +++ b/test/fixtures/optionalFileSyncDatabase.ts @@ -0,0 +1,78 @@ +import { randomUUID } from "node:crypto"; +import PouchDB from "pouchdb-core"; +import MemoryAdapter from "pouchdb-adapter-memory"; +import { vi } from "vitest"; +import { + DEFAULT_SETTINGS, + type EntryDoc, + type SavingEntry, + type FilePathWithPrefix, + type DocumentID, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { createTextBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { createLiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; +import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; + +PouchDB.plugin(MemoryAdapter); + +export function optionalFileEntry(path: string, content: string): SavingEntry { + return { + _id: path as DocumentID, + path: path as FilePathWithPrefix, + ctime: 1_000, + mtime: 2_000, + size: content.length, + type: "plain", + datatype: "plain", + data: createTextBlob(content), + children: [], + eden: {}, + }; +} + +export async function createOptionalFileSyncDatabase() { + const name = `optional-file-sync-${randomUUID()}`; + const database = new PouchDB(name, { adapter: "memory" }); + const settings = { + ...DEFAULT_SETTINGS, + syncInternalFiles: true, + useOnlyLocalChunk: true, + usePluginSync: true, + usePluginSyncV2: true, + pluginSyncExtendedSetting: {}, + }; + const services = { + API: { addLog: vi.fn(), getSystemConfigDir: () => ".obsidian" }, + setting: { currentSettings: () => settings, getDeviceAndVaultName: () => "device" }, + path: { + path2id: async (path: string) => path, + id2path: (id: string, entry?: { path?: string }) => entry?.path ?? id, + getPath: (entry: { path: FilePathWithPrefix }) => entry.path, + }, + context: { events: createLiveSyncEventHub() }, + database: { createPouchDBInstance: () => database }, + databaseEvents: { + onDatabaseInitialisation: async () => true, + onDatabaseHasReady: async () => true, + onCloseDatabase: async () => true, + onUnloadDatabase: async () => true, + }, + replicator: { onCloseActiveReplication: async () => true }, + }; + const localDatabase = new LiveSyncLocalDB(name, { services } as unknown as LiveSyncLocalDBEnv); + if (!(await localDatabase.initializeDatabase())) { + await database.destroy(); + throw new Error("The optional-file test database could not initialise"); + } + return { + database, + localDatabase, + services, + settings, + async dispose() { + localDatabase.offRemoteChunkFetchedHandler?.(); + await localDatabase.managers.teardownManagers(); + await database.destroy(); + }, + }; +} diff --git a/updates.md b/updates.md index ca2e3497..9a2da6ee 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Customisation Sync + +#### Fixed + +- We can reload the customisation list and apply every file in a multi-file item on the first attempt. (#1271) + - Reloading or clearing the list also prevents earlier pending updates from restoring removed files and items. + ## 1.0.39 20th October, 2026 From d1e4c340543b936c2508fdfe4ab8fdba79a7f5a3 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 09:12:02 +0000 Subject: [PATCH 32/68] Prepare consistent Chunk sizes and fresh Fast Fetch integration --- docs/settings.md | 27 +- src/common/replicatorProviders.ts | 5 + src/common/replicatorProviders.unit.spec.ts | 16 +- src/common/replicatorResources.unit.spec.ts | 30 + src/common/replicatorResources/index.ts | 1 + .../initialFetchFastForward.ts | 17 + .../features/SetupManager.unit.spec.ts | 19 +- .../scripts/couchdb-manual-setup-workflow.ts | 72 ++- updates.md | 16 + utils/chunk-size-benchmark/README.md | 72 +++ .../cloudant-json-boundary.mjs | 108 ++++ utils/chunk-size-benchmark/cloudant-limit.mjs | 50 ++ utils/chunk-size-benchmark/compose.yml | 30 + utils/chunk-size-benchmark/corpus.mjs | 310 +++++++++ .../datachannel-metrics.cjs | 23 + utils/chunk-size-benchmark/local.mjs | 596 ++++++++++++++++++ utils/chunk-size-benchmark/p2p.mjs | 159 +++++ .../rabin-window.test.mjs | 20 + utils/chunk-size-benchmark/remote.mjs | 210 ++++++ utils/chunk-size-benchmark/report.mjs | 85 +++ utils/chunk-size-benchmark/summarise.mjs | 83 +++ utils/chunk-size-benchmark/verify-results.mjs | 82 +++ utils/flyio/generate_setupuri.test.ts | 18 +- utils/setup/generate_setup_uri.test.ts | 7 +- 24 files changed, 2042 insertions(+), 14 deletions(-) create mode 100644 src/common/replicatorResources/initialFetchFastForward.ts create mode 100644 utils/chunk-size-benchmark/README.md create mode 100644 utils/chunk-size-benchmark/cloudant-json-boundary.mjs create mode 100644 utils/chunk-size-benchmark/cloudant-limit.mjs create mode 100644 utils/chunk-size-benchmark/compose.yml create mode 100644 utils/chunk-size-benchmark/corpus.mjs create mode 100644 utils/chunk-size-benchmark/datachannel-metrics.cjs create mode 100644 utils/chunk-size-benchmark/local.mjs create mode 100644 utils/chunk-size-benchmark/p2p.mjs create mode 100644 utils/chunk-size-benchmark/rabin-window.test.mjs create mode 100644 utils/chunk-size-benchmark/remote.mjs create mode 100644 utils/chunk-size-benchmark/report.mjs create mode 100644 utils/chunk-size-benchmark/summarise.mjs create mode 100644 utils/chunk-size-benchmark/verify-results.mjs diff --git a/docs/settings.md b/docs/settings.md index 0885589c..96684b21 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -936,6 +936,18 @@ Setting key: hashCacheMaxAmount Setting key: customChunkSize +New setup uses 0 for every remote type. With V3 Rabin–Karp, this gives binary +Chunks a 100 KiB maximum, a 25 KiB minimum, and an approximate 25 KiB candidate +gap. The candidate gap is not the mean Chunk length. Text below 4 MiB retains +adaptive smaller boundaries; large text uses the binary thresholds with +UTF-8-safe cuts. + +Larger values can reduce document counts for mostly immutable binary files, +but can increase the new content stored after an edit. Existing explicit and +agreed values are retained. Updating the plug-in does not re-chunk the Vault; +editing, forced writes, Chunk recreation, or a full Rebuild can create new +boundaries while existing content remains readable. + #### Chunk Splitter Setting key: chunkSplitterVersion @@ -957,7 +969,10 @@ If this enabled, chunks will be split into semantically meaningful segments. Not #### Fetch chunks on demand Setting key: readChunksOnline -(ex. Read chunks online) If this option is enabled, LiveSync reads chunks online directly instead of replicating them locally. Increasing Custom chunk size is recommended. +If enabled, this plug-in obtains missing Chunks when they are needed instead +of including all Chunks in ordinary pull replication. New setup retains the +same Chunk-size recommendation of 0. This setting does not re-chunk content +already stored by another device. #### Batch size of on-demand fetching @@ -1010,12 +1025,18 @@ The setting remains disabled by default because the measured storage and transfe #### Batch size Setting key: batch_size -Number of changes to sync at a time. Defaults to 50. Minimum is 2. +Number of changes in an ordinary CouchDB replication batch. The schema default +is 25, and the minimum is 2. Changing this value alone preserves the +replication checkpoint. Larger counts can reduce round trips, but do not bound +the final request size or memory use. #### Batch limit Setting key: batches_limit -Number of batches to process at a time. Defaults to 40. Minimum is 2. This along with batch size controls how many docs are kept in memory at a time. +Number of changes batches which PouchDB can read ahead. The schema default is +25, and the minimum is 2. This does not create parallel body transfers; +PouchDB processes one current replication batch at a time. It does not control +the separate P2P or Object Storage transfer units. #### Use timeouts instead of heartbeats diff --git a/src/common/replicatorProviders.ts b/src/common/replicatorProviders.ts index 845d8c2e..1e62c25d 100644 --- a/src/common/replicatorProviders.ts +++ b/src/common/replicatorProviders.ts @@ -29,6 +29,7 @@ import { } from "./replicatorConfigurationIdentity"; import { createCouchDBConnectionProbeFactory, + createCouchDBInitialFetchFastForwardResourceFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, createCouchDBSynchronisationInformationResourceFactory, @@ -125,6 +126,9 @@ export function createCentralReplicatorProviderDefinitions( [REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION]: supportedCapability( createCouchDBSynchronisationInformationResourceFactory(host) ), + [REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]: supportedCapability( + createCouchDBInitialFetchFastForwardResourceFactory(host) + ), }, centralRemoteAdministration: COUCHDB_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY, userInitiatedOneShot: supportedCapability(couchDBUserInitiatedOneShot), @@ -151,6 +155,7 @@ export function createCentralReplicatorProviderDefinitions( createObjectStorageSecuritySeedResourceFactory(host) ), [REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION]: CAPABILITY_NOT_APPLICABLE, + [REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]: CAPABILITY_NOT_APPLICABLE, }, centralRemoteAdministration: OBJECT_STORAGE_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY, userInitiatedOneShot: supportedCapability(objectStorageUserInitiatedOneShot), diff --git a/src/common/replicatorProviders.unit.spec.ts b/src/common/replicatorProviders.unit.spec.ts index 54a2c441..01a7d6be 100644 --- a/src/common/replicatorProviders.unit.spec.ts +++ b/src/common/replicatorProviders.unit.spec.ts @@ -44,7 +44,15 @@ describe("central Replicator provider definitions", () => { it("keeps the retained remote-resource catalogue bounded", () => { expect .soft(Object.values(REMOTE_RESOURCE_KINDS).sort()) - .toEqual(["connection", "preferred-tweak", "security-seed", "synchronisation-information"].sort()); + .toEqual( + [ + "connection", + "preferred-tweak", + "security-seed", + "synchronisation-information", + "initial-fetch-fast-forward", + ].sort() + ); }); it("lets Journal prepare its own fresh Security Seed while CouchDB uses central preparation", () => { @@ -114,10 +122,16 @@ describe("central Replicator provider definitions", () => { expect(couchResources?.[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION].kind).toBe( CAPABILITY_SUPPORT_KINDS.SUPPORTED ); + expect(couchResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]?.kind).toBe( + CAPABILITY_SUPPORT_KINDS.SUPPORTED + ); expect(objectResources?.[REMOTE_RESOURCE_KINDS.SECURITY_SEED].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(objectResources?.[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION].kind).toBe( CAPABILITY_SUPPORT_KINDS.NOT_APPLICABLE ); + expect(objectResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]?.kind).toBe( + CAPABILITY_SUPPORT_KINDS.NOT_APPLICABLE + ); expect(couchAdministration?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(objectAdministration?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect("activeRemoteReads" in definitions.get(REMOTE_COUCHDB)!).toBe(false); diff --git a/src/common/replicatorResources.unit.spec.ts b/src/common/replicatorResources.unit.spec.ts index 2215a48a..07550057 100644 --- a/src/common/replicatorResources.unit.spec.ts +++ b/src/common/replicatorResources.unit.spec.ts @@ -5,6 +5,7 @@ import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings"; const mocks = vi.hoisted(() => ({ logger: vi.fn(), + initialFetchFastForwardResource: undefined as unknown, couchDB: [] as Array<{ host: unknown; isMobile: ReturnType; @@ -12,6 +13,7 @@ const mocks = vi.hoisted(() => ({ getRemoteStatus: ReturnType; getRemotePreferredTweakValues: ReturnType; getReplicationPBKDF2Salt: ReturnType; + createInitialFetchFastForwardResource: ReturnType; closeReplication: ReturnType; }>, objectStorage: [] as Array<{ @@ -42,6 +44,7 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicato getRemoteStatus = vi.fn(); getRemotePreferredTweakValues = vi.fn(); getReplicationPBKDF2Salt = vi.fn(); + createInitialFetchFastForwardResource = vi.fn(() => mocks.initialFetchFastForwardResource); closeReplication = vi.fn(); constructor(host: unknown) { @@ -69,6 +72,7 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe import { createCouchDBConnectionProbeFactory, + createCouchDBInitialFetchFastForwardResourceFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, createCouchDBSynchronisationInformationResourceFactory, @@ -92,6 +96,7 @@ describe("replicator probe factories", () => { beforeEach(() => { mocks.couchDB.length = 0; mocks.objectStorage.length = 0; + mocks.initialFetchFastForwardResource = undefined; mocks.checkSyncInfo.mockReset().mockResolvedValue(true); mocks.logger.mockClear(); }); @@ -370,4 +375,29 @@ describe("replicator probe factories", () => { await resource.dispose(); expect(replicator.closeReplication).toHaveBeenCalledOnce(); }); + + it("creates an Initial Fetch Fast Forward resource from a settings snapshot and leaves ownership to the resource", async () => { + const host = { name: "host" }; + const settings = createSettings(); + const snapshot = { ...settings }; + const prepare = vi.fn(async () => undefined); + const dispose = vi.fn(async () => undefined); + const resource = { prepare, dispose }; + const factory = createCouchDBInitialFetchFastForwardResourceFactory(host as never); + mocks.initialFetchFastForwardResource = resource; + + const result = await factory(settings); + const replicator = mocks.couchDB[0]; + + settings.couchDB_URI = "https://changed.example.test"; + expect(replicator.createInitialFetchFastForwardResource).toHaveBeenCalledWith(snapshot); + expect(replicator.createInitialFetchFastForwardResource.mock.calls[0][0]).not.toBe(settings); + expect(replicator.host).toBe(host); + expect(result).toBe(resource); + + await expect(result.prepare()).resolves.toBeUndefined(); + await result.dispose(); + expect(prepare).toHaveBeenCalledOnce(); + expect(dispose).toHaveBeenCalledOnce(); + }); }); diff --git a/src/common/replicatorResources/index.ts b/src/common/replicatorResources/index.ts index 9888e85c..36a55545 100644 --- a/src/common/replicatorResources/index.ts +++ b/src/common/replicatorResources/index.ts @@ -14,3 +14,4 @@ export { type SecuritySeedResourceHost, } from "./securitySeed"; export { createCouchDBSynchronisationInformationResourceFactory } from "./synchronisationInformation"; +export { createCouchDBInitialFetchFastForwardResourceFactory } from "./initialFetchFastForward"; diff --git a/src/common/replicatorResources/initialFetchFastForward.ts b/src/common/replicatorResources/initialFetchFastForward.ts new file mode 100644 index 00000000..601f2d1d --- /dev/null +++ b/src/common/replicatorResources/initialFetchFastForward.ts @@ -0,0 +1,17 @@ +import type { InitialFetchFastForwardResourceFactory } from "@vrtmrz/livesync-commonlib/replication"; +import { + LiveSyncCouchDBReplicator, + type LiveSyncCouchDBReplicatorEnv, +} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; +import { snapshotRemoteSettings } from "./shared"; + +/** Create a private CouchDB Replicator for a fresh Fast Fetch checkpoint handoff. */ +export function createCouchDBInitialFetchFastForwardResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): InitialFetchFastForwardResourceFactory { + return (setting) => { + const snapshot = snapshotRemoteSettings(setting); + const replicator = new LiveSyncCouchDBReplicator(host); + return Promise.resolve(replicator.createInitialFetchFastForwardResource(snapshot)); + }; +} diff --git a/src/modules/features/SetupManager.unit.spec.ts b/src/modules/features/SetupManager.unit.spec.ts index c8dfe4ec..713b75f9 100644 --- a/src/modules/features/SetupManager.unit.spec.ts +++ b/src/modules/features/SetupManager.unit.spec.ts @@ -172,10 +172,12 @@ describe("SetupManager", () => { const { manager, dialogManager } = createSetupManager(); dialogManager.openWithExplicitCancel.mockResolvedValueOnce("configure-manually"); const configureManually = vi.spyOn(manager, "onConfigureManually").mockResolvedValue(true); + const newVaultSettings = createNewVaultSettings(); await manager.onOnboard(UserMode.NewUser); - expect(configureManually).toHaveBeenCalledWith(createNewVaultSettings(), UserMode.NewUser); + expect(newVaultSettings.customChunkSize).toBe(0); + expect(configureManually).toHaveBeenCalledWith(newVaultSettings, UserMode.NewUser); }); it("compatibility: normalises imported flat remote settings from a Setup URI before applying", async () => { @@ -487,8 +489,15 @@ describe("SetupManager", () => { expect(applySettings).not.toHaveBeenCalled(); }); - it("preserves modern profiles, display names, and the active selection from a Setup URI", async () => { + it.each([ + { importedSize: 0, expectedSize: 0 }, + { importedSize: undefined, expectedSize: 60 }, + ])("preserves modern Setup URI profiles and adopts the supplied chunk size $importedSize", async ({ importedSize, expectedSize }) => { const { manager, setting, dialogManager } = createSetupManager(); + setting.settings = { + ...setting.currentSettings(), + customChunkSize: 60, + }; const imported = { ...DEFAULT_SETTINGS, remoteConfigurations: { @@ -507,6 +516,8 @@ describe("SetupManager", () => { }, activeConfigurationId: "archive", } as ObsidianLiveSyncSettings; + if (importedSize === undefined) delete (imported as Partial).customChunkSize; + else imported.customChunkSize = importedSize; dialogManager.openWithExplicitCancel .mockResolvedValueOnce(imported) .mockResolvedValueOnce("compatible-existing-user"); @@ -514,6 +525,7 @@ describe("SetupManager", () => { await manager.onUseSetupURI(UserMode.Unknown, "mock-config://modern-settings"); const current = setting.currentSettings(); + expect(current.customChunkSize).toBe(expectedSize); expect(current.remoteConfigurations).toEqual(imported.remoteConfigurations); expect(current.activeConfigurationId).toBe("archive"); expect(Object.keys(current.remoteConfigurations).some((id) => id.startsWith("legacy-"))).toBe(false); @@ -523,6 +535,7 @@ describe("SetupManager", () => { const { manager, setting, dialogManager } = createSetupManager(); setting.settings = { ...setting.currentSettings(), + customChunkSize: 60, isConfigured: true, remoteConfigurations: { existing: { @@ -554,6 +567,7 @@ describe("SetupManager", () => { await manager.onCouchDBManualSetup(UserMode.ExistingUser, setting.currentSettings()); const current = setting.currentSettings(); + expect(current.customChunkSize).toBe(60); expect(current.remoteConfigurations.existing).toBeDefined(); expect(Object.keys(current.remoteConfigurations)).toHaveLength(2); expect(current.activeConfigurationId).not.toBe("existing"); @@ -662,6 +676,7 @@ describe("SetupManager", () => { await manager.onP2PManualSetup(UserMode.NewUser, setting.currentSettings()); const current = setting.currentSettings(); + expect(current.customChunkSize).toBe(0); expect(Object.keys(current.remoteConfigurations)).toHaveLength(1); expect(current.activeConfigurationId).not.toBe(""); expect(current.P2P_ActiveRemoteConfigurationId).toBe(current.activeConfigurationId); diff --git a/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts b/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts index 37e5e22f..34aa2e5e 100644 --- a/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts +++ b/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts @@ -47,6 +47,16 @@ const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_SETUP_URI_TIMEOUT_MS ?? 3000 const e2eePassphrase = `%${randomBytes(24).toString("base64url")}`; const notePath = "E2E/manual-couchdb/from-first-device.md"; const noteContent = "# Manual CouchDB setup\n\nThis note was sent by the manually configured first device.\n"; +const binaryPath = "E2E/manual-couchdb/content-boundaries.png"; +const binaryBlockSize = 32 * 1024; +function binaryFixture(): Uint8Array { + const bytes = new Uint8Array(binaryBlockSize * 8); + for (let index = 0; index < 8; index++) { + bytes[index * binaryBlockSize] = index + 2; + bytes[(index + 1) * binaryBlockSize - 1] = 1; + } + return bytes; +} const returnNotePath = "E2E/manual-couchdb/from-second-device.md"; const returnNoteContent = "# Manual CouchDB return journey\n\nThis note returned through a Setup URI generated by the first device.\n"; @@ -395,6 +405,32 @@ async function waitForVaultFile( throw new Error(`Timed out waiting for ${path}. Last content:\n${lastContent}`); } +async function waitForBinaryFile(vault: TemporaryVault): Promise { + const expected = Buffer.from(binaryFixture()); + const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ?? 30000); + while (Date.now() < deadline) { + try { + if ((await readFile(join(vault.path, binaryPath))).equals(expected)) return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error("The fetched binary did not match its exact original bytes."); +} + +async function readChunkInventory(context: RunnerContext, session: ObsidianLiveSyncSession): Promise { + return await evalObsidianJson( + context.cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + const rows=(await core.localDatabase.allDocsRaw({include_docs:true})).rows; + return JSON.stringify(rows.filter(row=>row.doc?.type==='leaf').map(row=>row.id).sort()); + })()`, + session.cliEnv + ); +} + async function waitForRemoteEntry(context: RunnerContext, entry: { id: string; children: string[] }): Promise { await waitForCouchDbDocs(context.couchDb, context.dbName, (docs) => { const ids = new Set(docs.map((doc) => doc._id)); @@ -681,6 +717,8 @@ async function main(): Promise { process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true" ? randomBytes(32).toString("base64url") : undefined; let firstEntryId: string | undefined; let returnEntryId: string | undefined; + let binaryRevision: string | undefined; + let chunkInventory: string[] = []; try { await assertCouchDbReachable(couchDb); @@ -712,10 +750,34 @@ async function main(): Promise { } await writeNoteViaObsidian(context.cliBinary, session.cliEnv, notePath, noteContent); + const size = await evalObsidianJson( + context.cliBinary, + `JSON.stringify(app.plugins.plugins['obsidian-livesync'].core.settings.customChunkSize)`, + session.cliEnv + ); + assertEqual(size, 0, "New manual setup did not retain the common Chunk-size recommendation."); + await evalObsidianJson( + context.cliBinary, + `(async()=>{ + const bytes=new Uint8Array(${binaryBlockSize}*8); + for(let index=0;index<8;index++){ + bytes[index*${binaryBlockSize}]=index+2; + bytes[(index+1)*${binaryBlockSize}-1]=1; + } + await app.vault.createBinary(${JSON.stringify(binaryPath)},bytes.buffer); + return JSON.stringify(true); + })()`, + session.cliEnv + ); + const binaryEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, binaryPath); + assertEqual(binaryEntry.children.length, 8, "V3 size 0 did not use the expected content boundaries."); + binaryRevision = binaryEntry.rev; const entry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, notePath); firstEntryId = entry.id; await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForRemoteEntry(context, entry); + await waitForRemoteEntry(context, binaryEntry); + chunkInventory = await readChunkInventory(context, session); } catch (error) { await captureFailure(session, "first-device"); throw error; @@ -729,6 +791,13 @@ async function main(): Promise { session = await startUnconfiguredSession(context, vaultA); try { await assertRestoredE2EEPassphrase(session, context.cliBinary); + const unchanged = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, binaryPath); + assertEqual(unchanged.rev, binaryRevision, "Restart rewrote unchanged binary Metadata."); + assertEqual( + JSON.stringify(await readChunkInventory(context, session)), + JSON.stringify(chunkInventory), + "Restart created Chunks for unchanged Vault files." + ); await scheduleRemoteOverwrite(session.remoteDebuggingPort); screenshots.push(await confirmRebuild(session.remoteDebuggingPort, e2eeRebuildCaptures)); screenshots.push( @@ -775,6 +844,7 @@ async function main(): Promise { await assertPersistedE2EE(vaultB, independentIdSource); await pushLocalChanges(context.cliBinary, session.cliEnv); await waitForVaultFile(vaultB, notePath, noteContent); + await waitForBinaryFile(vaultB); if (independentIdSource) { assertEqual( await readDocumentId(context.cliBinary, session.cliEnv, notePath), @@ -815,7 +885,7 @@ async function main(): Promise { } console.log( - `Manual CouchDB onboarding created and tested its database, generated a second-device Setup URI, and completed a bidirectional note round-trip. Screenshots: ${screenshots.join(", ")}` + `Manual CouchDB onboarding retained size 0, used binary content boundaries, preserved unchanged revisions and Chunks across restart, fetched exact binary bytes, and completed a bidirectional note round-trip. Screenshots: ${screenshots.join(", ")}` ); } finally { await stopTrackedSessions(context).catch((error: unknown) => { diff --git a/updates.md b/updates.md index ca2e3497..2bf14f77 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,22 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We can reuse unchanged content more reliably after edits with the V3 Rabin–Karp chunk splitter. + - Corrected rolling-window processing lets boundary detection recover after an insertion. Existing content remains readable, and installing this update does not trigger a full re-chunk of our Vaults. + - Chunk-size values 0 and 1 now use content-defined binary boundaries below their maximum lengths, improving reuse after an insertion instead of producing fixed-length pieces. + - Editing or explicitly rewriting a file, recreating its chunks, or rebuilding from our files can produce different chunk boundaries and additional chunks. Older and updated devices can retain different chunk layouts; this affects reuse and storage, not the file contents. +- We can preserve all conflict branches, including deleted branches, during Fast Setup. + - A fresh, complete Fast Fetch can also hand its verified progress to ordinary synchronisation before file watching resumes, reducing the first subsequent identifier scan. Existing resume cursors and On Demand chunk retrieval retain their existing behaviour. + +#### Changed + +- New setup uses the same Chunk-size recommendation of 0 across remote types, and Setting Doctor no longer recommends increasing every self-hosted V3 configuration to 60. + - Existing explicit and agreed values are retained. Smaller Chunks favour reuse during editing, but increase document counts and can make first uploads and large rebuilds slower. Mostly immutable binary Vaults can still benefit from a larger explicit value. + ## 1.0.39 20th October, 2026 diff --git a/utils/chunk-size-benchmark/README.md b/utils/chunk-size-benchmark/README.md new file mode 100644 index 00000000..f3db9ebd --- /dev/null +++ b/utils/chunk-size-benchmark/README.md @@ -0,0 +1,72 @@ +# Chunk-size research harness + +These scripts support the October 2026 Chunk-size investigation. The decision, +measurements, and their limits are recorded in Commonlib's +`docs/adr/2026_10_chunking_and_transport_batching.md` and its evidence companion. +They are research tools with experiment-specific assertions, separate from +the maintained unit and integration suites. + +The original experiments used Commonlib 0.1.44 and unpublished candidates with +the same version label. `local.mjs` checks that version explicitly. Record the +installed package's source revision and tarball digest when comparing results; +the version label alone does not identify the tested implementation. Scripts +which import internal `dist` paths also depend on that package layout. + +## Local measurements + +Install the exact Commonlib candidate to assess, then run from the repository +root with Node.js 24: + +```sh +NODE_OPTIONS=--max-old-space-size=3072 node utils/chunk-size-benchmark/local.mjs --output .bench/chunk-size --sizes 0,1,4 --repeats 1 +node --test utils/chunk-size-benchmark/rabin-window.test.mjs +``` + +The local run generates a deterministic corpus from repository Markdown, +images, and seeded binary data. It records initial, overwrite, and insertion +stages, Chunk identifiers, reconstructed content checks, and timings. Keep +the generated corpus when comparing revisions: source Markdown may change. +Generated Vaults, logs, and results belong under `.bench/` and are not source +files for the normal test runner. + +## Transport measurements + +Build the CLI and start the disposable fixtures required for the chosen +transport. These use loopback ports and public benchmark-only credentials. +The scripts create and delete their own test databases and buckets. +GNU `time`, Docker Compose, and a Linux CLI runtime are required. + +```sh +NODE_OPTIONS=--max-old-space-size=3072 npm run build --workspace self-hosted-livesync-cli +docker compose -p livesync-chunk-benchmark -f utils/chunk-size-benchmark/compose.yml up -d couchdb object-storage +node utils/chunk-size-benchmark/remote.mjs --output .bench/chunk-size --sizes 0,1,4 --repeats 1 --rtt-ms 0 +docker compose -p livesync-chunk-benchmark -f utils/chunk-size-benchmark/compose.yml down --volumes +``` + +`remote.mjs` accepts `--providers couchdb`, `--providers s3`, or both, plus +`--corpus`, `--source-cli`, and `--target-cli`. Its HTTP proxy adds the requested +RTT across the request and response legs. This models per-request delay rather +than network bandwidth, packet loss, or a production service. + +For P2P, start the `relay` fixture instead and run `p2p.mjs` with the same +output, corpus, size, repeat, and CLI options, omitting `--rtt-ms`. Its +DataChannel counters measure application payloads and record the selected +connection path. An archived CLI entry point needs all of its generated +CommonJS chunks beside it. Run transports sequentially and stop the named +Compose project afterwards. + +## Historical analysis helpers + +- `summarise.mjs`, `report.mjs`, and `verify-results.mjs` consume the original + result layouts. The verifier requires that experiment's exact variants, + configuration counts, and files; it is not a general mixed-version verifier. +- `cloudant-limit.mjs` temporarily changes the disposable CouchDB document + limit and restores it afterwards. `cloudant-json-boundary.mjs` probes final + serialised document sizes against the same fixture. These scripts model + published Cloudant constraints; they do not connect to IBM Cloudant or + certify its acceptance of a Vault. + +Some helpers retain historical output-directory and size defaults. Supply +explicit paths and inspect their assertions before reusing them for a new +experiment. The recorded implementation matrix checks file content in both +writer directions; revision-conflict retention has separate Commonlib tests. diff --git a/utils/chunk-size-benchmark/cloudant-json-boundary.mjs b/utils/chunk-size-benchmark/cloudant-json-boundary.mjs new file mode 100644 index 00000000..e7c8ad99 --- /dev/null +++ b/utils/chunk-size-benchmark/cloudant-json-boundary.mjs @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const packageRoot = dirname(fileURLToPath(import.meta.resolve('@vrtmrz/livesync-commonlib/package.json'))); +const { splitPiecesRabinKarp } = await import(pathToFileURL(join(packageRoot, 'dist/string_and_binary/chunks.js')).href); +const { getConfiguredFunctionsForEncryption } = await import(pathToFileURL(join(packageRoot, 'dist/pouchdb/encryption.js')).href); +const { compressDoc } = await import(pathToFileURL(join(packageRoot, 'dist/pouchdb/compress.js')).href); +const output = resolve(process.argv[2] || '.bench/rabin-window-fixed/cloudant-json-boundary'); +await fs.mkdir(output, { recursive: true }); +const uri = 'http://127.0.0.1:15994'; +const headers = { Authorization: `Basic ${Buffer.from('bench:benchmark-only-password').toString('base64')}`, 'Content-Type': 'application/json' }; +const configPath = '/_node/_local/_config/couchdb/max_document_size'; +const previousResponse = await fetch(uri + configPath, { headers }); +assert(previousResponse.ok || previousResponse.status === 404); +const previous = previousResponse.ok ? await previousResponse.json() : undefined; +const requestConfigPath = '/_node/_local/_config/chttpd/max_http_request_size'; +const previousRequestResponse = await fetch(uri + requestConfigPath, { headers }); +assert(previousRequestResponse.ok || previousRequestResponse.status === 404); +const previousRequestLimit = previousRequestResponse.ok ? await previousRequestResponse.json() : undefined; +const database = `chunkbench-json-budget-${Date.now()}`; +const inputs = { + binary: new Blob([new Uint8Array(4 * 1024 * 1024)], { type: 'application/octet-stream' }), + nulText: new Blob(['\0'.repeat(4 * 1024 * 1024)], { type: 'text/plain' }), + escapedText: new Blob(['\\'.repeat(4 * 1024 * 1024)], { type: 'text/plain' }), +}; +const cases = [ + { kind: 'binary', customChunkSize: 4, encryption: 'v1', compression: false, expectedAccepted: true }, + { kind: 'binary', customChunkSize: 4, encryption: 'v2', compression: false, expectedAccepted: true }, + { kind: 'binary', customChunkSize: 5, encryption: 'v2', compression: false, expectedAccepted: false }, + { kind: 'nulText', customChunkSize: 0, encryption: 'none', compression: false, expectedAccepted: true }, + { kind: 'nulText', customChunkSize: 4, encryption: 'none', compression: false }, + { kind: 'escapedText', customChunkSize: 4, encryption: 'none', compression: false }, + { kind: 'nulText', customChunkSize: 4, encryption: 'v2', compression: false, expectedAccepted: true }, + { kind: 'nulText', customChunkSize: 4, encryption: 'none', compression: true, expectedAccepted: true }, +]; +const records = []; +const record = { scope: 'Synthetic leaf-document probes using the installed splitter, compression, encryption, and JSON serialisation, sent to the owned CouchDB fixture with a 1,000,000-byte document limit and 64,000,000-byte request limit. This is not an IBM Cloudant service measurement or a whole-file synchronisation test. CouchDB 3.5.0 estimates document size from parsed strings and does not count JSON escapes; Cloudant internal size accounting has not been verified.', source: 'https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-limits', sizeCalculationSource: 'https://github.com/apache/couchdb/blob/3.5.0/src/couch/src/couch_ejson_size.erl', documentLimitBytes: 1000000, requestLimitBytes: 64000000, previousDocumentLimit: previous ?? null, previousRequestLimit: previousRequestLimit ?? null, cases: records, batches: [] }; +try { + assert((await fetch(uri + configPath, { method: 'PUT', headers, body: JSON.stringify('1000000') })).ok); + assert((await fetch(uri + requestConfigPath, { method: 'PUT', headers, body: JSON.stringify('64000000') })).ok); + assert((await fetch(`${uri}/${database}`, { method: 'PUT', headers })).ok); + for (const condition of cases) { + const input = inputs[condition.kind]; + const chunks = await splitPiecesRabinKarp(input, 102400 * (condition.customChunkSize + 1), condition.kind !== 'binary', 20); + let largest = ''; + for await (const chunk of chunks()) if (chunk.length > largest.length) largest = chunk; + const id = `h:+${createHash('sha256').update(JSON.stringify(condition)).digest('hex')}`; + let doc = { _id: id, type: 'leaf', data: largest }; + if (condition.compression) doc = await compressDoc(doc); + if (condition.encryption !== 'none') { + const { incoming, outgoing } = getConfiguredFunctionsForEncryption('public-json-budget-probe-passphrase', false, false, async () => new Uint8Array(16), condition.encryption); + const encrypted = await incoming(doc); + assert.equal(encrypted.e_, true); + const decoded = await outgoing(structuredClone(encrypted)); + assert.equal(decoded.data, doc.data); + doc = encrypted; + } + doc = { ...doc, _rev: `1-${'0'.repeat(32)}`, _revisions: { start: 1, ids: ['0'.repeat(32)] } }; + const jsonBytes = Buffer.byteLength(JSON.stringify(doc)); + const body = JSON.stringify({ docs: [doc], new_edits: false }); + const response = await fetch(`${uri}/${database}/_bulk_docs`, { method: 'POST', headers, body }); + const result = await response.json(); + const accepted = response.ok && Array.isArray(result) && result.every(entry => !entry.error); + const measurement = { ...condition, largestPieceUtf8Bytes: Buffer.byteLength(largest), jsonBytes, jsonBytesUnderLimit: jsonBytes < record.documentLimitBytes, requestBodyBytes: Buffer.byteLength(body), status: response.status, accepted, response: result }; + records.push(measurement); + await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2)); + if (condition.expectedAccepted !== undefined) assert.equal(accepted, condition.expectedAccepted, JSON.stringify(measurement)); + assert(Buffer.byteLength(body) < record.requestLimitBytes); + console.log(JSON.stringify({ ...measurement, response: undefined })); + } + const batchInput = new Uint8Array(512000 * 25); + for (let index = 0; index < 25; index++) batchInput[(index + 1) * 512000 - 1] = index + 1; + const splitBatch = await splitPiecesRabinKarp(new Blob([batchInput], { type: 'text/plain' }), 512000, true, 20); + const batchPieces = []; + for await (const piece of splitBatch()) batchPieces.push(piece); + assert.equal(batchPieces.length, 25); + assert(batchPieces.every(piece => Buffer.byteLength(piece) === 512000)); + assert.equal(new Set(batchPieces).size, 25); + for (const count of [25, 20]) { + const docs = batchPieces.slice(0, count).map((piece, index) => ({ + _id: `h:+${createHash('sha256').update(`escaped-batch-${count}-${index}`).digest('hex')}`, + type: 'leaf', data: piece, + _rev: `1-${'0'.repeat(32)}`, _revisions: { start: 1, ids: ['0'.repeat(32)] }, + })); + const body = JSON.stringify({ docs, new_edits: false }); + const response = await fetch(`${uri}/${database}/_bulk_docs`, { method: 'POST', headers, body }); + const result = await response.json(); + const accepted = response.ok && Array.isArray(result) && result.every(entry => !entry.error); + const measurement = { count, rawBytesPerLeaf: 512000, requestBodyBytes: Buffer.byteLength(body), largestDocumentJsonBytes: Math.max(...docs.map(doc => Buffer.byteLength(JSON.stringify(doc)))), status: response.status, accepted, response: result }; + record.batches.push(measurement); + await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2)); + assert.equal(accepted, count === 20, JSON.stringify(measurement)); + if (count === 25) assert.equal(response.status, 413); + console.log(JSON.stringify({ ...measurement, response: undefined })); + } +} finally { + await fetch(`${uri}/${database}`, { method: 'DELETE', headers }); + const restored = await fetch(uri + configPath, previous === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previous) }); + record.configurationRestored = restored.ok; + const requestRestored = await fetch(uri + requestConfigPath, previousRequestLimit === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previousRequestLimit) }); + record.requestConfigurationRestored = requestRestored.ok; + await fs.writeFile(join(output, 'results.json'), JSON.stringify(record, null, 2)); + assert(record.configurationRestored); + assert(record.requestConfigurationRestored); +} diff --git a/utils/chunk-size-benchmark/cloudant-limit.mjs b/utils/chunk-size-benchmark/cloudant-limit.mjs new file mode 100644 index 00000000..b88d8fed --- /dev/null +++ b/utils/chunk-size-benchmark/cloudant-limit.mjs @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const options = new Map(); +for (let index = 2; index < process.argv.length; index += 2) { + assert(['--output', '--corpus'].includes(process.argv[index]), `Unknown option: ${process.argv[index]}`); + assert(process.argv[index + 1], `Missing value for ${process.argv[index]}`); + options.set(process.argv[index], process.argv[index + 1]); +} +const output = resolve(options.get('--output') || '.bench/chunk-size-cloudant-model'); +const corpusPath = resolve(options.get('--corpus') || '.bench/chunk-size/corpus/manifest.json'); +const configUri = 'http://127.0.0.1:15994/_node/_local/_config/couchdb/max_document_size'; +const headers = { Authorization: `Basic ${Buffer.from('bench:benchmark-only-password').toString('base64')}`, 'Content-Type': 'application/json' }; +const previousResponse = await fetch(configUri, { headers }); +assert(previousResponse.ok || previousResponse.status === 404); +const previous = previousResponse.ok ? await previousResponse.json() : undefined; +const manifest = JSON.parse(await fs.readFile(corpusPath, 'utf8')); +manifest.stages = manifest.stages.filter((stage) => stage.name === 'initial'); +for (const stage of manifest.stages) stage.directory = resolve(dirname(corpusPath), stage.directory); +await fs.mkdir(output, { recursive: true }); +const initialManifest = join(output, 'initial-only-manifest.json'); +await fs.writeFile(initialManifest, JSON.stringify(manifest, null, 2)); +const record = { scope: 'CouchDB with a 1,000,000-byte document limit, modelling the published Cloudant document-size constraint. The IBM Cloudant service itself is not measured.', source: 'https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-limits', limitBytes: 1000000, publishedRequestLimitBytes: 64000000, variant: process.env.LIVESYNC_BENCH_VARIANT || 'unlabelled', previousConfiguredValue: previous ?? null }; +try { + const configured = await fetch(configUri, { method: 'PUT', headers, body: JSON.stringify(String(record.limitBytes)) }); + assert(configured.ok, `Could not set fixture document limit: ${configured.status}`); + const child = spawn(process.execPath, [join(root, 'utils/chunk-size-benchmark/remote.mjs'), '--output', output, '--corpus', initialManifest, '--providers', 'couchdb', '--sizes', '4,10', '--repeats', '1'], { + cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=3072' }, + }); + let log = ''; child.stdout.on('data', (data) => { log += data; process.stdout.write(data); }); child.stderr.on('data', (data) => { log += data; }); + record.exitCode = await new Promise((accept, reject) => { child.once('error', reject); child.once('close', accept); }); + await fs.writeFile(join(output, 'run.log'), log); + const measured = JSON.parse(await fs.readFile(join(output, 'remote/results-rtt0.json'), 'utf8')); + record.cases = measured.results.map((entry) => ({ name: entry.name, customChunkSize: entry.customChunkSize, complete: entry.complete, error: entry.error, stages: entry.stages.map((stage) => ({ name: stage.name, largestDocumentJsonBytes: stage.remote.largestDocumentJsonBytes, maxUploadRequestBodyBytes: stage.upload.http.maxRequestBodyBytes, verifiedFiles: stage.verifiedFiles })) })); + assert(record.cases.some((entry) => entry.customChunkSize === 4 && entry.complete), 'The 500 KiB candidate must complete under the modelled limit'); + assert(record.cases.some((entry) => entry.customChunkSize === 10 && !entry.complete), 'The 1,100 KiB candidate must hit the modelled document limit'); + const rejectionLog = await fs.readFile(join(output, 'remote/couchdb-c10-r1/logs/initial-upload.txt'), 'utf8'); + record.sizeRejectionObserved = /413|document_too_large|document.*too large|too_large|request entity too large/iu.test(rejectionLog); + assert(record.sizeRejectionObserved, 'The rejection log must identify the document-size limit'); +} finally { + const restored = await fetch(configUri, previous === undefined ? { method: 'DELETE', headers } : { method: 'PUT', headers, body: JSON.stringify(previous) }); + record.configurationRestored = restored.ok; + await fs.writeFile(join(output, 'constraint-result.json'), JSON.stringify(record, null, 2)); + assert(record.configurationRestored, 'Fixture document limit restoration failed'); +} +console.log(JSON.stringify(record)); diff --git a/utils/chunk-size-benchmark/compose.yml b/utils/chunk-size-benchmark/compose.yml new file mode 100644 index 00000000..275fa271 --- /dev/null +++ b/utils/chunk-size-benchmark/compose.yml @@ -0,0 +1,30 @@ +services: + relay: + image: ghcr.io/hoytech/strfry@sha256:36f1886d185a88ca57c66ebe52e6e9e8428dac2486eea0a5d50ff934f18b60c3 + command: ["--config", "/etc/strfry.conf", "relay"] + volumes: + - ../../test/fixtures/p2p-relay/strfry.conf:/etc/strfry.conf:ro + tmpfs: + - /app/strfry-db:rw,size=128m,mode=1777 + ports: + - "127.0.0.1:14094:7777" + mem_limit: 256m + couchdb: + image: couchdb:3.5.0@sha256:1c0a30a54535377bc9ae23f42efb1ca4e92abe796f78ec98b768ac1d5874cc1b + environment: + COUCHDB_USER: bench + COUCHDB_PASSWORD: benchmark-only-password + COUCHDB_SINGLE_NODE: "true" + ports: + - "127.0.0.1:15994:5984" + mem_limit: 768m + object-storage: + image: rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035 + environment: + RUSTFS_ACCESS_KEY: bench-access-key + RUSTFS_SECRET_KEY: benchmark-only-secret-key + RUSTFS_CONSOLE_ENABLE: "false" + command: ["/data"] + ports: + - "127.0.0.1:19094:9000" + mem_limit: 1g diff --git a/utils/chunk-size-benchmark/corpus.mjs b/utils/chunk-size-benchmark/corpus.mjs new file mode 100644 index 00000000..8c7232f6 --- /dev/null +++ b/utils/chunk-size-benchmark/corpus.mjs @@ -0,0 +1,310 @@ +import { createHash } from "node:crypto"; +import { copyFile, mkdir, readFile, readdir, stat, utimes, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const DEFAULT_REPO_ROOT = path.resolve(MODULE_DIR, "../.."); +const SEED = 0x4c53594e; +const KIB = 1024; +const MIB = 1024 * KIB; +const BINARY_BYTES = 8 * MIB; +const SMALL_MARKDOWN_BYTES = 512 * KIB; +const LARGE_MARKDOWN_BYTES = 5 * MIB; +const BINARY_EDIT_BYTES = 4 * KIB; +const INITIAL_MTIME = new Date("2020-01-01T00:00:00.000Z"); +const STAGE_MTIMES = { + overwrite: new Date("2020-01-02T00:00:00.000Z"), + insert: new Date("2020-01-03T00:00:00.000Z"), +}; + +const MARKDOWN_FILES = [ + "README.md", + "devs.md", + "docs/settings.md", + "docs/troubleshooting.md", + "docs/quick_setup.md", + "docs/datastructure.md", + "docs/p2p.md", + "docs/tips/fast-setup.md", + "docs/design_docs/intention_of_chunks.md", + "docs/specs_data_compression.md", +]; + +const SNIPPET_FILES = [ + "docs/terms.md", + "docs/glossary.md", + "docs/quick_setup_ja.md", + "docs/design_docs/unicode_path_identity.md", + "docs/settings.md", +]; + +const CORPUS_PATHS = { + markdownSmall: "benchmark/generated-small.md", + markdownLarge: "benchmark/generated-large.md", + binary: "benchmark/random-8m.bin", + cloudantImage: "assets/cloudant-setup.png", + setupImage: "assets/quick-setup-first-setup-uri.png", +}; + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function createPrng(seed) { + let state = seed >>> 0; + return () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return state >>> 0; + }; +} + +function makeRandomBytes(length, seed) { + const next = createPrng(seed); + const output = Buffer.allocUnsafe(length); + for (let offset = 0; offset < length; offset += 4) { + const value = next(); + output[offset] = value & 0xff; + if (offset + 1 < length) output[offset + 1] = (value >>> 8) & 0xff; + if (offset + 2 < length) output[offset + 2] = (value >>> 16) & 0xff; + if (offset + 3 < length) output[offset + 3] = (value >>> 24) & 0xff; + } + return output; +} + +function extractParagraphs(source) { + const withoutFrontmatter = source.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/u, ""); + return withoutFrontmatter + .replace(/\r\n/gu, "\n") + .split(/\n{2,}/u) + .map((paragraph) => paragraph.trim()) + .filter((paragraph) => { + if (paragraph.length < 80) return false; + if (/^(?:#{1,6}\s|!\[|\|)/u.test(paragraph)) return false; + return true; + }); +} + +async function readSnippetParagraphs(repoRoot) { + const excerpts = []; + for (const file of SNIPPET_FILES) { + const source = await readFile(path.join(repoRoot, file), "utf8"); + for (const paragraph of extractParagraphs(source)) { + const excerpt = Array.from(paragraph).slice(0, 1000).join(""); + excerpts.push({ source: file, text: excerpt }); + } + } + if (excerpts.length === 0) throw new Error("No repository Markdown snippets were available."); + return excerpts; +} + +function buildGeneratedMarkdown({ targetBytes, title, variant, excerpts }) { + const lines = [ + `# ${title}`, + "", + "A deterministic UTF-8 fixture contains varied passages from the project documentation.", + "", + ]; + let byteLength = Buffer.byteLength(lines.join("\n"), "utf8"); + const profiles = ["replication", "metadata", "recovery", "integrity", "configuration", "history", "incremental", "transport"]; + const unicodeLabels = ["rÊsumÊ", "cafÊ", "æ—ĨæœŦčĒž", "中文", "ΔÎŋÎēΚÎŧÎŽ", "ÃĨngstrÃļm", "naïve", "é›Ē", "Îģ", "maÃąana"]; + let record = 0; + while (byteLength < targetBytes) { + const excerpt = excerpts[(record * 7 + variant.length) % excerpts.length]; + const profile = profiles[(record * 3 + variant.length) % profiles.length]; + const unicode = unicodeLabels[(record * 5 + variant.length) % unicodeLabels.length]; + const tag = ((SEED ^ (record * 0x9e3779b1)) >>> 0).toString(16).padStart(8, "0"); + const paragraph = [ + `## Record ${String(record + 1).padStart(6, "0")} ¡ ${profile} ¡ ${variant}`, + "", + excerpt.text, + "", + `Fixture label ${tag} uses ${unicode}; source excerpt: ${excerpt.source}; revision ${record % 97}.`, + "", + ].join("\n"); + lines.push(paragraph); + byteLength += Buffer.byteLength(paragraph, "utf8"); + record += 1; + } + return lines.join("\n"); +} + +function insertNearStart(markdown, paragraph) { + const firstGap = markdown.indexOf("\n\n"); + const introGap = markdown.indexOf("\n\n", firstGap + 2); + if (firstGap < 0 || introGap < 0) throw new Error("Generated Markdown fixture has no introductory paragraph."); + const insertionPoint = introGap + 2; + return `${markdown.slice(0, insertionPoint)}${paragraph}\n\n${markdown.slice(insertionPoint)}`; +} + +async function writeFileWithMtime(directory, relativePath, contents, mtime) { + const outputPath = path.join(directory, relativePath); + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, contents); + await utimes(outputPath, mtime, mtime); +} + +async function copyStage(sourceDirectory, targetDirectory, relativePaths) { + for (const relativePath of relativePaths) { + const sourcePath = path.join(sourceDirectory, relativePath); + const targetPath = path.join(targetDirectory, relativePath); + await mkdir(path.dirname(targetPath), { recursive: true }); + const sourceStat = await stat(sourcePath); + await copyFile(sourcePath, targetPath); + await utimes(targetPath, sourceStat.atime, sourceStat.mtime); + } +} + +async function setMtime(directory, relativePath, mtime) { + const outputPath = path.join(directory, relativePath); + await utimes(outputPath, mtime, mtime); +} + +async function listFiles(directory) { + const files = []; + async function visit(currentDirectory) { + const entries = await readdir(currentDirectory, { withFileTypes: true }); + entries.sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + const fullPath = path.join(currentDirectory, entry.name); + if (entry.isDirectory()) { + await visit(fullPath); + } else if (entry.isFile()) { + files.push(path.relative(directory, fullPath).split(path.sep).join("/")); + } + } + } + await visit(directory); + return files.sort(); +} + +async function describeStage(stageName, stageDirectory, paths) { + const files = []; + for (const relativePath of paths) { + const bytes = await readFile(path.join(stageDirectory, relativePath)); + files.push({ path: relativePath, bytes: bytes.byteLength, sha256: sha256(bytes) }); + } + return { name: stageName, directory: stageName, files }; +} + +export async function createCorpus(outDir, repoRoot = DEFAULT_REPO_ROOT) { + const outputDirectory = path.resolve(outDir); + const sourceRoot = path.resolve(repoRoot); + const initialDirectory = path.join(outputDirectory, "initial"); + const overwriteDirectory = path.join(outputDirectory, "overwrite"); + const insertDirectory = path.join(outputDirectory, "insert"); + + await mkdir(outputDirectory, { recursive: true }); + + const initialContents = new Map(); + for (const relativePath of MARKDOWN_FILES) { + initialContents.set(relativePath, await readFile(path.join(sourceRoot, relativePath))); + } + + const excerpts = await readSnippetParagraphs(sourceRoot); + initialContents.set( + CORPUS_PATHS.markdownSmall, + Buffer.from( + buildGeneratedMarkdown({ + targetBytes: SMALL_MARKDOWN_BYTES, + title: "Generated Markdown ¡ 512 KiB", + variant: "small", + excerpts, + }), + "utf8", + ), + ); + initialContents.set( + CORPUS_PATHS.markdownLarge, + Buffer.from( + buildGeneratedMarkdown({ + targetBytes: LARGE_MARKDOWN_BYTES, + title: "Generated Markdown ¡ 5 MiB", + variant: "large", + excerpts, + }), + "utf8", + ), + ); + initialContents.set(CORPUS_PATHS.binary, makeRandomBytes(BINARY_BYTES, SEED)); + initialContents.set(CORPUS_PATHS.cloudantImage, await readFile(path.join(sourceRoot, "instruction_images/cloudant_1.png"))); + initialContents.set(CORPUS_PATHS.setupImage, await readFile(path.join(sourceRoot, "images/quick-setup/guide-quick-setup-first-setup-uri.png"))); + + const relativePaths = [...initialContents.keys()].sort(); + for (const [relativePath, contents] of initialContents) { + await writeFileWithMtime(initialDirectory, relativePath, contents, INITIAL_MTIME); + } + + await copyStage(initialDirectory, overwriteDirectory, relativePaths); + const smallMarkdownPath = path.join(overwriteDirectory, CORPUS_PATHS.markdownSmall); + const smallMarkdownBefore = await readFile(smallMarkdownPath, "utf8"); + const originalIntro = "A deterministic UTF-8 fixture contains varied passages from the project documentation."; + const changedIntro = "A deterministic UTF-8 fixture records a short content update using varied project passages."; + if (!smallMarkdownBefore.includes(originalIntro)) throw new Error("Could not locate the generated Markdown paragraph to overwrite."); + await writeFile(smallMarkdownPath, smallMarkdownBefore.replace(originalIntro, changedIntro)); + const binaryPath = path.join(overwriteDirectory, CORPUS_PATHS.binary); + const initialBinary = await readFile(binaryPath); + const binaryOverwriteOffset = Math.floor(initialBinary.byteLength * 0.25); + const overwrittenBinary = Buffer.from(initialBinary); + makeRandomBytes(BINARY_EDIT_BYTES, (SEED ^ 0x00a55a5a) >>> 0).copy(overwrittenBinary, binaryOverwriteOffset); + await writeFile(binaryPath, overwrittenBinary); + const alternatePng = await readFile(path.join(sourceRoot, "instruction_images/cloudant_2.png")); + await writeFile(path.join(overwriteDirectory, CORPUS_PATHS.cloudantImage), alternatePng); + for (const relativePath of [CORPUS_PATHS.markdownSmall, CORPUS_PATHS.binary, CORPUS_PATHS.cloudantImage]) { + await setMtime(overwriteDirectory, relativePath, STAGE_MTIMES.overwrite); + } + + await copyStage(overwriteDirectory, insertDirectory, relativePaths); + const insertedBinaryPath = path.join(insertDirectory, CORPUS_PATHS.binary); + const overwrittenBinaryForInsert = await readFile(insertedBinaryPath); + const binaryInsertOffset = Math.floor(overwrittenBinaryForInsert.byteLength * 0.25); + const insertedBinary = Buffer.concat([ + overwrittenBinaryForInsert.subarray(0, binaryInsertOffset), + makeRandomBytes(BINARY_EDIT_BYTES, (SEED ^ 0x005aa5a5) >>> 0), + overwrittenBinaryForInsert.subarray(binaryInsertOffset), + ]); + await writeFile(insertedBinaryPath, insertedBinary); + + const insertedParagraphs = [ + [CORPUS_PATHS.markdownSmall, "The insert stage adds record 0047, profile delta-small, while retaining Unicode labels such as æ—ĨæœŦčĒž and cafÊ."], + [CORPUS_PATHS.markdownLarge, "The insert stage adds record 0913, profile delta-large, while retaining Unicode labels such as 中文 and ΔÎŋÎēΚÎŧÎŽ."], + ]; + for (const [relativePath, paragraph] of insertedParagraphs) { + const markdownPath = path.join(insertDirectory, relativePath); + const markdown = await readFile(markdownPath, "utf8"); + await writeFile(markdownPath, insertNearStart(markdown, paragraph)); + } + for (const relativePath of [CORPUS_PATHS.binary, ...insertedParagraphs.map(([relativePath]) => relativePath)]) { + await setMtime(insertDirectory, relativePath, STAGE_MTIMES.insert); + } + + const actualPaths = await listFiles(initialDirectory); + for (const directory of [overwriteDirectory, insertDirectory]) { + const stagePaths = await listFiles(directory); + if (JSON.stringify(stagePaths) !== JSON.stringify(actualPaths)) { + throw new Error(`Corpus stage ${path.basename(directory)} has different relative paths.`); + } + } + + const sourceForPath = new Map(MARKDOWN_FILES.map((file) => [file, `repo:${file}`])); + sourceForPath.set(CORPUS_PATHS.markdownSmall, "generated:utf8-markdown-512KiB-from-repository-snippets"); + sourceForPath.set(CORPUS_PATHS.markdownLarge, "generated:utf8-markdown-5MiB-from-repository-snippets"); + sourceForPath.set(CORPUS_PATHS.binary, `generated:seeded-high-entropy-binary-seed-${SEED.toString(16)}`); + sourceForPath.set(CORPUS_PATHS.cloudantImage, "repo:instruction_images/cloudant_1.png"); + sourceForPath.set(CORPUS_PATHS.setupImage, "repo:images/quick-setup/guide-quick-setup-first-setup-uri.png"); + + const files = actualPaths.map((relativePath) => { + const kind = relativePath.endsWith(".md") ? "markdown" : "binary"; + return { path: relativePath, kind, source: sourceForPath.get(relativePath) }; + }); + const stages = [ + await describeStage("initial", initialDirectory, actualPaths), + await describeStage("overwrite", overwriteDirectory, actualPaths), + await describeStage("insert", insertDirectory, actualPaths), + ]; + const manifest = { seed: SEED, files, stages }; + await writeFile(path.join(outputDirectory, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + return manifest; +} diff --git a/utils/chunk-size-benchmark/datachannel-metrics.cjs b/utils/chunk-size-benchmark/datachannel-metrics.cjs new file mode 100644 index 00000000..fe13ed31 --- /dev/null +++ b/utils/chunk-size-benchmark/datachannel-metrics.cjs @@ -0,0 +1,23 @@ +const { writeFileSync } = require('node:fs'); +const { RTCDataChannel } = require('werift'); + +const output = process.env.LIVESYNC_BENCH_DATACHANNEL_JSON; +if (!output) throw new Error('LIVESYNC_BENCH_DATACHANNEL_JSON is required'); +const channels = new Set(); +const originalSend = RTCDataChannel.prototype.send; +RTCDataChannel.prototype.send = function (...args) { + channels.add(this); + return originalSend.apply(this, args); +}; +process.once('exit', () => { + const rows = [...channels].map((channel) => ({ label: channel.label, payloadBytesSent: channel.bytesSent, messagesSent: channel.messagesSent })); + const cpu = process.cpuUsage(); + writeFileSync(output, JSON.stringify({ + source: 'werift 0.24.4 RTCDataChannel counters; observed channels through a benchmark-only preload', + scope: 'Application DataChannel payload; includes application retransmissions, excludes SCTP/DTLS/UDP/IP overhead, transport retransmissions, and relay signalling', + payloadBytesSent: rows.reduce((sum, row) => sum + row.payloadBytesSent, 0), + messagesSent: rows.reduce((sum, row) => sum + row.messagesSent, 0), + processUsage: { userCpuMs: cpu.user / 1000, systemCpuMs: cpu.system / 1000, maxResidentSetKiB: process.resourceUsage().maxRSS }, + channels: rows, + }, null, 2)); +}); diff --git a/utils/chunk-size-benchmark/local.mjs b/utils/chunk-size-benchmark/local.mjs new file mode 100644 index 00000000..6c5eeb98 --- /dev/null +++ b/utils/chunk-size-benchmark/local.mjs @@ -0,0 +1,596 @@ +import { createHash } from "node:crypto"; +import { readFile, mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { HashManager } from "@vrtmrz/livesync-commonlib/hashing"; +import { createCorpus } from "./corpus.mjs"; + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const PROJECT_ROOT = path.resolve(MODULE_DIR, "../.."); +const DEFAULT_SIZES = [0, 1, 4, 10, 20, 40, 60]; +const DEFAULT_REPEATS = 3; +const ID_DERIVATION_KEY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const LEGACY_PASSPHRASE = "public-local-chunk-size-benchmark-fixture"; +const CHUNK_MANAGER_WITHOUT_CACHE = { + getCachedChunk: () => undefined, + getChunkIDFromCache: () => false, +}; + +const COMMONLIB_PACKAGE_JSON = fileURLToPath(import.meta.resolve("@vrtmrz/livesync-commonlib/package.json")); +const COMMONLIB_ROOT = path.dirname(COMMONLIB_PACKAGE_JSON); +const COMMONLIB_PACKAGE = JSON.parse(await readFile(COMMONLIB_PACKAGE_JSON, "utf8")); +const { ContentSplitterRabinKarp } = await import( + pathToFileURL(path.join(COMMONLIB_ROOT, "dist/ContentSplitter/ContentSplitterRabinKarp.js")).href +); +const { prepareChunk } = await import( + pathToFileURL(path.join(COMMONLIB_ROOT, "dist/managers/EntryManager/EntryManagerImpls.js")).href +); + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function parseArguments(args) { + const parsed = { output: undefined, sizes: [...DEFAULT_SIZES], repeats: DEFAULT_REPEATS, help: false }; + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + if (argument === "--help" || argument === "-h") { + parsed.help = true; + continue; + } + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${argument}.`); + if (argument === "--output") { + parsed.output = value; + } else if (argument === "--sizes") { + parsed.sizes = value.split(",").map((item) => { + const trimmed = item.trim(); + if (!/^\d+$/u.test(trimmed)) throw new Error(`Invalid customChunkSize value: ${item}`); + const size = Number(trimmed); + if (!Number.isSafeInteger(size) || size > 10000) throw new Error(`customChunkSize is out of range: ${item}`); + return size; + }); + if (parsed.sizes.length === 0 || new Set(parsed.sizes).size !== parsed.sizes.length) { + throw new Error("--sizes must contain one or more distinct integers."); + } + } else if (argument === "--repeats") { + if (!/^\d+$/u.test(value)) throw new Error(`Invalid repeat count: ${value}`); + parsed.repeats = Number(value); + if (!Number.isSafeInteger(parsed.repeats) || parsed.repeats < 1 || parsed.repeats > 100) { + throw new Error("--repeats must be an integer from 1 to 100."); + } + } else { + throw new Error(`Unknown argument: ${argument}`); + } + index += 1; + } + if (!parsed.help && !parsed.output) throw new Error("--output is required."); + return parsed; +} + +function usage() { + return "Usage: node utils/chunk-size-benchmark/local.mjs --output [--sizes 0,1,4,10,20,40,60] [--repeats 3]"; +} + +function settingsFor(customChunkSize, mode) { + return { + chunkSplitterVersion: "v3-rabin-karp", + customChunkSize, + minimumChunkSize: 20, + disableWorkerForGeneratingChunks: true, + processSmallFilesInUIThread: false, + encrypt: true, + E2EEAlgorithm: "v2", + hashAlg: "xxhash64", + passphrase: mode === "legacy" ? LEGACY_PASSPHRASE : "", + idDerivationVersion: mode === "primary" ? 1 : 0, + idDerivationKey: mode === "primary" ? ID_DERIVATION_KEY : "", + enableCompression: false, + useEden: false, + }; +} + +async function createCondition(customChunkSize) { + const primarySettings = settingsFor(customChunkSize, "primary"); + const legacySettings = settingsFor(customChunkSize, "legacy"); + const primarySettingService = { currentSettings: () => primarySettings }; + const legacySettingService = { currentSettings: () => legacySettings }; + const splitter = new ContentSplitterRabinKarp({ settingService: primarySettingService }); + const primaryHashManager = new HashManager({ settingService: primarySettingService }); + const legacyHashManager = new HashManager({ settingService: legacySettingService }); + await Promise.all([splitter.initialised, primaryHashManager.initialise(), legacyHashManager.initialise()]); + return { customChunkSize, splitter, primaryHashManager, legacyHashManager }; +} + +async function warmCondition(condition, file) { + await splitAndHashFile(condition, file, new Map(), new Map()); +} + +async function loadCorpus(manifest, corpusDirectory) { + const filesByPath = new Map(manifest.files.map((file) => [file.path, file])); + const loadedStages = []; + for (const stage of manifest.stages) { + const stageDirectory = path.join(corpusDirectory, stage.directory); + const loadedFiles = []; + for (const expected of stage.files) { + const metadata = filesByPath.get(expected.path); + if (!metadata) throw new Error(`Corpus manifest has no file metadata for ${expected.path}.`); + const data = await readFile(path.join(stageDirectory, expected.path)); + if (data.byteLength !== expected.bytes || sha256(data) !== expected.sha256) { + throw new Error(`Corpus file does not match its manifest: ${stage.name}/${expected.path}`); + } + loadedFiles.push({ ...metadata, expectedSha256: expected.sha256, data }); + } + loadedStages.push({ name: stage.name, files: loadedFiles }); + } + return loadedStages; +} + +function decodeChunk(kind, piece) { + return kind === "markdown" ? Buffer.from(piece, "utf8") : Buffer.from(piece, "base64"); +} + +function assertSameContentForId(idsToPieces, id, piece, label) { + const existing = idsToPieces.get(id); + if (existing !== undefined && existing !== piece) { + throw new Error(`${label} collision: chunk ID ${id} was produced for different content.`); + } + if (existing === undefined) idsToPieces.set(id, piece); +} + +async function splitAndHashFile(condition, file, primaryIdContents, legacyIdContents) { + const blobType = file.kind === "markdown" ? "text/plain" : "application/octet-stream"; + const blob = new Blob([file.data], { type: blobType }); + const entry = { path: file.path, data: blob }; + const primaryChunks = []; + const primaryCpuStart = process.cpuUsage(); + const primaryStart = process.hrtime.bigint(); + const generator = await condition.splitter.splitContent(entry); + for await (const piece of generator) { + const prepared = await prepareChunk( + { hashManager: condition.primaryHashManager, chunkManager: CHUNK_MANAGER_WITHOUT_CACHE }, + piece, + ); + assertSameContentForId(primaryIdContents, prepared.id, piece, "Primary keyed hash"); + primaryChunks.push({ piece, id: prepared.id }); + } + const primaryElapsedMs = Number(process.hrtime.bigint() - primaryStart) / 1e6; + const primaryCpu = process.cpuUsage(primaryCpuStart); + const primaryCpuMs = (primaryCpu.user + primaryCpu.system) / 1000; + + const legacyCpuStart = process.cpuUsage(); + const legacyStart = process.hrtime.bigint(); + for (const chunk of primaryChunks) { + const prepared = await prepareChunk( + { hashManager: condition.legacyHashManager, chunkManager: CHUNK_MANAGER_WITHOUT_CACHE }, + chunk.piece, + ); + assertSameContentForId(legacyIdContents, prepared.id, chunk.piece, "Legacy encrypted xxhash64"); + chunk.legacyId = prepared.id; + } + const legacyIdElapsedMs = Number(process.hrtime.bigint() - legacyStart) / 1e6; + const legacyCpu = process.cpuUsage(legacyCpuStart); + const legacyIdCpuMs = (legacyCpu.user + legacyCpu.system) / 1000; + + const decoded = primaryChunks.map((chunk) => decodeChunk(file.kind, chunk.piece)); + const reconstructed = Buffer.concat(decoded); + if (!reconstructed.equals(file.data) || sha256(reconstructed) !== file.expectedSha256) { + throw new Error(`Splitter output did not reconstruct ${file.path} in the ${file.kind} encoding.`); + } + + const chunks = primaryChunks.map((chunk, index) => { + const raw = decoded[index]; + return { + id: chunk.id, + legacyId: chunk.legacyId, + rawBytes: raw.byteLength, + storedUtf8Bytes: Buffer.byteLength(chunk.piece, "utf8"), + dataSha256: sha256(raw), + idUtf8Bytes: Buffer.byteLength(chunk.id, "utf8"), + legacyIdUtf8Bytes: Buffer.byteLength(chunk.legacyId, "utf8"), + piece: chunk.piece, + }; + }); + const fingerprintInput = chunks.map(({ id, legacyId, rawBytes, dataSha256 }) => [id, legacyId, rawBytes, dataSha256]); + const chunkFingerprintSha256 = sha256(Buffer.from(JSON.stringify(fingerprintInput), "utf8")); + return { + chunks, + primaryElapsedMs, + primaryCpuMs, + legacyIdElapsedMs, + legacyIdCpuMs, + chunkFingerprintSha256, + }; +} + +function percentile(sortedValues, fraction) { + if (sortedValues.length === 0) return 0; + return sortedValues[Math.max(0, Math.ceil(sortedValues.length * fraction) - 1)]; +} + +function summarizeRawChunkBytes(values) { + const sorted = [...values].sort((left, right) => left - right); + return { + count: sorted.length, + p50: percentile(sorted, 0.5), + p90: percentile(sorted, 0.9), + p95: percentile(sorted, 0.95), + p99: percentile(sorted, 0.99), + max: sorted.at(-1) ?? 0, + }; +} + +function summariseIdLengths(chunks, key) { + const lengths = chunks.map((chunk) => Buffer.byteLength(chunk[key], "utf8")).sort((left, right) => left - right); + const uniqueLengths = new Map(); + for (const chunk of chunks) uniqueLengths.set(chunk[key], Buffer.byteLength(chunk[key], "utf8")); + const uniqueValues = [...uniqueLengths.values()].sort((left, right) => left - right); + return { + occurrences: lengths.length, + totalUtf8Bytes: lengths.reduce((sum, value) => sum + value, 0), + minLengthBytes: lengths[0] ?? 0, + medianLengthBytes: percentile(lengths, 0.5), + maxLengthBytes: lengths.at(-1) ?? 0, + uniqueIds: uniqueValues.length, + uniqueTotalUtf8Bytes: uniqueValues.reduce((sum, value) => sum + value, 0), + }; +} + +function uniqueById(chunks, idKey = "id") { + const unique = new Map(); + for (const chunk of chunks) { + if (!unique.has(chunk[idKey])) unique.set(chunk[idKey], chunk); + } + return unique; +} + +function dataBytes(chunks, key) { + return chunks.reduce((sum, chunk) => sum + chunk[key], 0); +} + +function summarizeFile(file, measured, includeChunkArray) { + const { chunks } = measured; + const unique = uniqueById(chunks); + const fileMetric = { + path: file.path, + kind: file.kind, + sourceBytes: file.data.byteLength, + chunkCount: chunks.length, + uniqueChunkCount: unique.size, + storedChunkDataUtf8Bytes: dataBytes(chunks, "storedUtf8Bytes"), + uniqueChunkDataUtf8Bytes: dataBytes([...unique.values()], "storedUtf8Bytes"), + uniqueChunkRawBytes: dataBytes([...unique.values()], "rawBytes"), + rawChunkBytes: summarizeRawChunkBytes(chunks.map((chunk) => chunk.rawBytes)), + primaryId: summariseIdLengths(chunks, "id"), + legacyId: summariseIdLengths(chunks, "legacyId"), + chunkFingerprintSha256: measured.chunkFingerprintSha256, + timing: { + primarySplitAndIdElapsedMs: measured.primaryElapsedMs, + primarySplitAndIdCpuMs: measured.primaryCpuMs, + legacyIdElapsedMs: measured.legacyIdElapsedMs, + legacyIdCpuMs: measured.legacyIdCpuMs, + }, + }; + if (includeChunkArray) { + fileMetric.chunks = chunks.map(({ id, legacyId, rawBytes, storedUtf8Bytes, dataSha256, idUtf8Bytes, legacyIdUtf8Bytes }) => ({ + id, + legacyId, + rawBytes, + storedUtf8Bytes, + dataSha256, + idUtf8Bytes, + legacyIdUtf8Bytes, + })); + } + return fileMetric; +} + +function aggregateKind(stage, kind, fileResults, historyIds, previousIds) { + const files = fileResults.filter((item) => item.file.kind === kind); + const chunks = files.flatMap((item) => item.measured.chunks); + const unique = uniqueById(chunks); + const uniqueChunks = [...unique.values()]; + const newVsHistory = uniqueChunks.filter((chunk) => !historyIds.has(chunk.id)); + const newVsPrevious = previousIds === null ? null : uniqueChunks.filter((chunk) => !previousIds.has(chunk.id)); + const storedBytes = dataBytes(chunks, "storedUtf8Bytes"); + const primaryIds = summariseIdLengths(chunks, "id"); + const legacyIds = summariseIdLengths(chunks, "legacyId"); + + return { + kind, + fileCount: files.length, + sourceBytes: files.reduce((sum, item) => sum + item.file.data.byteLength, 0), + chunkCount: chunks.length, + uniqueChunkCount: unique.size, + storedChunkDataUtf8Bytes: storedBytes, + uniqueChunkDataUtf8Bytes: dataBytes(uniqueChunks, "storedUtf8Bytes"), + uniqueChunkRawBytes: dataBytes(uniqueChunks, "rawBytes"), + rawChunkBytes: summarizeRawChunkBytes(chunks.map((chunk) => chunk.rawBytes)), + primaryId: primaryIds, + legacyId: legacyIds, + newUniqueChunksVsHistory: newVsHistory.length, + newUniqueChunkDataUtf8BytesVsHistory: dataBytes(newVsHistory, "storedUtf8Bytes"), + newUniqueChunkRawBytesVsHistory: dataBytes(newVsHistory, "rawBytes"), + newUniqueChunksVsPreviousStage: newVsPrevious === null ? null : newVsPrevious.length, + newUniqueChunkDataUtf8BytesVsPreviousStage: newVsPrevious === null ? null : dataBytes(newVsPrevious, "storedUtf8Bytes"), + newUniqueChunkRawBytesVsPreviousStage: newVsPrevious === null ? null : dataBytes(newVsPrevious, "rawBytes"), + primaryElapsedMs: files.reduce((sum, item) => sum + item.measured.primaryElapsedMs, 0), + primaryCpuMs: files.reduce((sum, item) => sum + item.measured.primaryCpuMs, 0), + legacyIdElapsedMs: files.reduce((sum, item) => sum + item.measured.legacyIdElapsedMs, 0), + legacyIdCpuMs: files.reduce((sum, item) => sum + item.measured.legacyIdCpuMs, 0), + }; +} + +async function runCondition(condition, stages, repeat, order, includeChunkArray) { + const primaryIdContents = new Map(); + const legacyIdContents = new Map(); + const historyIds = new Set(); + let previousIds = null; + const stageResults = []; + + for (const stage of stages) { + const fileResults = []; + for (const file of stage.files) { + const measured = await splitAndHashFile(condition, file, primaryIdContents, legacyIdContents); + fileResults.push({ file, measured }); + } + + const currentIds = new Set(); + for (const item of fileResults) { + for (const chunk of item.measured.chunks) currentIds.add(chunk.id); + } + const kinds = [...new Set(fileResults.map((item) => item.file.kind))].sort(); + const byKind = kinds.map((kind) => aggregateKind(stage, kind, fileResults, historyIds, previousIds)); + const stageFingerprint = fileResults + .map((item) => `${item.file.path}\0${item.measured.chunkFingerprintSha256}`) + .sort() + .join("\n"); + stageResults.push({ + name: stage.name, + fingerprintSha256: sha256(Buffer.from(stageFingerprint, "utf8")), + byKind, + files: fileResults.map((item) => summarizeFile(item.file, item.measured, includeChunkArray)), + }); + + for (const id of currentIds) historyIds.add(id); + previousIds = currentIds; + } + + return { repeat, conditionOrder: order, stages: stageResults }; +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + if (sorted.length === 0) return 0; + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; +} + +function orderForRepeat(sizes, repeat) { + const offset = repeat % sizes.length; + return [...sizes.slice(offset), ...sizes.slice(0, offset)]; +} + +function timingsForRuns(runs, stageName, kind) { + const samples = runs.map((run) => { + const stage = run.stages.find((item) => item.name === stageName); + const group = stage.byKind.find((item) => item.kind === kind); + return { + repeat: run.repeat, + primaryElapsedMs: group.primaryElapsedMs, + primaryCpuMs: group.primaryCpuMs, + legacyIdElapsedMs: group.legacyIdElapsedMs, + legacyIdCpuMs: group.legacyIdCpuMs, + }; + }); + return { + primarySplitAndIdElapsedMsMedian: median(samples.map((sample) => sample.primaryElapsedMs)), + primarySplitAndIdCpuMsMedian: median(samples.map((sample) => sample.primaryCpuMs)), + legacyIdElapsedMsMedian: median(samples.map((sample) => sample.legacyIdElapsedMs)), + legacyIdCpuMsMedian: median(samples.map((sample) => sample.legacyIdCpuMs)), + repeats: samples, + }; +} + +function timingsForFile(runs, stageName, filePath) { + const samples = runs.map((run) => { + const stage = run.stages.find((item) => item.name === stageName); + const file = stage.files.find((item) => item.path === filePath); + return { repeat: run.repeat, ...file.timing }; + }); + return { + primarySplitAndIdElapsedMsMedian: median(samples.map((sample) => sample.primarySplitAndIdElapsedMs)), + primarySplitAndIdCpuMsMedian: median(samples.map((sample) => sample.primarySplitAndIdCpuMs)), + legacyIdElapsedMsMedian: median(samples.map((sample) => sample.legacyIdElapsedMs)), + legacyIdCpuMsMedian: median(samples.map((sample) => sample.legacyIdCpuMs)), + repeats: samples, + }; +} + +function assertRepeatStability(runs, customChunkSize) { + const first = runs[0]; + for (const run of runs.slice(1)) { + for (const stage of first.stages) { + const currentStage = run.stages.find((item) => item.name === stage.name); + if (currentStage.fingerprintSha256 !== stage.fingerprintSha256) { + throw new Error(`Chunk boundaries or IDs changed across repeats for customChunkSize=${customChunkSize}, stage=${stage.name}.`); + } + for (const file of stage.files) { + const currentFile = currentStage.files.find((item) => item.path === file.path); + if (currentFile.chunkFingerprintSha256 !== file.chunkFingerprintSha256) { + throw new Error(`Chunk IDs changed across repeats for ${stage.name}/${file.path} at customChunkSize=${customChunkSize}.`); + } + } + } + } +} + +function compareCandidateBoundaries(candidates, sizes) { + const candidate40 = candidates.get(40); + const candidate60 = candidates.get(60); + if (!sizes.includes(40) || !sizes.includes(60)) return null; + for (const stage of candidate40.stages) { + const otherStage = candidate60.stages.find((item) => item.name === stage.name); + for (const file of stage.files) { + const other = otherStage.files.find((item) => item.path === file.path); + if (file.chunkFingerprintSha256 !== other.chunkFingerprintSha256) { + throw new Error(`customChunkSize=40 and 60 differ for ${stage.name}/${file.path}.`); + } + } + } + return { + customChunkSize40And60SamePerFileBoundariesAndIds: true, + comparedStages: candidate40.stages.length, + comparedFilesPerStage: candidate40.stages[0]?.files.length ?? 0, + }; +} + +async function main() { + const args = parseArguments(process.argv.slice(2)); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return; + } + + const outputDirectory = path.resolve(args.output); + const corpusDirectory = path.join(outputDirectory, "corpus"); + const resultPath = path.join(outputDirectory, "results.json"); + await mkdir(outputDirectory, { recursive: true }); + const startedAt = new Date().toISOString(); + if (COMMONLIB_PACKAGE.version !== "0.1.44") { + throw new Error(`Expected @vrtmrz/livesync-commonlib 0.1.44, found ${COMMONLIB_PACKAGE.version}.`); + } + const manifest = await createCorpus(corpusDirectory, PROJECT_ROOT); + const manifestJson = JSON.stringify(manifest); + const corpusManifestSha256 = sha256(Buffer.from(manifestJson, "utf8")); + const stages = await loadCorpus(manifest, corpusDirectory); + const packageJson = JSON.parse(await readFile(path.join(PROJECT_ROOT, "package.json"), "utf8")); + + if (ID_DERIVATION_KEY.length !== 64 || !/^[0-9a-f]{64}$/u.test(ID_DERIVATION_KEY)) { + throw new Error("The benchmark ID derivation fixture must be 64 lower-case hex characters."); + } + + const conditions = new Map(); + for (const size of args.sizes) { + const condition = await createCondition(size); + const warmupFile = stages[0].files.find((file) => file.kind === "markdown" && file.source.startsWith("repo:")); + if (!warmupFile) throw new Error("Corpus has no repository Markdown file for warm-up."); + await warmCondition(condition, warmupFile); + conditions.set(size, condition); + } + + const runsBySize = new Map(args.sizes.map((size) => [size, []])); + for (let repeat = 0; repeat < args.repeats; repeat += 1) { + const order = orderForRepeat(args.sizes, repeat); + for (const size of order) { + const includeChunkArray = repeat === 0 && (size === 40 || size === 60); + const run = await runCondition(conditions.get(size), stages, repeat + 1, order, includeChunkArray); + runsBySize.get(size).push(run); + } + } + + const candidates = new Map(); + for (const size of args.sizes) { + const runs = runsBySize.get(size); + assertRepeatStability(runs, size); + const firstRun = runs[0]; + const detailedStages = firstRun.stages.map((stage) => ({ + name: stage.name, + fingerprintSha256: stage.fingerprintSha256, + byKind: stage.byKind.map((group) => { + const { primaryElapsedMs, primaryCpuMs, legacyIdElapsedMs, legacyIdCpuMs, ...metrics } = group; + return { + ...metrics, + timing: timingsForRuns(runs, stage.name, group.kind), + }; + }), + files: stage.files.map(({ timing, ...file }) => ({ + ...file, + timing: timingsForFile(runs, stage.name, file.path), + })), + })); + const candidate = { + customChunkSize: size, + chunkSizeCeilingBytes: Math.floor(102400 * (size + 1)), + stages: detailedStages, + runs: runs.map((run) => ({ + repeat: run.repeat, + conditionOrder: run.conditionOrder, + stages: run.stages.map((stage) => ({ name: stage.name, fingerprintSha256: stage.fingerprintSha256 })), + })), + }; + candidates.set(size, candidate); + } + + const comparisons = compareCandidateBoundaries(candidates, args.sizes); + const results = { + schemaVersion: 1, + benchmark: "local Rabin-Karp splitting and chunk-ID measurement", + startedAt, + completedAt: new Date().toISOString(), + runtime: { + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + osRelease: os.release(), + cpuModel: os.cpus()[0]?.model ?? "unknown", + logicalCpuCount: os.cpus().length, + projectVersion: packageJson.version, + commonlibVersion: COMMONLIB_PACKAGE.version, + variant: process.env.LIVESYNC_BENCH_VARIANT || "unlabelled", + commonlibChunksSha256: sha256(await readFile(path.join(COMMONLIB_ROOT, "dist/string_and_binary/chunks.js"))), + }, + settings: { + splitter: "v3-rabin-karp", + minimumChunkSize: 20, + workerDisabled: true, + E2EEAlgorithm: "v2", + encrypt: true, + hashAlg: "xxhash64", + primaryIdDerivation: { + version: 1, + keyFixture: "fixed public benchmark fixture; 64 lower-case hex characters omitted", + }, + legacyIdComparison: { + idDerivationVersion: 0, + algorithm: "encrypted xxhash64 with the fixed public legacy passphrase fixture", + passphrase: "omitted", + }, + enableCompression: false, + useEden: false, + customChunkSizeValues: args.sizes, + warmup: "one actual repository Markdown file per condition; excluded from timings", + timings: "primary includes splitting and actual EntryManager prepareChunk hashing; legacy timing hashes the same emitted chunks separately", + }, + corpus: { + seed: manifest.seed, + manifestSha256: corpusManifestSha256, + fileCount: manifest.files.length, + stages: manifest.stages.map((stage) => ({ + name: stage.name, + directory: stage.directory, + fileCount: stage.files.length, + sourceBytes: stage.files.reduce((sum, file) => sum + file.bytes, 0), + })), + }, + assertions: { + everyFileReconstructedFromSplitterOutput: true, + duplicateIdsHadIdenticalChunkContent: true, + sameBoundariesAndIdsAcrossRepeats: true, + ...(comparisons ?? {}), + }, + candidates: [...candidates.values()], + scope: "Local splitting and hashing only; no network or request timing is measured.", + }; + + await writeFile(resultPath, `${JSON.stringify(results, null, 2)}\n`); + process.stdout.write(`${JSON.stringify({ resultPath, corpusManifestSha256, candidateCount: args.sizes.length, repeats: args.repeats })}\n`); +} + +main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/utils/chunk-size-benchmark/p2p.mjs b/utils/chunk-size-benchmark/p2p.mjs new file mode 100644 index 00000000..b98655f9 --- /dev/null +++ b/utils/chunk-size-benchmark/p2p.mjs @@ -0,0 +1,159 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import { createNewVaultSettings, REMOTE_P2P, SALT_OF_PASSPHRASE } from '@vrtmrz/livesync-commonlib/compat/common/types'; +import { encryptString } from '@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const cli = join(root, 'src/apps/cli/dist/index.cjs'); +const preload = join(root, 'utils/chunk-size-benchmark/datachannel-metrics.cjs'); +const args = process.argv.slice(2); +const option = (key, fallback) => { const at = args.indexOf(`--${key}`); return at < 0 ? fallback : args[at + 1]; }; +const sourceCli = resolve(option('source-cli', cli)); +const targetCli = resolve(option('target-cli', cli)); +const output = resolve(option('output', join(root, '.bench/chunk-size'))); +const corpusPath = resolve(option('corpus', join(output, 'corpus/manifest.json'))); +const sizes = option('sizes', '0,4,10,40').split(',').map(Number); +const repeats = Number(option('repeats', '3')); +const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex'); +const cliSha256 = sha256(await fs.readFile(cli)); +const sourceCliSha256 = sha256(await fs.readFile(sourceCli)); +const targetCliSha256 = sha256(await fs.readFile(targetCli)); +const environment = { ...process.env, NODE_OPTIONS: '--max-old-space-size=2048' }; + +async function runCli(vault, settings, command, commandArgs, logPath, extraEnvironment = {}, entrypoint = cli) { + const marker = '__BENCH_TIME__'; + const started = performance.now(); + const nodeArgs = extraEnvironment.LIVESYNC_BENCH_DATACHANNEL_JSON ? ['--require', preload] : []; + const child = spawn('/usr/bin/time', ['-f', `${marker}%U\t%S\t%M`, process.execPath, ...nodeArgs, entrypoint, vault, '--settings', settings, command, ...commandArgs], { + cwd: root, detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: { ...environment, ...extraEnvironment }, + }); + let stdout = ''; let stderr = ''; + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + const timeout = setTimeout(() => { try { process.kill(-child.pid, 'SIGTERM'); } catch (error) { if (error.code !== 'ESRCH') throw error; } }, 240000); + const code = await new Promise((accept, reject) => { child.once('error', reject); child.once('close', accept); }); + clearTimeout(timeout); + await fs.writeFile(logPath, `${stdout}\n${stderr}`); + if (code !== 0) throw new Error(`CLI ${command} exited ${code}; ${logPath}\n${stderr.slice(-2000)}`); + const line = stderr.split(/\r?\n/).find((row) => row.startsWith(marker)); + assert(line, 'GNU time output is missing'); + const [userSeconds, systemSeconds, maxResidentSetKiB] = line.slice(marker.length).split('\t').map(Number); + return { elapsedMs: performance.now() - started, userCpuMs: userSeconds * 1000, systemCpuMs: systemSeconds * 1000, maxResidentSetKiB }; +} + +async function startHost(vault, settings, logPath, metricsPath) { + const child = spawn(process.execPath, ['--require', preload, sourceCli, vault, '--settings', settings, 'p2p-host'], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: { ...environment, LIVESYNC_BENCH_DATACHANNEL_JSON: metricsPath } }); + let combined = ''; let ended = false; let failure; + const closed = new Promise((accept) => { child.once('error', (error) => { failure = error; ended = true; accept(); }); child.once('close', () => { ended = true; accept(); }); }); + child.stdout.on('data', (data) => { combined += data; }); child.stderr.on('data', (data) => { combined += data; }); + const stop = async () => { + if (!ended) child.kill('SIGTERM'); + const timeout = setTimeout(() => { if (!ended) child.kill('SIGKILL'); }, 10000); + await closed; clearTimeout(timeout); await fs.writeFile(logPath, combined); + }; + const started = performance.now(); + try { + while (!combined.includes('P2P host is running')) { + if (ended || performance.now() - started > 25000) throw failure || new Error(`P2P host did not start: ${combined.slice(-2500)}`); + await delay(100); + } + } catch (error) { await stop(); throw error; } + return { stop, readyMs: performance.now() - started }; +} + +async function settingsFor(size, room, name) { + return { + ...createNewVaultSettings(), isConfigured: true, remoteType: REMOTE_P2P, + encrypt: true, passphrase: 'chunk-size-benchmark-only-passphrase', usePathObfuscation: true, encryptInternalMetadata: true, + idDerivationVersion: 1, idDerivationKey: '', encryptedIdDerivationKey: await encryptString('0123456789abcdef'.repeat(4), '*' + SALT_OF_PASSPHRASE), + E2EEAlgorithm: 'v2', hashAlg: 'xxhash64', chunkSplitterVersion: 'v3-rabin-karp', minimumChunkSize: 20, customChunkSize: size, + enableCompression: false, useEden: false, useIndexedDBAdapter: false, handleFilenameCaseSensitive: false, pathNormalizationVersion: 0, + autoAcceptCompatibleTweak: false, readChunksOnline: false, liveSync: false, syncOnStart: false, syncOnSave: false, periodicReplication: false, + syncOnFileOpen: false, syncMaxSizeInMB: 50, + P2P_Enabled: true, P2P_AutoStart: false, P2P_AutoBroadcast: false, P2P_IsHeadless: true, + P2P_AppID: 'livesync-chunk-size-benchmark', P2P_roomID: room, P2P_passphrase: 'chunk-size-local-relay-fixture', + P2P_relays: 'ws://127.0.0.1:14094/', P2P_AutoAcceptingPeers: '~.*', P2P_AutoDenyingPeers: '', + P2P_turnServers: '', P2P_turnUsername: '', P2P_turnCredential: '', P2P_DevicePeerName: name, + }; +} + +async function copyStage(stage, vault, previous) { + const old = new Map((previous?.files || []).map((file) => [file.path, file.sha256])); + const changed = []; + for (const file of stage.files) { + if (old.get(file.path) === file.sha256) continue; + const from = join(stage.directory, file.path); const to = join(vault, file.path); + await fs.mkdir(dirname(to), { recursive: true }); await fs.copyFile(from, to); + const stat = await fs.stat(from); await fs.utimes(to, stat.atime, stat.mtime); changed.push(file.path); + } + return changed; +} + +await fs.mkdir(join(output, 'p2p'), { recursive: true }); +const manifest = JSON.parse(await fs.readFile(corpusPath, 'utf8')); +for (const stage of manifest.stages) stage.directory = resolve(dirname(corpusPath), stage.directory); +const corpusDigest = sha256(JSON.stringify(manifest.stages.map((stage) => ({ name: stage.name, files: stage.files })))); +const resultPath = join(output, 'p2p/results.json'); +let results = []; +try { const saved = JSON.parse(await fs.readFile(resultPath, 'utf8')); assert.equal(saved.corpusDigest, corpusDigest); assert.equal(saved.sourceCliSha256 ?? saved.cliSha256, sourceCliSha256); assert.equal(saved.targetCliSha256 ?? saved.cliSha256, targetCliSha256); results = saved.results; } catch (error) { if (error.code !== 'ENOENT') throw error; } +const save = () => fs.writeFile(resultPath, JSON.stringify({ version: '1.0.39', commonlib: '0.1.44', variant: process.env.LIVESYNC_BENCH_VARIANT || 'unlabelled', cliSha256, sourceCliSha256, targetCliSha256, corpusDigest, + scope: 'Local WebRTC direct path; timed target command includes process start-up, peer discovery, connection establishment, pull and reverse synchronisation. DataChannel payload bytes are measured using werift counters observed through a preload on both endpoints; they exclude transport overhead, transport retransmissions, and relay signalling. ICE byte counters are retained as diagnostics and report zero in this runtime. GNU time covers the target process; the preload also records CPU/RSS on each endpoint. Source mirroring and host start-up are recorded separately. E2EE V2 settings with independent IDs; compression disabled.', results }, null, 2)); +for (let repeat = 0; repeat < repeats; repeat++) { + const rotation = repeat % sizes.length; const order = [...sizes.slice(rotation), ...sizes.slice(0, rotation)]; + for (const size of order) { + const name = `p2p-c${size}-r${repeat + 1}`; + if (results.some((result) => result.name === name && result.complete)) continue; + const work = join(output, 'p2p', name); await fs.mkdir(join(work, 'logs'), { recursive: true }); + const a = join(work, 'source'); const b = join(work, 'target'); + await fs.rm(a, { recursive: true, force: true }); await fs.rm(b, { recursive: true, force: true }); + await fs.mkdir(a, { recursive: true }); await fs.mkdir(b, { recursive: true }); + const room = `chunk-size-${Date.now().toString(36)}-${name}`; + const hostName = `host-${name}`; + const settingsA = join(work, 'source-settings.json'); const settingsB = join(work, 'target-settings.json'); + await fs.writeFile(settingsA, JSON.stringify(await settingsFor(size, room, hostName))); + await fs.writeFile(settingsB, JSON.stringify(await settingsFor(size, room, `target-${name}`))); + const result = { name, customChunkSize: size, repeat: repeat + 1, stages: [], complete: false }; + results.push(result); await save(); + let previous; + try { + for (const stage of manifest.stages) { + const log = (label) => join(work, 'logs', `${stage.name}-${label}.txt`); + const changedFiles = await copyStage(stage, a, previous); + const mirror = await runCli(a, settingsA, 'mirror', [], log('mirror'), {}, sourceCli); + const hostMetricsPath = join(work, `${stage.name}-host-datachannel.json`); + const targetMetricsPath = join(work, `${stage.name}-target-datachannel.json`); + const host = await startHost(a, settingsA, log('host'), hostMetricsPath); + const statsPath = join(work, `${stage.name}-stats.jsonl`); await fs.rm(statsPath, { force: true }); + let sync; + try { sync = await runCli(b, settingsB, 'p2p-sync', [hostName, '120'], log('sync'), { LIVESYNC_P2P_STATS_JSONL: statsPath, LIVESYNC_BENCH_DATACHANNEL_JSON: targetMetricsPath }, targetCli); } + finally { await host.stop(); } + const rows = (await fs.readFile(statsPath, 'utf8')).trim().split(/\r?\n/); + const stats = JSON.parse(rows.at(-1)); + assert(stats.candidatePathCollected, 'ICE statistics were not collected'); + assert(!stats.selectedPath.includes('relay'), `Unexpected TURN path: ${stats.selectedPath}`); + const dataChannel = { host: JSON.parse(await fs.readFile(hostMetricsPath, 'utf8')), target: JSON.parse(await fs.readFile(targetMetricsPath, 'utf8')) }; + assert(dataChannel.host.payloadBytesSent > 0 && dataChannel.target.payloadBytesSent > 0, 'DataChannel counters must record both directions'); + const materialisedByPull = []; + for (const file of stage.files) { + let bytes; try { bytes = await fs.readFile(join(b, file.path)); } catch (error) { if (error.code !== 'ENOENT') throw error; } + if (!bytes || sha256(bytes) !== file.sha256) { + const to = join(work, 'verification', file.path); await fs.mkdir(dirname(to), { recursive: true }); + const measurement = await runCli(b, settingsB, 'pull', [file.path, to], log(`verify-${file.path.replace(/[^a-zA-Z0-9]/g, '-')}`), {}, targetCli); + bytes = await fs.readFile(to); materialisedByPull.push({ path: file.path, measurement }); + } + assert.equal(sha256(bytes), file.sha256, `${name} ${stage.name} ${file.path}`); + } + result.stages.push({ name: stage.name, changedFiles, mirror, hostReadyMs: host.readyMs, sync, stats, dataChannel, verifiedFiles: stage.files.length, materialisedByPull }); + await save(); console.log(JSON.stringify({ case: name, stage: stage.name, syncMs: Math.round(sync.elapsedMs), selectedPath: stats.selectedPath, hostPayloadBytesSent: dataChannel.host.payloadBytesSent, targetPayloadBytesSent: dataChannel.target.payloadBytesSent })); + previous = stage; + } + result.complete = true; await save(); + } catch (error) { result.error = String(error); await save(); throw error; } + } +} +console.log(`Completed P2P measurements: ${resultPath}`); diff --git a/utils/chunk-size-benchmark/rabin-window.test.mjs b/utils/chunk-size-benchmark/rabin-window.test.mjs new file mode 100644 index 00000000..78cf170a --- /dev/null +++ b/utils/chunk-size-benchmark/rabin-window.test.mjs @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import test from 'node:test'; + +const packageRoot = dirname(fileURLToPath(import.meta.resolve('@vrtmrz/livesync-commonlib/package.json'))); +const { splitPiecesRabinKarp } = await import(pathToFileURL(join(packageRoot, 'dist/string_and_binary/chunks.js')).href); + +test('zero windows lose the contribution of a marker in the preceding chunk', async () => { + const blockBytes = 300 * 1024; + const input = new Uint8Array(blockBytes * 3); + for (let end = blockBytes; end <= input.length; end += blockBytes) input[end - 1] = 1; + const pieces = await splitPiecesRabinKarp(new Blob([input], { type: 'application/octet-stream' }), 1100 * 1024, false, 20, 'markers.bin', false); + const chunks = []; + for await (const encoded of pieces()) chunks.push(Buffer.from(encoded, 'base64')); + assert.deepEqual(Buffer.concat(chunks), Buffer.from(input), 'all input bytes must survive splitting'); + // A 48-byte zero window hashes to zero; each isolated marker hashes to one. + // Markers are further apart than the 256 KiB minimum and below the maximum. + assert.deepEqual(chunks.map((chunk) => chunk.length), [blockBytes, blockBytes, blockBytes]); +}); diff --git a/utils/chunk-size-benchmark/remote.mjs b/utils/chunk-size-benchmark/remote.mjs new file mode 100644 index 00000000..5dc7683e --- /dev/null +++ b/utils/chunk-size-benchmark/remote.mjs @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import http from 'node:http'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import { createNewVaultSettings, REMOTE_COUCHDB, REMOTE_MINIO, SALT_OF_PASSPHRASE } from '@vrtmrz/livesync-commonlib/compat/common/types'; +import { encryptString } from '@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption'; +import { S3Client, CreateBucketCommand, ListObjectsV2Command, DeleteObjectsCommand, DeleteBucketCommand } from '@aws-sdk/client-s3'; + +const here = dirname(fileURLToPath(import.meta.url)); +const root = resolve(here, '../..'); +const cli = join(root, 'src/apps/cli/dist/index.cjs'); +const args = process.argv.slice(2); +const option = (key, fallback) => { const at = args.indexOf(`--${key}`); return at < 0 ? fallback : args[at + 1]; }; +const sourceCli = resolve(option('source-cli', cli)); +const targetCli = resolve(option('target-cli', cli)); +const output = resolve(option('output', join(root, '.bench/chunk-size'))); +const corpusPath = resolve(option('corpus', join(output, 'corpus/manifest.json'))); +const sizes = option('sizes', '0,4,10,20,40').split(',').map(Number); +const providers = option('providers', 'couchdb,s3').split(','); +const repeats = Number(option('repeats', '3')); +const rttMs = Number(option('rtt-ms', '0')); +const user = 'bench'; +const password = 'benchmark-only-password'; +const accessKeyId = 'bench-access-key'; +const secretAccessKey = 'benchmark-only-secret-key'; +const couchUri = 'http://127.0.0.1:15994'; +const s3Uri = 'http://127.0.0.1:19094'; +const auth = `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; +const key = '0123456789abcdef'.repeat(4); +const sha256 = (value) => createHash('sha256').update(value).digest('hex'); +const sourceCliSha256 = sha256(await fs.readFile(sourceCli)); +const targetCliSha256 = sha256(await fs.readFile(targetCli)); +const credentials = { accessKeyId, secretAccessKey }; +const s3 = new S3Client({ endpoint: s3Uri, region: 'us-east-1', forcePathStyle: true, credentials }); + +function emptyCounters() { return { requests: 0, requestBodyBytes: 0, responseBodyBytes: 0, maxRequestBodyBytes: 0, byMethod: {}, byOperation: {} }; } + +async function startProxy(backend) { + let counters = emptyCounters(); + const server = http.createServer(async (request, response) => { + counters.requests++; + counters.byMethod[request.method] = (counters.byMethod[request.method] || 0) + 1; + const url = new URL(request.url, backend); + const operation = url.pathname.split('/').find((part) => part.startsWith('_')) || (url.searchParams.has('list-type') ? 'list' : 'object'); + counters.byOperation[operation] = (counters.byOperation[operation] || 0) + 1; + if (rttMs) await delay(rttMs / 2); + const target = new URL(request.url, backend); + const upstream = http.request(target, { method: request.method, headers: request.headers }, async (incoming) => { + if (rttMs) await delay(rttMs / 2); + response.writeHead(incoming.statusCode, incoming.headers); + incoming.on('data', (chunk) => { counters.responseBodyBytes += chunk.length; }); + incoming.pipe(response); + }); + upstream.on('error', (error) => { response.writeHead(502); response.end(String(error)); }); + let requestBytes = 0; + request.on('data', (chunk) => { counters.requestBodyBytes += chunk.length; requestBytes += chunk.length; }); + request.on('end', () => { counters.maxRequestBodyBytes = Math.max(counters.maxRequestBodyBytes, requestBytes); }); + request.pipe(upstream); + }); + await new Promise((accept, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', accept); }); + return { uri: `http://127.0.0.1:${server.address().port}`, reset: () => { counters = emptyCounters(); }, snapshot: () => structuredClone(counters), close: () => new Promise((accept) => server.close(accept)) }; +} + +async function couch(path, method = 'GET', body) { + const response = await fetch(`${couchUri}${path}`, { method, headers: { Authorization: auth, 'Content-Type': 'application/json' }, body: body === undefined ? undefined : JSON.stringify(body) }); + if (!response.ok) throw new Error(`CouchDB ${method} ${path}: ${response.status} ${await response.text()}`); + return response.json(); +} + +async function ready() { + for (let attempt = 0; attempt < 30; attempt++) { + try { await couch('/_up'); await s3.send(new ListObjectsV2Command({ Bucket: 'missing-benchmark-bucket' })).catch((error) => { if (error.$metadata?.httpStatusCode !== 404) throw error; }); return; } + catch (error) { if (attempt === 29) throw error; await delay(1000); } + } +} + +async function runCli(directory, settings, command, commandArgs, logPath, entrypoint = cli) { + const marker = '__BENCH_TIME__'; + const started = performance.now(); + const child = spawn('/usr/bin/time', ['-f', `${marker}%U\t%S\t%M`, process.execPath, entrypoint, directory, '--settings', settings, command, ...commandArgs], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=2048' } }); + let stdout = ''; let stderr = ''; + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + const timeout = setTimeout(() => child.kill('SIGTERM'), 180000); + const code = await new Promise((accept, reject) => { child.once('error', reject); child.once('exit', accept); }); + clearTimeout(timeout); + const elapsedMs = performance.now() - started; + await fs.writeFile(logPath, `${stdout}\n${stderr}`); + if (code !== 0) throw new Error(`CLI ${command} exited ${code}; ${logPath}\n${stderr.slice(-2200)}`); + const line = stderr.split(/\r?\n/).find((row) => row.startsWith(marker)); + assert(line, 'GNU time output is missing'); + const [userSeconds, systemSeconds, maxResidentSetKiB] = line.slice(marker.length).split('\t').map(Number); + return { elapsedMs, userCpuMs: userSeconds * 1000, systemCpuMs: systemSeconds * 1000, maxResidentSetKiB }; +} + +async function settingsFor(provider, endpoint, resource, size) { + return { + ...createNewVaultSettings(), isConfigured: true, remoteType: provider === 'couchdb' ? REMOTE_COUCHDB : REMOTE_MINIO, + couchDB_URI: endpoint, couchDB_DBNAME: resource, couchDB_USER: user, couchDB_PASSWORD: password, + endpoint, bucket: resource, accessKey: accessKeyId, secretKey: secretAccessKey, region: 'us-east-1', forcePathStyle: true, + encrypt: true, passphrase: 'chunk-size-benchmark-only-passphrase', usePathObfuscation: true, encryptInternalMetadata: true, + idDerivationVersion: 1, idDerivationKey: '', encryptedIdDerivationKey: await encryptString(key, '*' + SALT_OF_PASSPHRASE), E2EEAlgorithm: 'v2', hashAlg: 'xxhash64', + chunkSplitterVersion: 'v3-rabin-karp', minimumChunkSize: 20, customChunkSize: size, autoAcceptCompatibleTweak: false, + enableCompression: false, useEden: false, useIndexedDBAdapter: false, handleFilenameCaseSensitive: false, pathNormalizationVersion: 0, + readChunksOnline: false, liveSync: false, syncOnStart: false, syncOnSave: false, periodicReplication: false, + syncOnFileOpen: false, syncMaxSizeInMB: 50, P2P_Enabled: false, + }; +} + +async function copyStage(stage, vault, previous) { + const old = new Map((previous?.files || []).map((file) => [file.path, file.sha256])); + const changed = []; + for (const file of stage.files) { + if (old.get(file.path) === file.sha256) continue; + const from = join(stage.directory, file.path); const to = join(vault, file.path); + await fs.mkdir(dirname(to), { recursive: true }); await fs.copyFile(from, to); + const stat = await fs.stat(from); await fs.utimes(to, stat.atime, stat.mtime); changed.push(file.path); + } + return changed; +} + +async function inspect(provider, resource, previousIds) { + if (provider === 'couchdb') { + const info = await couch(`/${resource}`); + const all = await couch(`/${resource}/_all_docs?include_docs=true`); + const chunks = all.rows.filter(({ doc }) => doc?.type === 'leaf'); + const added = chunks.filter(({ id }) => !previousIds.has(id)); + const ids = chunks.map(({ id }) => id); + const jsonBytes = ({ doc }) => Buffer.byteLength(JSON.stringify(doc)); + return { ids, metrics: { documentCount: info.doc_count, chunkCount: chunks.length, newChunkCount: added.length, chunkJsonBytes: chunks.reduce((sum, row) => sum + jsonBytes(row), 0), newChunkJsonBytes: added.reduce((sum, row) => sum + jsonBytes(row), 0), largestChunkJsonBytes: Math.max(0, ...chunks.map(jsonBytes)), largestDocumentJsonBytes: Math.max(0, ...all.rows.map(jsonBytes)), storedChunkDataBytes: chunks.reduce((sum, { doc }) => sum + Buffer.byteLength(doc.data || ''), 0), sizes: info.sizes, idLengthMin: Math.min(...ids.map((id) => id.length)), idLengthMax: Math.max(...ids.map((id) => id.length)) } }; + } + const objects = []; let token; + do { const result = await s3.send(new ListObjectsV2Command({ Bucket: resource, ContinuationToken: token })); objects.push(...result.Contents || []); token = result.NextContinuationToken; } while (token); + return { ids: objects.map(({ Key }) => Key), metrics: { objectCount: objects.length, storedObjectBytes: objects.reduce((sum, { Size }) => sum + Size, 0), newObjectCount: objects.filter(({ Key }) => !previousIds.has(Key)).length } }; +} + +async function removeResource(provider, resource) { + if (provider === 'couchdb') { await couch(`/${resource}`, 'DELETE'); return; } + let token; + do { + const listing = await s3.send(new ListObjectsV2Command({ Bucket: resource, ContinuationToken: token })); + if (listing.Contents?.length) await s3.send(new DeleteObjectsCommand({ Bucket: resource, Delete: { Objects: listing.Contents.map(({ Key }) => ({ Key })) } })); + token = listing.NextContinuationToken; + } while (token); + await s3.send(new DeleteBucketCommand({ Bucket: resource })); +} + +await fs.mkdir(join(output, 'remote'), { recursive: true }); +const manifest = JSON.parse(await fs.readFile(corpusPath, 'utf8')); +for (const stage of manifest.stages) stage.directory = resolve(dirname(corpusPath), stage.directory); +const digest = sha256(JSON.stringify(manifest.stages.map((stage) => ({ name: stage.name, files: stage.files })))); +const resultPath = join(output, 'remote', `results-rtt${rttMs}.json`); +let results = []; +try { const saved = JSON.parse(await fs.readFile(resultPath, 'utf8')); assert.equal(saved.corpusDigest, digest); assert.equal(saved.sourceCliSha256, sourceCliSha256); assert.equal(saved.targetCliSha256, targetCliSha256); results = saved.results; } catch (error) { if (error.code !== 'ENOENT') throw error; } +const proxies = { couchdb: await startProxy(couchUri), s3: await startProxy(s3Uri) }; +const save = async () => fs.writeFile(resultPath, JSON.stringify({ version: '1.0.39', commonlib: '0.1.44', variant: process.env.LIVESYNC_BENCH_VARIANT || 'unlabelled', sourceCliSha256, targetCliSha256, corpusDigest: digest, rttMs, mode: 'local services; HTTP body bytes exclude headers; full chunk download; E2EE V2 with independent IDs; compression disabled', results }, null, 2)); +try { + await ready(); + for (let repeat = 0; repeat < repeats; repeat++) { + const rotation = repeat % sizes.length; const order = [...sizes.slice(rotation), ...sizes.slice(0, rotation)]; + for (const provider of (repeat % 2 ? [...providers].reverse() : providers)) for (const size of order) { + const name = `${provider}-c${size}-r${repeat + 1}`; + if (results.some((result) => result.name === name && result.complete)) continue; + const resource = `chunks-${Date.now().toString(36)}-${name}`; + const work = join(output, 'remote', name); await fs.mkdir(join(work, 'logs'), { recursive: true }); + const a = join(work, 'source'); const b = join(work, 'target'); + await fs.rm(a, { recursive: true, force: true }); await fs.rm(b, { recursive: true, force: true }); + await fs.mkdir(a, { recursive: true }); await fs.mkdir(b, { recursive: true }); + const settingsA = join(work, 'source-settings.json'); const settingsB = join(work, 'target-settings.json'); + const settings = await settingsFor(provider, proxies[provider].uri, resource, size); + await fs.writeFile(settingsA, JSON.stringify(settings)); await fs.writeFile(settingsB, JSON.stringify(settings)); + if (provider === 'couchdb') await couch(`/${resource}`, 'PUT'); else await s3.send(new CreateBucketCommand({ Bucket: resource })); + const result = { name, provider, customChunkSize: size, repeat: repeat + 1, stages: [], complete: false }; + results.push(result); await save(); + try { + let previous; let ids = new Set(); + for (const stage of manifest.stages) { + const changedFiles = await copyStage(stage, a, previous); + const log = (label) => join(work, 'logs', `${stage.name}-${label}.txt`); + const mirror = await runCli(a, settingsA, 'mirror', [], log('mirror'), sourceCli); + proxies[provider].reset(); + const upload = await runCli(a, settingsA, 'sync', [], log('upload'), sourceCli); upload.http = proxies[provider].snapshot(); + const remote = await inspect(provider, resource, ids); ids = new Set(remote.ids); + proxies[provider].reset(); + const download = await runCli(b, settingsB, 'sync', [], log('download'), targetCli); download.http = proxies[provider].snapshot(); + const verified = []; const materialisedByPull = []; + for (const file of stage.files) { + let bytes; try { bytes = await fs.readFile(join(b, file.path)); } catch (error) { if (error.code !== 'ENOENT') throw error; } + if (!bytes || sha256(bytes) !== file.sha256) { + const target = join(work, 'verification', file.path); await fs.mkdir(dirname(target), { recursive: true }); + const measurement = await runCli(b, settingsB, 'pull', [file.path, target], log(`verify-${file.path.replace(/[^a-zA-Z0-9]/g, '-')}`), targetCli); + bytes = await fs.readFile(target); materialisedByPull.push({ path: file.path, measurement }); + } + assert.equal(sha256(bytes), file.sha256, `${name} ${stage.name} ${file.path}`); verified.push(file.path); + } + result.stages.push({ name: stage.name, changedFiles, mirror, upload, download, remote: remote.metrics, verifiedFiles: verified.length, materialisedByPull }); + await save(); console.log(JSON.stringify({ case: name, stage: stage.name, uploadMs: Math.round(upload.elapsedMs), downloadMs: Math.round(download.elapsedMs), requestBytes: upload.http.requestBodyBytes, responseBytes: download.http.responseBodyBytes, remote: remote.metrics })); + previous = stage; + } + result.complete = true; await save(); + } catch (error) { result.error = String(error); await save(); throw error; } + finally { await removeResource(provider, resource); } + } + } +} finally { await Promise.all(Object.values(proxies).map((proxy) => proxy.close())); s3.destroy(); } +console.log(`Completed remote measurements: ${resultPath}`); diff --git a/utils/chunk-size-benchmark/report.mjs b/utils/chunk-size-benchmark/report.mjs new file mode 100644 index 00000000..3c0d5c87 --- /dev/null +++ b/utils/chunk-size-benchmark/report.mjs @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import { join, resolve } from 'node:path'; + +const directory = resolve(process.argv[2] || '.bench/rabin-window-fixed'); +const baselineDirectory = resolve(process.argv[3] || '.bench/chunk-size'); +const load = async (file, base = directory) => JSON.parse(await fs.readFile(join(base, file), 'utf8')); +const summary = await load('summary.json'); +const baseline = await load('summary.json', baselineDirectory); +const comparison = await load('comparison.json'); +const validation = await load('validation.json'); +const boundary = await load('cloudant-json-boundary/results.json'); +const consumer = await load('context/validated-consumer.json'); +const recovery = await load('context/cli-bundle-recovery.json'); +assert.equal(validation.completedRemoteCases, 30); +assert.equal(validation.completedP2PCases, 12); +assert.equal(summary.corpus.manifestSha256, baseline.corpus.manifestSha256); +const mib = bytes => (bytes / 1048576).toFixed(3); +const second = milliseconds => (milliseconds / 1000).toFixed(2); +const getRemote = (data, provider, size, stage) => data.remote.find(row => row.provider === provider && row.customChunkSize === size && row.stage === stage); +const getP2P = (data, size, stage) => data.p2p.find(row => row.customChunkSize === size && row.stage === stage); +const getLocal = (data, size, stage) => data.local.find(row => row.customChunkSize === size && row.stage === stage); +const decrease = (before, after) => ((1 - after / before) * 100).toFixed(1); +const lines = []; +const paragraph = text => lines.push(text, ''); +paragraph('Rabin–KarpįĒ“å‡Ļį†äŋŽæ­Ŗ垌ぎ原æ¸ŦīŧˆSelf-hosted LiveSync 1.0.39 / Commonlib 0.1.44ぎæœĒå…Ŧ開äŋŽæ­Ŗãƒ‘ãƒƒã‚ąãƒŧジīŧ‰'); +paragraph('įĒ“å‡Ļį†ã¯ã€äģŠå›žãŽæŒŋå…Ĩ垌ぎチãƒŖãƒŗク再刊į”¨ã‚’åĻ¨ã’ãĻいた原因ぎ一つだãŖた。最小限ぎäŋŽæ­ŖをCommonlibへ加え、前回と厌全ãĢ同じå…Ĩ力ãƒģč¨­åŽšãƒģ候čŖœã‚’再æ¸Ŧ厚した。äģŠå›žãŽæˇˇåœ¨ã‚ŗãƒŧパ゚では、通常ぎ新čĻæŽ¨åĨ¨å€¤ã‚’4īŧˆä¸Šé™500 KiBīŧ‰ã¸æƒãˆã‚‹æĄˆãŒæœ‰åŠ›ãĢãĒãŖた。Cloudant寞åŋœã¯æŽĨįļšå…ˆã”とぎ推åĨ¨å€¤ã§ã¯ãĒく、将æĨぎ同期先もåĢむvaultå…ąé€šãŽãƒ‡ãƒŧã‚ŋåˆļį´„としãĻæ‰ąã†åŋ…čĻãŒã‚る。ただし数値4ã ã‘ã§å…¨č¨­åŽšãŽCloudantäē’換性をäŋč¨ŧするįĩčĢ–ではãĒい。初期値、推åĨ¨å€¤ã€UI、およãŗį§ģ行å‡Ļį†ã¯ãžã å¤‰æ›´ã—ãĻいãĒい。'); +paragraph(`LiveSyncぎåŸēį‚šã¯ ${consumer.liveSyncBaseRevision}、CommonlibぎåŸēį‚šã¯ ${consumer.commonlibBaseRevision}。Commonlib所有ãƒĒポジトãƒĒãƒŧぎäŊœæĨ­ãƒ„ãƒĒãƒŧでäŋŽæ­Ŗし、単äŊ“ãƒģåĸƒį•ŒãƒģãƒĒãƒĒãƒŧ゚ツãƒŧãƒĢãƒģãƒ‘ãƒƒã‚ąãƒŧジ検č¨ŧを通したtarballをLiveSyncへfile䞝存としãĻå…Ĩれた。å…Ŧ開į‰ˆ0.1.44をįŊŽæ›ã—ãĻå…Ŧ開したもぎではãĒい。候čŖœãŽåŽŸäŊ“は ${consumer.packageIntegrity}、イãƒŗ゚トãƒŧãƒĢ済ãŋchunks.jsぎSHA-256は ${consumer.installedChunksSha256}。`); +paragraph('äŋŽæ­Ŗはsrc/string_and_binary/chunks.tsぎįĒ“æ›´æ–°æĄäģļを pos >= start + windowSize から pos >= windowSize へ変える1čĄŒã¨ã€ããŽį›Žįš„ã‚’į¤ēすã‚ŗãƒĄãƒŗトである。åĸƒį•Œã§startãŒé€˛ã‚“ã§ã‚‚ã€hashぎ48バイトįĒ“ã‚’é€Ŗįļšã—ãĻ更新する。åŊĸåŧã€ID導å‡ēæ–šæŗ•ã€č¨­åŽšã‚­ãƒŧ、最小ãƒģåšŗ均ãƒģ最大チãƒŖãƒŗクį›Žæ¨™ã¯å¤‰æ›´ã—ãĻいãĒい。'); +paragraph('先ãĢ3äģļãŽæŒ¯ã‚‹čˆžã„ãŽå›žå¸°ãƒ†ã‚šãƒˆã‚’čŋŊ加し、æœĒäŋŽæ­Ŗぎå…Ŧ開0.1.44į›¸åŊ“ã‚Ŋãƒŧ゚で3äģļとも期垅したåĸƒį•ŒassertionãŒå¤ąæ•—ã™ã‚‹ã“ã¨ã‚’įĸēčĒã—た。バイナãƒĒ300/300/300 KiBぎį›Žå°åˆ—は垓æĨ300/256/256/88 KiBとãĒり、æŒŋå…Ĩ垌ぎ同一æœĢ尞チãƒŖãƒŗクを再刊į”¨ã§ããĒかãŖた。テキ゚トぎį›Žå°åˆ—でも過åŽģぎチãƒŖãƒŗクぎåŊąéŸŋが掋ãŖた。äŋŽæ­Ŗ垌は3äģļã¨ã‚‚æˆåŠŸã—ã€å†æ§‹æˆã—ãŸå…¨ãƒã‚¤ãƒˆã‚‚ä¸€č‡´ã—ãŸã€‚Commonlib全123ãƒ•ã‚Ąã‚¤ãƒĢãƒģ2,416テ゚ト、型、boundary、release tools、packed consumerが成功した。LiveSyncはnpm 10クãƒĒãƒŧãƒŗイãƒŗ゚トãƒŧãƒĢ、通常npm ci、check、単äŊ“1,297成功ãƒģ1 skipped、およãŗObsidian bundle/CLI/WebApp/WebPeerビãƒĢドが成功した。Community lintぎæ—ĸ存51 warningsは掋ãŖãĻいる。'); +paragraph(`å…Ĩ力は前回と同じ15ãƒ•ã‚Ąã‚¤ãƒĢã€åˆæœŸåˆč¨ˆ ${mib(summary.corpus.stages[0].sourceBytes)} MiB。通常ぎプロジェクトMarkdown10æœŦ、į´„512 KiBとį´„5 MiBぎį”ŸæˆMarkdown、PNG2枚、およãŗå›ē厚seedぎ8 MiBéĢ˜ã‚¨ãƒŗトロピãƒŧバイナãƒĒである。manifest SHA-256は ${summary.corpus.manifestSha256}。æŽĩ階は初回、4 KiBぎバイナãƒĒ上書きį­‰ã€įļšã„ãĻバイナãƒĒ途中4 KiBæŒŋå…Ĩとį”ŸæˆMarkdownへぎæŽĩčŊæŒŋå…Ĩã€‚č¨­åŽšã¯Rabin–Karp、minimumChunkSize=20、E2EE V2、į‹ŦįĢ‹IDéĩversion 1、パ゚ãƒģ内部Metadataæš—åˇåŒ–ã€ãƒãƒŖãƒŗク圧į¸ŽãĒし、EdenãĒし、全chunk取垗でå›ē厚した。`); +paragraph('ロãƒŧã‚ĢãƒĢåˆ†å‰˛ã¯0,1,4,10,20,40,60を各3回。HTTPはCouchDB 3.5.0とRustFSを0,4,10,20,40各3回、P2Pは0,4,10,40各3回。同じポト上ぎį”Ÿæˆãƒ†ã‚šãƒˆvaultとã‚ŋ゚ク専į”¨ã‚ŗãƒŗテナをäŊŋį”¨ã—、WAN遅åģļã‚„ãƒ‘ã‚ąãƒƒãƒˆãƒ­ã‚šã¯åŠ ãˆãĻいãĒい。CPUč˛ čˇã‚’ã‹ã‘ã‚‹æ¤œč¨ŧãƒģæ¸Ŧ厚は同時ãĢåŽŸčĄŒã—ãĻいãĒい。Node v24.18.0 / Linux arm64 / Neoverse-N1 / čĢ–į†CPU3である。'); +paragraph('äģĨ下はHTTPぎ原送äŋĄæœŦ文ぎ3回中夎値。ヘッダãƒŧãƒģTLSはåĢめãĒい。同期į§’数はCLIčĩˇå‹•ã‚’åĢãŋ、äē‹å‰ãŽmirrorとčģĸé€åžŒãŽãƒ•ã‚Ąã‚¤ãƒĢ垊元ãƒģSHAį…§åˆã‚’åĢぞãĒい。OSキãƒŖãƒƒã‚ˇãƒĨやã‚ĩãƒŧビ゚は各回ãĢ再čĩˇå‹•ã—ãĻいãĒい。S3ぎJournalパック圧į¸Žã¯enableCompressionãĢよるチãƒŖãƒŗク圧į¸Žã¨ã¯åˆĨである。'); +lines.push('remote | value | 初期送äŋĄMiB | 上書き送äŋĄMiB | æŒŋå…Ĩ送äŋĄMiB | 初期送äŋĄį§’ | 初期受äŋĄį§’ | 初期HTTPčĻæą‚æ•°'); +for (const provider of ['couchdb', 's3']) for (const size of [0, 4, 10, 20, 40]) { + const initial = getRemote(summary, provider, size, 'initial'); + const overwrite = getRemote(summary, provider, size, 'overwrite'); + const insert = getRemote(summary, provider, size, 'insert'); + lines.push([provider, size, mib(initial.upload_requestBodyBytes.median), mib(overwrite.upload_requestBodyBytes.median), mib(insert.upload_requestBodyBytes.median), second(initial.upload_elapsedMs.median), second(initial.download_elapsedMs.median), initial.upload_requests.median].join(' | ')); +} +lines.push(''); +paragraph('P2Pは全回host<->hostįĩŒčˇ¯ã€‚werift 0.24.4ぎcandidate-pair bytesã‚Ģã‚Ļãƒŗã‚ŋãƒŧは0ãŽãŸã‚æŽĄį”¨ã›ãšã€RTCDataChannelぎ原送äŋĄpayloadã‚’ä¸ĄįĢ¯ã§čĻŗæ¸Ŧã—ãŸã€‚ä¸‹čĄ¨ã¯é€äŋĄå…ƒhost側で、RPCåˆļåžĄã¨ã‚ĸプãƒĒåą¤å†é€ã‚’åĢãŋ、SCTP/DTLS/UDP/IPぎヘッダãƒŧ、下äŊåą¤å†é€ã€Nostr signallingはåĢぞãĒい。HTTPæœŦ文量と厌全ãĢåŒã˜åą¤ã§ã¯ãĒい。同期時間はæŽĸį´ĸãƒģæŽĨįļšãƒģ双斚向同期をåĢむ。'); +lines.push('value | 初期送äŋĄMiB | 上書き送äŋĄMiB | æŒŋå…Ĩ送äŋĄMiB 中夎値 [min..max] | 初期同期į§’ | 上書き同期į§’ | æŒŋå…Ĩ同期į§’'); +for (const size of [0, 4, 10, 40]) { + const initial = getP2P(summary, size, 'initial'); const overwrite = getP2P(summary, size, 'overwrite'); const insert = getP2P(summary, size, 'insert'); + lines.push([size, mib(initial.host_payloadBytesSent.median), mib(overwrite.host_payloadBytesSent.median), `${mib(insert.host_payloadBytesSent.median)} [${mib(insert.host_payloadBytesSent.min)}..${mib(insert.host_payloadBytesSent.max)}]`, second(initial.elapsedMs.median), second(overwrite.elapsedMs.median), second(insert.elapsedMs.median)].join(' | ')); +} +lines.push(''); +paragraph('値4ぎ途中æŒŋå…ĨãĢおけるäŋŽæ­Ŗ前→äŋŽæ­Ŗ垌ぎ比čŧƒã¯æŦĄãŽã¨ãŠã‚Šã€‚'); +lines.push('å¯žčąĄ | äŋŽæ­Ŗ前MiB | äŋŽæ­Ŗ垌MiB | 減少įŽ‡'); +const oldRaw = getLocal(baseline, 4, 'insert').binaryNewRawBytes; const newRaw = getLocal(summary, 4, 'insert').binaryNewRawBytes; +lines.push(`新しくåŋ…čĻãĒバイナãƒĒぎį”Ÿãƒ‡ãƒŧã‚ŋ | ${mib(oldRaw)} | ${mib(newRaw)} | ${decrease(oldRaw, newRaw)}%`); +for (const provider of ['couchdb', 's3']) { + const oldValue = getRemote(baseline, provider, 4, 'insert').upload_requestBodyBytes.median; const newValue = getRemote(summary, provider, 4, 'insert').upload_requestBodyBytes.median; + lines.push(`${provider}送äŋĄæœŦ文 | ${mib(oldValue)} | ${mib(newValue)} | ${decrease(oldValue, newValue)}%`); +} +const oldPeer = getP2P(baseline, 4, 'insert').host_payloadBytesSent.median; const newPeer = getP2P(summary, 4, 'insert').host_payloadBytesSent.median; +lines.push(`P2P host payload | ${mib(oldPeer)} | ${mib(newPeer)} | ${decrease(oldPeer, newPeer)}%`, ''); +paragraph('通常ぎ4 MiBæœĒæē€ãŽMarkdown11æœŦ×3æŽĩ階は、äŋŽæ­Ŗ垌も7候čŖœãŽå…¨ãĻã§åˆ†å‰˛åĸƒį•Œã¨IDãŒä¸€č‡´ã—ãŸã€‚customChunkSizeを上げãĻもこぎ領域ぎID数は減らãĒい。新しいį‹ŦįĢ‹IDéĩæ–šåŧãŽIDは67æ–‡å­—ã§ã€æ—§æš—åˇåŒ–xxhash64比čŧƒã¯14〜16文字、中夎値16文字だãŖた。IDå‘Ŋ名ぎ変更がIDをįŸ­ãã—たという前提はįŊŽã‘ãĒい。'); +paragraph('0/1ではバイナãƒĒãƒģ大きいテキ゚トぎ最小値が最大値へクナãƒŗプされ、100/200 KiBぎå›ēåŽšåˆ†å‰˛ãĢãĒるため、įĒ“äŋŽæ­Ŗ垌もæŒŋå…Ĩ時ぎ再同期ぎ刊į‚šã¯å‡ēãĒã„ã€‚å°ã•ã„åą€æ‰€ä¸Šæ›¸ãã ã‘ãĒら0が有刊だが、äģŠå›žãŽæŒŋå…ĨをåĢむįˇ¨é›†ã§ã¯4ぎ斚が大嚅ãĢ少ãĒく済む。40と60は原čŖ…上ぎ4 MiB上限ãĢ達し、全15ãƒ•ã‚Ąã‚¤ãƒĢ×3æŽĩéšŽã§åˆ†å‰˛åĸƒį•Œã¨IDãŒåŽŒå…¨ä¸€č‡´ã—ãŸã€‚60ãŽé‡č¤‡ã™ã‚‹ãƒãƒƒãƒˆãƒ¯ãƒŧクæ¸Ŧ厚はįœã„た。'); +for (const provider of ['couchdb', 's3']) { + const four = getRemote(summary, provider, 4, 'initial').upload_requestBodyBytes.median; + const forty = getRemote(summary, provider, 40, 'initial').upload_requestBodyBytes.median; + paragraph(`${provider}で4から40へåĸ—やした初回送äŋĄé‡ãŽå‰Šæ¸›ã¯ ${((1 - forty / four) * 100).toFixed(3)}%į¨‹åēĻだãŖた。上書きãƒģæŒŋå…Ĩでは40ぎ送äŋĄé‡ãŒå¤§ããã€äģŠå›žãŽæĄäģļから大きいチãƒŖãƒŗクをįļ­æŒã™ã‚‹æ€§čƒŊ上ぎ栚拠は垗られãĒかãŖた。`); +} +paragraph('S3ではおぎ候čŖœã‚‚初回7ã‚Ēブジェクトãƒģ14 HTTPčĻæą‚だãŖた。4と40はįˇ¨é›†æ™‚もそれぞれ1ã‚ĒブジェクトčŋŊ加ãƒģ6 HTTPčĻæą‚で、chunk数ぎ削減がそぎぞぞPUT削減ãĢãĒる構成ではãĒい。AWSį­‰ãŽåŽŸã‚ĩãƒŧビ゚ぎčĒ˛é‡‘ã‚„WAN性čƒŊをæ¸ŦãŖた値ではãĒい。P2Pでは回ごとぎ再送į­‰ãŽå¤‰å‹•ãŒã‚るため、中夎値だけでãĒくsummary.jsonぎmin/maxも参į…§ã™ã‚‹ã€‚150msぎ断į‰‡å†čĻæą‚ã¨ãŽå› æžœã¯č¨ˆæ¸ŦしãĻおらず、こぎå‡Ļį†ã¯å¤‰æ›´ã—ãĻいãĒい。'); +paragraph('CloudantぎįžčĄŒIBMå…Ŧåŧlimitsīŧˆ2026-10-23įĸēčĒīŧ‰ã¯ã€å˜ä¸€JSON文書1 MB、ãƒĒクエ゚ト全äŊ“64MB、æˇģäģ˜10 MB。https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-limits 。chunksはæˇģäģ˜ã§ã¯ãĒくJSON文書としãĻ送る。ãƒĸデãƒĢãĢはäŋåŽˆįš„ãĢ1,000,000 / 64,000,000 bytesをäŊŋãŖãŸã€‚é€šå¸¸ãŽæš—åˇåŒ–æĄäģļではCouchDBãĢ文書上限だけをかけ、4は15ãƒ•ã‚Ąã‚¤ãƒĢぎ垊元ぞで成功、10は413とãĒãŖた。4ぎ最大äŋå­˜æ–‡æ›¸ã¯910,461 bytes、通常HTTPæ¸Ŧ厚ぎ最大ãƒĒクエ゚トはį´„10.27 MBã§ã€ä¸Ąæ–šãŽä¸Šé™å†…ã ãŖた。'); +paragraph('åˆĨぎåĸƒį•Œprobeã§ã¯ã€åŽŸéš›ãŽåˆ†å‰˛å™¨ãƒģ圧į¸Žãƒģæš—åˇåŒ–é–ĸ数をį”¨ã„、ãƒŦプãƒĒã‚ąãƒŧã‚ˇãƒ§ãƒŗぎrevisionフã‚ŖãƒŧãƒĢドをåĢむ合成leafを送äŋĄã—た。įĩæžœã¯æŦĄãŽã¨ãŠã‚Šã€‚'); +lines.push('å…Ĩ力 | value | æš—åˇåŒ– | 圧į¸Ž | JSON文書bytes | CouchDB HTTP status'); +for (const row of boundary.cases) lines.push([row.kind, row.customChunkSize, row.encryption, row.compression, row.jsonBytes, row.status].join(' | ')); +lines.push(''); +paragraph('値4ぎバイナãƒĒはraw 512,000 bytes → base64 682,668 bytes → E2EE V1/V2でį´„910.5 KBとãĒãŖた。値5ぎE2EE V2は1,092,573 bytesとãĒり、文書上限で413がčŋ”ãŖた。したがãŖãĻæš—åˇåŒ–ãƒã‚¤ãƒŠãƒĒãĢ寞しãĻ4は整数ぎ候čŖœã¨ã—ãĻ上限ãĢčŋ‘く、äŊ™čŖ•ã‚’掋す候čŖœã§ã‚る。'); +paragraph('ただし、CouchDB 3.5.0ぎmax_document_sizeはparsed document bodyをcouch_ejson_size:encoded_sizeでæĻ‚įŽ—し、文字列はUTF-8長ãĢ2をčļŗすだけでJSON escapeを数えãĒい。こぎためNULぎæœĒæš—åˇåŒ–ãƒ†ã‚­ã‚šãƒˆã¯wire上3,072,213 bytesでも、ãƒĸデãƒĢぎ1 MB文書åˆļ限では受į†ã•ã‚ŒãŸã€‚Cloudant内部ぎįŽ—厚が厌全ãĢ同一であるとはįĸēčĒã—ãĻいãĒい。単į´”ãĢwire JSONが1 MBčļ…ãĒらåŋ…ずCloudantぎ文書åˆļé™ã§å¤ąæ•—ã™ã‚‹ã€ã¨ã“ãŽįĩæžœã‹ã‚‰æ–­åŽšã—ãĻはいけãĒい。å…ŦåŧCouchDBã‚Ŋãƒŧ゚はcontext/couch_doc-3.5.0.erlとcontext/couch_ejson_size-3.5.0.erlãĢäŋå­˜ã—た。'); +paragraph('一æ‹ŦãƒĒクエ゚トはåˆĨぎåˆļé™ã§ã‚ã‚‹ã€‚åˆ†å‰˛å™¨ãŒäŊœãŖた500 KiBぎį•°ãĒるåˆļåžĄæ–‡å­—ãƒ†ã‚­ã‚šãƒˆ25個をぞとめると76,805,358 bytesとãĒり、64,000,000-byte requeståˆļ限で413ãĢãĒãŖた。20個ãĒら61,444,288 bytesで受į†ã•ã‚ŒãŸã€‚これは合成bulk requestぎåĸƒį•Œprobeであり、ã‚ĸプãƒĒã‚ąãƒŧã‚ˇãƒ§ãƒŗぎ通常同期全äŊ“で同じbulkがįĩ„ぞれると原č¨ŧしたテ゚トではãĒい。įžåœ¨ãŽé€šå¸¸Couch同期ぎbatch_size=25ã§ã¯ã€æš—åˇåŒ–æ–‡æ›¸ãŒ1 MBäģĨ内ãĒら25 MBį¨‹åēĻã ãŒã€æš—åˇåŒ–ã—ãĒいテキ゚トぎescape膨åŧĩãĢはäģļ数åˆļ限だけでは十分でãĒい。optional bulk chunk sendãĢはį‹Ŧč‡Ēぎbyteåˆļ限もあるぎで、そぎįĩŒčˇ¯ã¨é€šå¸¸Pouch replicationぎįĩŒčˇ¯ã‚’åŒēåˆĨするåŋ…čĻãŒã‚る。'); +paragraph('CloudantãƒĸデãƒĢはロãƒŧã‚ĢãƒĢCouchDBでåˆļé™ã‚’æ¨Ąã—ãŸã‚‚ãŽã§ã‚ã‚Šã€IBM Cloudantã‚ĩãƒŧビ゚č‡ĒäŊ“ぎ同期ãƒģ性čƒŊはæ¸Ŧ厚しãĻいãĒい。全probe垌ãĢ严åˆļé™č¨­åŽšã‚’åžŠå…ƒã—ã€åˆæˆæ–‡æ›¸ãŽDBも削除した。Cloudantぎäē’換性をį´„束するãĒら、leafだけでãĒくMetadataをåĢむ最įĩ‚文書とバッチぎ原ã‚ĩイã‚ēã‚‚æ–šé‡ãŽå¯žčąĄãĢするぎがåĻĨåŊ“である。'); +paragraph(`旧į‰ˆã¨äŋŽæ­Ŗį‰ˆãŽæˇˇåœ¨ã¯ã€åŒã˜æ—§äžå­˜ãƒĒãƒŗクからビãƒĢドを垊元し、旧entryが前回ぎäŋå­˜į‰Šã¨åŽŒå…¨ä¸€č‡´ã™ã‚‹ã“とをįĸēčĒã—ãĻ原æ¸Ŧしたīŧˆ${recovery.originalEntryBitIdenticalToBaselineArchive}īŧ‰ã€‚旧→新、新→旧ぎ各3æŽĩ階×15ãƒ•ã‚Ąã‚¤ãƒĢã€č¨ˆ${validation.mixedVersionVerifiedFileStages}äģļでSHA-256ãŒä¸€č‡´ã—ãŸã€‚ãƒãƒŖãƒŗクぎäŋå­˜åŊĸåŧã¨IDæ–šåŧã¯åŧ•ãįļšãčĒ­ã‚ã‚‹ã€‚再刊į”¨åŠšįŽ‡ã‚’揃えるãĢは書čžŧãŋįĢ¯æœĢãŽåˆ†å‰˛åŽŸčŖ…も揃える斚がよい。最初ぎäē’換性čŠĻčĄŒã¯æ—§bundleぎäģ˜éšCJS不čļŗã§åŽŸčĄŒå‰ãĢå¤ąæ•—ã—ã€ããŽãƒ­ã‚°ã¨åžŠå…ƒč¨˜éŒ˛ã‚’äŋå­˜ã—た。垊元垌ぎ全čŠĻčĄŒã¯æˆåŠŸã—ãŸã€‚`); +const cutover = comparison.cutover.find(row => row.customChunkSize === 40); +paragraph(`įĒ“ぎäŋŽæ­Ŗã§åˆ†å‰˛åĸƒį•ŒãŒå¤‰ã‚ã‚‹ãŸã‚ã€æ—ĸå­˜ãƒ•ã‚Ąã‚¤ãƒĢを再äŋå­˜ã™ã‚‹åˆå›žãĢはčŋŊ加チãƒŖãƒŗクが多くåŋ…čĻãĢãĒり垗る。ロãƒŧã‚ĢãƒĢぎ同じ初期デãƒŧã‚ŋを値40で比čŧƒã™ã‚‹ã¨ã€æ–°æ–šåŧãŽunique raw ${mib(cutover.totalNewAlgorithmUniqueRawBytes)} MiB中、旧初期chunkãĢ存在しãĒい部分は ${mib(cutover.bytesNotPresentInOldAlgorithmInitialChunks)} MiBだãŖた。これは内厚/IDį…§åˆãĢよるįŽ—厚で、原際ぎã‚ĸップグãƒŦãƒŧド時ぎčģĸ送量をæ¸Ŧ厚した値ではãĒい。äģĨ垌ぎ同じ斚åŧã§ãŽįˇ¨é›†ãĢ寞する攚善とは分けãĻæ‰ąã†ã€‚`); +paragraph('č¨­åŽšæ•´į†ã¸ãŽææĄˆã¯ã€é€šå¸¸ãŽæ–°čĻäŊœæˆãƒģ推åĨ¨å€¤ã‚’remoteį¨ŽåˆĨãĢよらず4ã¸æƒãˆã‚‹æ–šå‘ã§é€˛ã‚ã€æ—§č¨­åŽšãŽįœį•Ĩ値やæ—ĸ存remoteで合意済ãŋぎtweakを一型ãĢ書き換えãĒいことである。60と10を標æē–候čŖœã¨ã—ãĻ分ける栚拠はäģŠå›žåŧąã„。一斚、Cloudantへ将æĨ同期する意å›ŗを問うãĒら、įžåœ¨ãŽæŽĨįļšå…ˆã§ã¯ãĒくvault全äŊ“ぎäē’換斚針としãĻæ‰ąã„ã€å…¨æ›¸čžŧãŋįĢ¯æœĢで厈るåŋ…čĻãŒã‚ã‚‹ã€‚æš—åˇåŒ–ãƒģJSON化ãƒģバッチ化垌ぎ上限を内部で導å‡ēし、ãƒĻãƒŧã‚ļãƒŧへįĩ„合せぎ数字を選ばせることはéŋけたい。'); +paragraph('įžåœ¨ãŽremote profile URIはæŽĨįļšæŦ„だけをåĢãŋ、customChunkSizeã¯å…ąé€šč¨­åŽšã§ã‚ã‚‹ã€‚é€šå¸¸ãŽprofile切æ›ŋでそぎ値を変えãĒいが、čĩˇå‹•æ™‚ぎCloudantæŽĨįļšå…ˆčŖœæ­Ŗやpreferred tweakæŽĄį”¨ã§ã¯å¤‰ã‚ã‚‹įĩŒčˇ¯ãŒã‚る。äŋå­˜æ¸ˆãŋleafはčģĸ送時ãĢå†åˆ†å‰˛ã•ã‚ŒãĒい。Cloudant寞åŋœã‚’垌から有劚化しãĻ値だけ下げãĻも、æ—ĸ存ぎ大きいleafがそぎぞぞ掋るため、äē’換斚針を変えるį§ģčĄŒã§ã¯æ—ĸ存DBぎ検æŸģãƒģ再構成もåˆĨé€”č¨­č¨ˆã™ã‚‹åŋ…čĻãŒã‚る。これはäģŠå›žæœĒ原čŖ…ãŽč¨­č¨ˆæĄˆã§ã‚ã‚‹ã€‚'); +paragraph(`検č¨ŧįĩæžœã¯ã€ä¸ģæ¸Ŧ厚42ã‚ąãƒŧã‚šãƒģ126æŽĩ階×15ãƒ•ã‚Ąã‚¤ãƒĢ=${validation.verifiedFileStages}äģļぎSHA-256ä¸€č‡´ã€äŋå­˜åžŒč¨­åŽš${validation.checkedPersistedConfigurations}äģļãŽæĄäģļä¸€č‡´ã€æˇˇåœ¨į‰ˆ${validation.mixedVersionVerifiedFileStages}äģļぎSHA-256ä¸€č‡´ã€‚é€šå¸¸Markdown、40/60åŒä¸€ã€å…¨åˆ†å‰˛ãŽå†æ§‹æˆã€ååžŠã§ãŽIDãƒģåĸƒį•Œå†įžæ€§ã€ãŠã‚ˆãŗ同じIDãŽå†…åŽšä¸€č‡´ã‚‚æˆåŠŸã—ãŸã€‚`); +paragraph('äŋå­˜į‰Šã¯į”ŸJSON、summary/CSV、comparison.json、validation.json、全å…Ĩ力corpus、各caseぎログãƒģč¨­åŽšãƒģP2Pã‚Ģã‚Ļãƒŗã‚ŋãƒŧ、scripts、contextぎ厌全ãĒ旧/新CLI bundleとlockfile、commonlibぎäŋŽæ­ŖåˇŽåˆ†ãƒģ回帰テ゚トãƒģ検č¨ŧログãƒģtarball、およãŗartifacts.sha256ã§ã‚ã‚‹ã€‚åˇ¨å¤§ãĒäŊœæĨ­į”¨DB複čŖŊはåĢめず、再į”Ÿæˆã§ãã‚‹å…Ĩ力とč¨ŧ拠をäŋå­˜ã™ã‚‹ã€‚æ—§baselineはčĻĒデã‚ŖãƒŦクトãƒĒãƒŧãĢそぎぞぞ掋しãĻいる。'); +paragraph('再įžæ™‚はLiveSyncå›ē厚ã‚ŗミットとCommonlibå›ē厚ã‚ŗミットぎäŊœæĨ­ãƒ„ãƒĒãƒŧをį”¨æ„ã—、Commonlibぎproduction.patchとsource内ぎ回帰テ゚トを反映する。Commonlibぎ元ãƒĒポジトãƒĒãƒŧで検č¨ŧし、äŋå­˜æ¸ˆãŋtarballと同じ原äŊ“ã‚’LiveSyncãĢfile䞝存としãĻå…Ĩれる。scriptsをLiveSyncぎutils/chunk-size-benchmarkへįŊŽãã€npm 10 ci、通常npm ci、check、unit、全app buildぎ順ãĢåŽŸčĄŒã™ã‚‹ã€‚æ—§į‰ˆãŽå†æ§‹į¯‰ãĢはå…Ŧ開0.1.44ãƒ‘ãƒƒã‚ąãƒŧジをäŊŋã„ã€åŽŸčĄŒæœŦäŊ“だけでãĒくdist全äŊ“ã‚’åˆĨデã‚ŖãƒŦクトãƒĒãƒŧへ掋す。新しいå‡ē力先とLIVESYNC_BENCH_VARIANT=rolling-window-fixedをäŊŋãŖãĻlocal/remote/cloudant probes/compatibility/p2pを逐æŦĄåŽŸčĄŒã—、summarise、compare、verify-results、reportぎ順ãĢé›†č¨ˆã™ã‚‹ã€‚ãƒ­ãƒŧã‚ĢãƒĢ専į”¨fixtureぎčĩˇå‹•ãƒģ停æ­ĸãĢもcomposeぎproject名を指厚する。'); +await fs.writeFile(join(directory, 'report.txt'), lines.join('\n')); +console.log(JSON.stringify({ report: join(directory, 'report.txt'), bytes: Buffer.byteLength(lines.join('\n')) })); diff --git a/utils/chunk-size-benchmark/summarise.mjs b/utils/chunk-size-benchmark/summarise.mjs new file mode 100644 index 00000000..4ece8484 --- /dev/null +++ b/utils/chunk-size-benchmark/summarise.mjs @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import { join, resolve } from 'node:path'; + +const directory = resolve(process.argv[2] || '.bench/chunk-size'); +const local = JSON.parse(await fs.readFile(join(directory, 'results.json'), 'utf8')); +const optionalJson = async (path) => { try { return JSON.parse(await fs.readFile(path, 'utf8')); } catch (error) { if (error.code === 'ENOENT') return undefined; throw error; } }; +const remote = await optionalJson(join(directory, 'remote/results-rtt0.json')); +const p2p = await optionalJson(join(directory, 'p2p/results.json')); +const median = (numbers) => { const sorted = [...numbers].sort((a, b) => a - b); const mid = Math.floor(sorted.length / 2); return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; }; +const stats = (numbers) => ({ n: numbers.length, median: median(numbers), min: Math.min(...numbers), max: Math.max(...numbers) }); + +let normalMarkdownFileStages = 0; +for (const stage of local.candidates[0].stages) { + for (const file of stage.files.filter((file) => file.kind === 'markdown' && file.sourceBytes < 4 * 1024 * 1024)) { + const fingerprints = local.candidates.map((candidate) => candidate.stages.find((entry) => entry.name === stage.name).files.find((entry) => entry.path === file.path).chunkFingerprintSha256); + assert.equal(new Set(fingerprints).size, 1, `${stage.name}/${file.path}`); normalMarkdownFileStages++; + } +} + +const localRows = local.candidates.flatMap((candidate) => candidate.stages.map((stage) => { + const binary = stage.byKind.find((group) => group.kind === 'binary'); + const markdown = stage.byKind.find((group) => group.kind === 'markdown'); + const random = stage.files.find((file) => file.path === 'benchmark/random-8m.bin'); + return { + customChunkSize: candidate.customChunkSize, stage: stage.name, rawCeilingBytes: candidate.chunkSizeCeilingBytes, + allChunks: stage.files.reduce((sum, file) => sum + file.chunkCount, 0), binaryChunks: binary.chunkCount, + randomBinaryChunks: random.chunkCount, randomBinaryCpuMs: random.timing.primarySplitAndIdCpuMsMedian, + binaryNewRawBytes: binary.newUniqueChunkRawBytesVsHistory, markdownNewRawBytes: markdown.newUniqueChunkRawBytesVsHistory, + splitAndIdCpuMs: binary.timing.primarySplitAndIdCpuMsMedian + markdown.timing.primarySplitAndIdCpuMsMedian, + }; +})); + +function grouped(entries, makeRow) { + const groups = new Map(); + for (const entry of entries) for (const stage of entry.stages) { + const key = `${entry.provider || 'p2p'}/${entry.customChunkSize}/${stage.name}`; + const group = groups.get(key) || { provider: entry.provider || 'p2p', customChunkSize: entry.customChunkSize, stage: stage.name, samples: [] }; + group.samples.push(stage); groups.set(key, group); + } + return [...groups.values()].map(makeRow); +} + +const remoteRows = grouped((remote?.results || []).filter((entry) => entry.complete), (group) => { + const result = { provider: group.provider, customChunkSize: group.customChunkSize, stage: group.stage, repeats: group.samples.length }; + for (const direction of ['mirror', 'upload', 'download']) { + for (const metric of ['elapsedMs', 'userCpuMs', 'systemCpuMs', 'maxResidentSetKiB']) result[`${direction}_${metric}`] = stats(group.samples.map((stage) => stage[direction][metric])); + if (direction !== 'mirror') for (const metric of ['requests', 'requestBodyBytes', 'responseBodyBytes', 'maxRequestBodyBytes']) { + const values = group.samples.map((stage) => stage[direction].http[metric]).filter(Number.isFinite); + if (values.length) result[`${direction}_${metric}`] = stats(values); + } + } + for (const metric of ['objectCount', 'storedObjectBytes', 'chunkCount', 'newChunkCount', 'chunkJsonBytes', 'largestChunkJsonBytes', 'largestDocumentJsonBytes']) { + const values = group.samples.map((stage) => stage.remote[metric]).filter(Number.isFinite); + if (values.length) result[metric] = stats(values); + } + result.verifiedFilesPerSample = group.samples.map((stage) => stage.verifiedFiles); + return result; +}); + +const p2pRows = grouped((p2p?.results || []).filter((entry) => entry.complete), (group) => { + const result = { provider: 'p2p', customChunkSize: group.customChunkSize, stage: group.stage, repeats: group.samples.length }; + for (const metric of ['elapsedMs', 'userCpuMs', 'systemCpuMs', 'maxResidentSetKiB']) result[metric] = stats(group.samples.map((stage) => stage.sync[metric])); + result.currentRoundTripTime = stats(group.samples.map((stage) => stage.stats.selectedPair.currentRoundTripTime)); + for (const side of ['host', 'target']) for (const metric of ['payloadBytesSent', 'messagesSent']) result[`${side}_${metric}`] = stats(group.samples.map((stage) => stage.dataChannel[side][metric])); + for (const side of ['host', 'target']) for (const metric of ['userCpuMs', 'systemCpuMs', 'maxResidentSetKiB']) result[`${side}_${metric}`] = stats(group.samples.map((stage) => stage.dataChannel[side].processUsage[metric])); + result.paths = [...new Set(group.samples.map((stage) => stage.stats.selectedPath))]; + result.verifiedFilesPerSample = group.samples.map((stage) => stage.verifiedFiles); + return result; +}); + +const summary = { version: local.runtime.projectVersion, commonlib: local.runtime.commonlibVersion, variant: local.runtime.variant, commonlibChunksSha256: local.runtime.commonlibChunksSha256, corpus: local.corpus, + assertions: { ...local.assertions, normalMarkdownIdenticalAcrossAllSevenValues: true, normalMarkdownFileStages }, local: localRows, remote: remoteRows, p2p: p2pRows }; +await fs.writeFile(join(directory, 'summary.json'), JSON.stringify(summary, null, 2)); + +function csv(rows, fields) { + const cell = (value) => JSON.stringify(typeof value === 'object' && value !== null ? value.median : value ?? ''); + return `${fields.join(',')}\n${rows.map((row) => fields.map((field) => cell(row[field])).join(',')).join('\n')}\n`; +} +await fs.writeFile(join(directory, 'local-summary.csv'), csv(localRows, Object.keys(localRows[0]))); +if (remoteRows.length) await fs.writeFile(join(directory, 'remote-summary.csv'), csv(remoteRows, Object.keys(remoteRows[0]).filter((key) => key !== 'verifiedFilesPerSample'))); +if (p2pRows.length) await fs.writeFile(join(directory, 'p2p-summary.csv'), csv(p2pRows, Object.keys(p2pRows[0]).filter((key) => !['paths', 'verifiedFilesPerSample'].includes(key)))); +console.log(JSON.stringify({ directory, completeRemoteCases: remote?.results.filter((entry) => entry.complete).length || 0, completeP2PCases: p2p?.results.filter((entry) => entry.complete).length || 0, assertions: summary.assertions })); diff --git a/utils/chunk-size-benchmark/verify-results.mjs b/utils/chunk-size-benchmark/verify-results.mjs new file mode 100644 index 00000000..0c3bcd42 --- /dev/null +++ b/utils/chunk-size-benchmark/verify-results.mjs @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import { join, resolve } from 'node:path'; + +const directory = resolve(process.argv[2] || '.bench/rabin-window-fixed'); +const load = async name => JSON.parse(await fs.readFile(join(directory, name), 'utf8')); +const local = await load('results.json'); +const summary = await load('summary.json'); +const remote = await load('remote/results-rtt0.json'); +const p2p = await load('p2p/results.json'); +const manifest = await load('corpus/manifest.json'); +const sha256 = value => createHash('sha256').update(value).digest('hex'); +assert.equal(remote.corpusDigest, p2p.corpusDigest); +assert.equal(remote.corpusDigest, sha256(JSON.stringify(manifest.stages.map(stage => ({ name: stage.name, files: stage.files }))))); +assert.equal(local.corpus.manifestSha256, sha256(JSON.stringify(manifest))); +assert.equal(remote.results.length, 30); +assert.equal(p2p.results.length, 12); +let verified = 0; +let configurations = 0; +for (const [subdirectory, result] of [['remote', remote], ['p2p', p2p]]) { + assert(result.variant === 'rolling-window-fixed'); + const groups = new Map(); + for (const entry of result.results) { + assert(entry.complete && !entry.error, entry.name); + assert.equal(entry.stages.length, 3); + const key = `${entry.provider || 'p2p'}/${entry.customChunkSize}`; + groups.set(key, (groups.get(key) || 0) + 1); + for (const stage of entry.stages) { assert.equal(stage.verifiedFiles, 15); verified += stage.verifiedFiles; } + for (const side of ['source', 'target']) { + const settings = await load(join(subdirectory, entry.name, `${side}-settings.json`)); + assert.equal(settings.customChunkSize, entry.customChunkSize); + assert.equal(settings.chunkSplitterVersion, 'v3-rabin-karp'); + assert.equal(settings.minimumChunkSize, 20); + assert.equal(settings.idDerivationVersion, 1); + assert.equal(settings.encrypt, true); + assert.equal(settings.E2EEAlgorithm, 'v2'); + assert.equal(settings.useEden, false); + assert.equal(settings.enableCompression, false); + assert.equal(settings.autoAcceptCompatibleTweak, false); + assert.equal(settings.batch_size, 25); + configurations++; + } + } + assert([...groups.values()].every(count => count === 3)); +} +assert.equal(verified, 1890); +assert.equal(configurations, 84); +let mixedVerified = 0; +const compatibility = []; +for (const name of ['compatibility-old-to-new', 'compatibility-new-to-old']) { + const result = await load(`${name}/remote/results-rtt0.json`); + const completed = result.results.filter(entry => entry.complete); + assert.equal(completed.length, 1); + assert.notEqual(result.sourceCliSha256, result.targetCliSha256); + const entry = completed[0]; + assert(entry.complete && !entry.error); + assert.equal(entry.stages.length, 3); + for (const stage of entry.stages) { assert.equal(stage.verifiedFiles, 15); mixedVerified += stage.verifiedFiles; } + compatibility.push({ name, sourceCliSha256: result.sourceCliSha256, targetCliSha256: result.targetCliSha256, verifiedFiles: 45, retainedIncompleteSetupAttempts: result.results.filter(entry => !entry.complete).length }); +} +const model = await load('cloudant-model/constraint-result.json'); +assert(model.configurationRestored && model.sizeRejectionObserved); +assert(model.cases.some(entry => entry.customChunkSize === 4 && entry.complete)); +assert(model.cases.some(entry => entry.customChunkSize === 10 && !entry.complete)); +const boundary = await load('cloudant-json-boundary/results.json'); +assert(boundary.configurationRestored && boundary.requestConfigurationRestored); +assert.equal(boundary.cases.length, 8); +assert.equal(boundary.batches.length, 2); +assert(boundary.batches.some(entry => entry.count === 25 && entry.status === 413 && !entry.accepted)); +assert(boundary.batches.some(entry => entry.count === 20 && entry.accepted)); +assert(Object.values(local.assertions).every(value => value === true || typeof value === 'number')); +const result = { completedRemoteCases: 30, completedP2PCases: 12, repeatsPerCondition: 3, + verifiedFileStages: verified, checkedPersistedConfigurations: configurations, + sameCorpusDigestForRemoteAndP2P: true, localAssertions: summary.assertions, + compatibility, mixedVersionVerifiedFileStages: mixedVerified, + cloudantConstraintModel: { candidate4Completed: true, candidate10RejectedWith413: true, fixtureConfigurationRestored: true }, + cloudantJSONBoundary: { cases: boundary.cases.length, batches: boundary.batches.length, request25RejectedWith413: true, request20Accepted: true, fixtureConfigurationRestored: true, cloudantServiceMeasured: false }, + rabinWindowDiagnostic: { command: 'node --test utils/chunk-size-benchmark/rabin-window.test.mjs', exitCode: 0, productionFixApplied: true }, +}; +await fs.writeFile(join(directory, 'validation.json'), JSON.stringify(result, null, 2)); +console.log(JSON.stringify(result)); diff --git a/utils/flyio/generate_setupuri.test.ts b/utils/flyio/generate_setupuri.test.ts index 7c35f03e..bae7ab1b 100644 --- a/utils/flyio/generate_setupuri.test.ts +++ b/utils/flyio/generate_setupuri.test.ts @@ -1,6 +1,7 @@ import { decodeSettingsFromSetupURI, DEFAULT_SETTINGS, + PREFERRED_SETTING_SELF_HOSTED, } from "../setup/livesync-commonlib.ts"; function assert(condition: unknown, message: string): asserts condition { @@ -9,8 +10,14 @@ function assert(condition: unknown, message: string): asserts condition { Deno.test("generates a current self-hosted Setup URI through the published Commonlib contract", async () => { const scriptPath = new URL("./generate_setupuri.ts", import.meta.url); + const candidateConfig = Deno.env.get("LIVESYNC_TEST_COMMONLIB_CONFIG"); const command = new Deno.Command(Deno.execPath(), { - args: ["run", "-A", scriptPath.pathname], + args: [ + "run", + "-A", + ...(candidateConfig ? ["--config", candidateConfig, "--no-lock"] : []), + scriptPath.pathname, + ], env: { hostname: "https://couch.example.test", username: "alice", @@ -38,7 +45,9 @@ Deno.test("generates a current self-hosted Setup URI through the published Commo const recoveryCode = stdout.match(/sls-id-v1:[0-9a-f]{64}/u)?.[0]; assert(recoveryCode, "the generator did not print an ID recovery code"); assert( - (effectiveSettings as typeof effectiveSettings & { idDerivationKey?: string }).idDerivationKey === + (effectiveSettings as typeof effectiveSettings & { + idDerivationKey?: string; + }).idDerivationKey === recoveryCode.slice("sls-id-v1:".length), "the CouchDB Setup URI did not contain the generated ID key", ); @@ -47,8 +56,9 @@ Deno.test("generates a current self-hosted Setup URI through the published Commo "the CouchDB Setup URI left the imported device unconfigured", ); assert( - effectiveSettings.customChunkSize === 60, - "the Setup URI did not use the current self-hosted chunk-size recommendation", + effectiveSettings.customChunkSize === + PREFERRED_SETTING_SELF_HOSTED.customChunkSize, + "the Setup URI did not use the self-hosted chunk-size preference from Commonlib", ); assert( effectiveSettings.chunkSplitterVersion === "v3-rabin-karp", diff --git a/utils/setup/generate_setup_uri.test.ts b/utils/setup/generate_setup_uri.test.ts index 2bf802ff..99e11e30 100644 --- a/utils/setup/generate_setup_uri.test.ts +++ b/utils/setup/generate_setup_uri.test.ts @@ -1,6 +1,7 @@ import { decodeSettingsFromSetupURI, DEFAULT_SETTINGS, + PREFERRED_JOURNAL_SYNC, } from "./livesync-commonlib.ts"; import { generateSetupURI } from "./generate_setup_uri.ts"; @@ -55,8 +56,8 @@ Deno.test("generates an Object Storage Setup URI with a selected S3 profile", as "the Setup URI left the imported device unconfigured", ); assert( - effective.customChunkSize === 10, - "the journal chunk-size preset was not applied", + effective.customChunkSize === PREFERRED_JOURNAL_SYNC.customChunkSize, + "the Object Storage Setup URI did not use its Commonlib journal chunk-size preference", ); assert( effective.liveSync, @@ -122,7 +123,7 @@ Deno.test("generates a random-room P2P Setup URI without copying a device identi ); assert( effective.customChunkSize === 0, - "the P2P profile inherited the self-hosted CouchDB chunk-size recommendation", + "the P2P Setup URI did not retain its zero chunk-size value", ); assert( effective.sendChunksBulkMaxSize === 1, From 22ef4d073f1a30d49b5eb7034e48b02d42b16d81 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 09:39:18 +0000 Subject: [PATCH 33/68] Document module filename capitalisation --- devs.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devs.md b/devs.md index a4d3849d..4e820e90 100644 --- a/devs.md +++ b/devs.md @@ -204,6 +204,9 @@ The [multiple-device conflict test procedure](test/README.md#multiple-device-con ### File Structure Conventions +- **Class modules**: Start a new or renamed TypeScript or JavaScript filename with an upper-case letter when the file contains a class with the same name. +- **Function modules**: Start the filename with a lower-case letter when the module primarily exports functions, such as `initialFetchFastForward.ts`. Follow the surrounding filename style for the rest of the basename. Corresponding test files use the same basename. +- **Existing files**: Retain established names for unrelated files and separate conventions for components or other file types. - **Platform-specific code**: Use `.platform.ts` suffix (replaced with `.obsidian.ts` in production builds via esbuild) - **Development code**: Use `.dev.ts` suffix (replaced with `.prod.ts` in production) - **Path aliases**: `@/*` maps to `src/*`; Commonlib uses package exports rather than a source alias From 404d4279c0a0113e29c0272d5ae40b1b6485b57b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 10:13:46 +0000 Subject: [PATCH 34/68] Add Joined devices dialogue and version notices --- devs.md | 8 + docs/glossary.md | 4 + docs/settings.md | 10 + package.json | 1 + .../messages/LiveSyncProvisionalMessages.ts | 56 +++ src/common/replicatorProviders.ts | 8 + src/common/replicatorProviders.unit.spec.ts | 10 +- src/common/replicatorResources/index.ts | 5 + .../replicatorResources/joinedDevices.ts | 40 ++ src/main.ts | 2 + .../SettingDialogue/PaneRemoteConfig.ts | 17 + .../PaneRemoteConfig.unit.spec.ts | 27 +- .../dialogues/JoinedDevicesDialog.svelte | 391 +++++++++++++++ src/serviceFeatures/joinedDevices.ts | 64 +++ .../joinedDevices.unit.spec.ts | 69 +++ src/serviceFeatures/joinedDevicesCommand.ts | 5 + .../joinedDevicesDialogTypes.ts | 19 + src/serviceFeatures/useJoinedDevices.ts | 212 ++++++++ .../useJoinedDevices.unit.spec.ts | 174 +++++++ test/e2e-obsidian/README.md | 2 + test/e2e-obsidian/scripts/joined-devices.ts | 462 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + updates.md | 8 + 23 files changed, 1589 insertions(+), 6 deletions(-) create mode 100644 src/common/replicatorResources/joinedDevices.ts create mode 100644 src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte create mode 100644 src/serviceFeatures/joinedDevices.ts create mode 100644 src/serviceFeatures/joinedDevices.unit.spec.ts create mode 100644 src/serviceFeatures/joinedDevicesCommand.ts create mode 100644 src/serviceFeatures/joinedDevicesDialogTypes.ts create mode 100644 src/serviceFeatures/useJoinedDevices.ts create mode 100644 src/serviceFeatures/useJoinedDevices.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/joined-devices.ts diff --git a/devs.md b/devs.md index a4d3849d..5201c047 100644 --- a/devs.md +++ b/devs.md @@ -166,6 +166,14 @@ Legacy Modules remain grouped by directory: - `features/` contains optional features; and - `extras/` contains development and testing tools. +### UI commands and dialogue ownership + +Declare shared UI requests with `declareUICommand` from `@vrtmrz/livesync-commonlib/ui-commands`. The host's `services.UI.commands` registry owns one handler per declaration. A serviceFeature registers the handler, releases it on unload, and owns the associated dialogue lifetime. Callers receive a typed result or an explicit `unavailable` result; handler failures propagate. + +`OPEN_JOINED_DEVICES` is shared by Remote Configuration, version Notices, and the command palette. `useJoinedDevicesFeature` registers it once and gives the dialogue narrow refresh, forget, and cancellation callbacks. Each remote operation uses a settings snapshot and releases its finite resource in `finally`. Results from an old remote configuration or a closed dialogue are discarded. A write already sent to the captured remote can still complete after the dialogue closes. + +The registry belongs to one UIService instance. Reuse the same declaration object at each call site, and keep active UI cleanup in the feature and dialogue owner. `UIService` disposes the registry when the host's context emits the plug-in-unloaded event. + ### Key Architectural Components - **LiveSyncLocalDB** (`@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB`): Local PouchDB database wrapper diff --git a/docs/glossary.md b/docs/glossary.md index 6f3f1c3e..a498ce5e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -98,6 +98,10 @@ recovery guidance, or diagnostics intended for users. - **ID recovery code:** A versioned text form of the saved ID key which can be shown on the current device and imported without deriving a different key. Treat it as a secret. +- **Joined devices:** The dialogue showing device records held by the active + CouchDB or Object Storage remote, with the current device first. Records + describe participation, not current online activity. **Forget device** removes + an obsolete device record; it does not delete Vault files or revoke credentials. - **JWT Authentication:** An experimental CouchDB authentication option which uses a JSON Web Token instead of standard credentials. It requires a private key or secret, algorithm, expiry duration, subject, and key ID. diff --git a/docs/settings.md b/docs/settings.md index 0885589c..a64dbde6 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -234,6 +234,16 @@ Setting key: remoteType The active connection type. This is automatically projected to the legacy configuration when you activate a connection profile. +### Joined devices + +Open **Joined devices** to read the device records on the active CouchDB or Object Storage connection. The same dialogue is available from the command palette and version Notices. **Refresh** reads the remote again. This device appears first, including when its record is absent from the remote. Opening or refreshing this dialogue does not create a missing CouchDB database. + +The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. + +Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices are advisory and do not stop synchronisation. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. + +**Forget device** removes another device's participation records after confirmation. It preserves the remote lock state and every other device. It does not delete Vault files or revoke credentials. A device can register again on an unlocked remote; on a locked remote, the existing recovery process applies. The result is checked after saving to CouchDB; if it cannot be confirmed, refresh the list and try again. CouchDB stores these records together in a local document which does not reject simultaneous updates, so changes from other devices can be overwritten. Pausing synchronisation on the other devices avoids that overlap. Object Storage uses conditional updates; if the server cannot support them, the record is retained. + ### 2. Notification #### Notify when the estimated remote storage size exceeds on start up diff --git a/package.json b/package.json index a6ff13d4..7ea27ee8 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts", "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", + "test:e2e:obsidian:joined-devices": "tsx test/e2e-obsidian/scripts/joined-devices.ts", "test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts", "test:e2e:obsidian:database-diagnostics-couchdb": "tsx test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 34ae100b..2b809f40 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -7,6 +7,62 @@ * remove it from this map in the same change. */ export const liveSyncProvisionalEnglishMessages = { + "Unknown platform": "Unknown platform", + "This removes only this device's record from the remote.": + "This removes only this device's record from the remote.", + "This remote is unlocked. A device that returns can register again.": + "This remote is unlocked. A device that returns can register again.", + "This remote is locked. Use the existing recovery process for devices that need to join again.": + "This remote is locked. Use the existing recovery process for devices that need to join again.", + "This remote is locked. If the device returns, use the existing recovery process to join it again.": + "This remote is locked. If the device returns, use the existing recovery process to join it again.", + "This remote does not support conditional updates, so the device record could not be removed.": + "This remote does not support conditional updates, so the device record could not be removed.", + "This remote does not currently have a record of joined devices.": + "This remote does not currently have a record of joined devices.", + "This list shows device information recorded by the remote. It does not show whether a device is online now.": + "This list shows device information recorded by the remote. It does not show whether a device is online now.", + "This device is locked out. Use the existing recovery process to join again.": + "This device is locked out. Use the existing recovery process to join again.", + "This device": "This device", + "This remote has been cleaned. Use the existing recovery process to join devices again as needed.": + "This remote has been cleaned. Use the existing recovery process to join devices again as needed.", + "The device record was forgotten.": "The device record was forgotten.", + "The device record was already absent.": "The device record was already absent.", + "The device list changed while you were trying to forget this device. Refresh the list before trying again.": + "The device list changed while you were trying to forget this device. Refresh the list before trying again.", + "The device information could not be loaded. Close this dialogue and open Joined devices again if the remote configuration has changed.": + "The device information could not be loaded. Close this dialogue and open Joined devices again if the remote configuration has changed.", + "Review device records for the active CouchDB or Object Storage connection.": + "Review device records for the active CouchDB or Object Storage connection.", + Refresh: "Refresh", + "Plug-in version": "Plug-in version", + Platform: "Platform", + "Other joined devices last reported an older plug-in version: ${versions}.": + "Other joined devices last reported an older plug-in version: ${versions}.", + "Other joined devices last reported a newer plug-in version: ${versions}.": + "Other joined devices last reported a newer plug-in version: ${versions}.", + Open: "Open", + "Not recorded": "Not recorded", + "Not accepted": "Not accepted", + "No joined devices are recorded on this remote.": "No joined devices are recorded on this remote.", + "Locked out": "Locked out", + "Loading joined devicesâ€Ļ": "Loading joined devicesâ€Ļ", + "Last recorded": "Last recorded", + "Joined devices could not be updated. Refresh the list and try again. If the remote configuration has changed, close this dialogue and open Joined devices again.": + "Joined devices could not be updated. Refresh the list and try again. If the remote configuration has changed, close this dialogue and open Joined devices again.", + "Joined devices are available after database startup with an active CouchDB or Object Storage connection.": + "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", + "Joined devices": "Joined devices", + "Forgettingâ€Ļ": "Forgettingâ€Ļ", + "Forgetting the record does not delete Vault files or revoke credentials.": + "Forgetting the record does not delete Vault files or revoke credentials.", + "Forget this device record?": "Forget this device record?", + "Forget device": "Forget device", + Close: "Close", + Cancel: "Cancel", + "App version": "App version", + Accepted: "Accepted", "Preparing follow-up reportâ€Ļ": "Preparing follow-up reportâ€Ļ", "Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.", "Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.": diff --git a/src/common/replicatorProviders.ts b/src/common/replicatorProviders.ts index 845d8c2e..f3cc40b0 100644 --- a/src/common/replicatorProviders.ts +++ b/src/common/replicatorProviders.ts @@ -28,6 +28,8 @@ import { getObjectStorageReplicatorConfigurationIdentity, } from "./replicatorConfigurationIdentity"; import { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, createCouchDBConnectionProbeFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, @@ -115,6 +117,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getCouchDBReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncCouchDBReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createCouchDBJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability(createCouchDBConnectionProbeFactory(host)), [REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK]: supportedCapability( createCouchDBPreferredTweakProbeFactory(host) @@ -141,6 +146,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getObjectStorageReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncJournalReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createObjectStorageJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability( createObjectStorageConnectionProbeFactory(host) ), diff --git a/src/common/replicatorProviders.unit.spec.ts b/src/common/replicatorProviders.unit.spec.ts index 54a2c441..586e24d6 100644 --- a/src/common/replicatorProviders.unit.spec.ts +++ b/src/common/replicatorProviders.unit.spec.ts @@ -44,7 +44,15 @@ describe("central Replicator provider definitions", () => { it("keeps the retained remote-resource catalogue bounded", () => { expect .soft(Object.values(REMOTE_RESOURCE_KINDS).sort()) - .toEqual(["connection", "preferred-tweak", "security-seed", "synchronisation-information"].sort()); + .toEqual( + [ + "connection", + "joined-devices", + "preferred-tweak", + "security-seed", + "synchronisation-information", + ].sort() + ); }); it("lets Journal prepare its own fresh Security Seed while CouchDB uses central preparation", () => { diff --git a/src/common/replicatorResources/index.ts b/src/common/replicatorResources/index.ts index 9888e85c..f2f359f1 100644 --- a/src/common/replicatorResources/index.ts +++ b/src/common/replicatorResources/index.ts @@ -14,3 +14,8 @@ export { type SecuritySeedResourceHost, } from "./securitySeed"; export { createCouchDBSynchronisationInformationResourceFactory } from "./synchronisationInformation"; + +export { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, +} from "./joinedDevices"; diff --git a/src/common/replicatorResources/joinedDevices.ts b/src/common/replicatorResources/joinedDevices.ts new file mode 100644 index 00000000..a9acf1c1 --- /dev/null +++ b/src/common/replicatorResources/joinedDevices.ts @@ -0,0 +1,40 @@ +import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { + JoinedDevicesResource, + JoinedDevicesResourceFactory, + JoinedDevicesSnapshot, + ForgetJoinedDeviceResult, +} from "@vrtmrz/livesync-commonlib/replication"; +import { + LiveSyncCouchDBReplicator, + type LiveSyncCouchDBReplicatorEnv, +} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; +import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator"; +import { createReplicatorDisposer, snapshotRemoteSettings, type ResourceReplicator } from "./shared"; + +interface JoinedDevicesReplicator extends ResourceReplicator { + getJoinedDevices(setting: RemoteDBSettings): Promise; + forgetJoinedDevice(setting: RemoteDBSettings, deviceId: string): Promise; +} + +function createResource(replicator: JoinedDevicesReplicator, snapshot: RemoteDBSettings): JoinedDevicesResource { + return { + read: () => replicator.getJoinedDevices(snapshot), + forget: (deviceId) => replicator.forgetJoinedDevice(snapshot, deviceId), + dispose: createReplicatorDisposer(replicator), + }; +} + +export function createCouchDBJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncCouchDBReplicator(host), snapshotRemoteSettings(setting))); +} + +export function createObjectStorageJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncJournalReplicator(host), snapshotRemoteSettings(setting))); +} diff --git a/src/main.ts b/src/main.ts index 4dcec641..8aebf8ad 100644 --- a/src/main.ts +++ b/src/main.ts @@ -51,6 +51,7 @@ import { useInteractiveConflictResolutionFeature } from "./serviceFeatures/inter import { ConflictResolveModal } from "./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts"; import { useObsidianReplicationRibbonFeature } from "./serviceFeatures/obsidianReplicationRibbon.ts"; import { useStartupLifecycleFeature } from "./serviceFeatures/startupLifecycle"; +import { useJoinedDevicesFeature } from "./serviceFeatures/useJoinedDevices"; import { useDatabaseDiagnosticsFeature } from "./serviceFeatures/useDatabaseDiagnostics"; export type LiveSyncCore = LiveSyncBaseCore; export default class ObsidianLiveSyncPlugin extends Plugin { @@ -197,6 +198,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { useP2PReplicatorUI(core, core, replicator, createInteractiveP2PReplication(replicator)); useObsidianReplicationRibbonFeature(core); useDatabaseDiagnosticsFeature(core); + useJoinedDevicesFeature(core, serviceHub.context.noticeGroups); useRemoteConfiguration(core); useSetupProtocolFeature(core, setupManager); diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts index 8c608165..41fd1df8 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts @@ -7,6 +7,7 @@ import { type ObsidianLiveSyncSettings, LOG_LEVEL_VERBOSE, } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { OPEN_JOINED_DEVICES } from "@/serviceFeatures/joinedDevicesCommand"; import { Menu, type ButtonComponent } from "@/deps.ts"; import { $msg } from "@/common/translation"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; @@ -101,6 +102,22 @@ export function paneRemoteConfig( paneEl: HTMLElement, { addPanel }: PageFunctions ): void { + new Setting(paneEl) + .setName($msg("Joined devices")) + .setDesc($msg("Review device records for the active CouchDB or Object Storage connection.")) + .addButton((button) => + button.setButtonText($msg("Open")).onClick(async () => { + const result = await this.services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + if (result.status === "unavailable" || result.value === "unavailable") { + this.services.API.addLog( + $msg( + "Joined devices are available after database startup with an active CouchDB or Object Storage connection." + ), + LOG_LEVEL_NOTICE + ); + } + }) + ); { /* E2EE */ const E2EEInitialProps = { diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts index 4d83048f..8b2a0187 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.unit.spec.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const runtime = vi.hoisted(() => ({ buttonClasses: [] as string[], - clickHandlers: [] as Array<() => Promise | void>, + clickHandlers: [] as Array<{ setting: string; run: () => Promise | void }>, panels: [] as Array<{ destroy: ReturnType }>, settingClasses: [] as string[], })); @@ -32,7 +32,10 @@ vi.mock("./LiveSyncSetting.ts", () => ({ }, }; - setName() { + settingName = ""; + + setName(name: string) { + this.settingName = name; return this; } @@ -41,6 +44,7 @@ vi.mock("./LiveSyncSetting.ts", () => ({ } addButton(callback: (button: unknown) => void) { + const setting = this.settingName; const button = { buttonEl: { classList: { @@ -53,7 +57,7 @@ vi.mock("./LiveSyncSetting.ts", () => ({ return this; }, onClick(callback: () => Promise | void) { - runtime.clickHandlers.push(callback); + runtime.clickHandlers.push({ setting, run: callback }); return this; }, setButtonText() { @@ -107,6 +111,7 @@ vi.mock("./remoteConfigBuffer.ts", () => ({ syncActivatedRemoteSettings: vi.fn(), })); +import { OPEN_JOINED_DEVICES } from "@/serviceFeatures/joinedDevicesCommand"; import { paneRemoteConfig } from "./PaneRemoteConfig.ts"; function createPanelElement(): HTMLElement { @@ -124,6 +129,18 @@ afterEach(() => { }); describe("paneRemoteConfig", () => { + it("opens joined devices through the shared UI command", async () => { + const invoke = vi.fn(async () => ({ status: "handled", value: "closed" })); + const host = { + editingSettings: { remoteConfigurations: {} }, + services: { UI: { commands: { invoke } } }, + }; + const addPanel = vi.fn(() => ({ then: vi.fn() })); + paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never); + await runtime.clickHandlers.find((handler) => handler.setting === "Joined devices")!.run(); + expect(invoke).toHaveBeenCalledWith(OPEN_JOINED_DEVICES, undefined); + }); + it("destroys the E2EE info panel when the settings page lifetime unloads", async () => { const callbacks: Array<() => unknown> = []; const lifetimeComponent = { @@ -187,7 +204,7 @@ describe("paneRemoteConfig", () => { })); paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never); - await runtime.clickHandlers[0](); + await runtime.clickHandlers.find((handler) => handler.setting === "Configure E2EE")!.run(); expect(setupManager.onlyE2EEConfiguration).toHaveBeenCalledOnce(); expect(host.editingSettings.encryptInternalMetadata).toBe(true); @@ -233,7 +250,7 @@ describe("paneRemoteConfig", () => { })); paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never); - await runtime.clickHandlers[0](); + await runtime.clickHandlers.find((handler) => handler.setting === "Configure E2EE")!.run(); expect(host.editingSettings.idDerivationVersion).toBe(1); expect(host.editingSettings.idDerivationKey).toBe(nextIdKey); diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte new file mode 100644 index 00000000..31b30356 --- /dev/null +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -0,0 +1,391 @@ + + + +
+

+ {translate("This list shows device information recorded by the remote. It does not show whether a device is online now.")} +

+ + {#if view} + {#if !view.milestonePresent} +

{translate("This remote does not currently have a record of joined devices.")}

+ {/if} + {#if view.cleaned} +

+ {translate( + "This remote has been cleaned. Use the existing recovery process to join devices again as needed." + )} +

+ {/if} + {#if view.locked} +

+ {translate( + "This remote is locked. Use the existing recovery process for devices that need to join again." + )} +

+ {/if} + {/if} + + {#if errorMessage} + + {/if} + {#if statusMessage} +

{statusMessage}

+ {/if} + {#if isRefreshing && !view} +

{translate("Loading joined devicesâ€Ļ")}

+ {/if} + + {#if view} + {#if view.devices.length > 0} +
    + {#each view.devices as device (device.id)} +
  • +
    +
    + {#if device.self} + {translate("This device")} + {/if} + {device.id} +
    +
    + + {device.accepted ? translate("Accepted") : translate("Not accepted")} + + {#if device.lockedOut} + {translate("Locked out")} + {/if} +
    +
    + + {#if device.self && device.lockedOut} +

    + {translate( + "This device is locked out. Use the existing recovery process to join again." + )} +

    + {/if} + +
    +
    +
    {translate("Platform")}
    +
    {device.platform || translate("Unknown platform")}
    +
    +
    +
    {translate("Plug-in version")}
    +
    {device.pluginVersion || translate("Not recorded")}
    +
    +
    +
    {translate("App version")}
    +
    {device.appVersion || translate("Not recorded")}
    +
    +
    +
    {translate("Last recorded")}
    +
    {formatLastConnected(device.lastConnected)}
    +
    +
    + + {#if !device.self} + {#if confirmingId === device.id} +
    +

    {translate("Forget this device record?")}

    +

    + {translate("This removes only this device's record from the remote.")} +

    +

    + {translate( + "Forgetting the record does not delete Vault files or revoke credentials." + )} +

    +

    + {view.locked + ? translate( + "This remote is locked. If the device returns, use the existing recovery process to join it again." + ) + : translate("This remote is unlocked. A device that returns can register again.")} +

    +
    + + +
    +
    + {:else} +
    + +
    + {/if} + {/if} +
    • + {/each} +
+ {:else if !isRefreshing} +

{translate("No joined devices are recorded on this remote.")}

+ {/if} + {/if} + +
+ + +
+
+ + diff --git a/src/serviceFeatures/joinedDevices.ts b/src/serviceFeatures/joinedDevices.ts new file mode 100644 index 00000000..64bcc2cd --- /dev/null +++ b/src/serviceFeatures/joinedDevices.ts @@ -0,0 +1,64 @@ +import type { JoinedDevice, JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; +import type { JoinedDevicesView } from "./joinedDevicesDialogTypes"; + +/** Compare recognised SemVer values; unknown development labels have no ordering. */ +export function comparePluginVersions(left: string, right: string): number | undefined { + const parse = (value: string) => { + const match = + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/.exec( + value + ); + if (!match) return undefined; + const prerelease = match[4]?.split("."); + if (prerelease?.some((part) => /^0\d+$/.test(part))) return undefined; + return { core: match.slice(1, 4).map(BigInt), prerelease }; + }; + const a = parse(left); + const b = parse(right); + if (!a || !b) return undefined; + for (let i = 0; i < 3; i++) { + if (a.core[i] !== b.core[i]) return a.core[i] > b.core[i] ? 1 : -1; + } + if (!a.prerelease || !b.prerelease) return a.prerelease ? -1 : b.prerelease ? 1 : 0; + for (let i = 0; i < Math.max(a.prerelease.length, b.prerelease.length); i++) { + const x = a.prerelease[i]; + const y = b.prerelease[i]; + if (x === undefined) return -1; + if (y === undefined) return 1; + if (x === y) continue; + const xn = /^\d+$/.test(x); + const yn = /^\d+$/.test(y); + if (xn && yn) return BigInt(x) > BigInt(y) ? 1 : -1; + if (xn !== yn) return xn ? -1 : 1; + return x > y ? 1 : -1; + } + return 0; +} + +export function joinedDevicesView( + snapshot: JoinedDevicesSnapshot, + self: Omit +): JoinedDevicesView { + const ownRecord = snapshot.devices.find((device) => device.id === self.id); + const ownDevice: JoinedDevice = { ...ownRecord, ...self, accepted: ownRecord?.accepted ?? false }; + return { + ...snapshot, + devices: [ownDevice, ...snapshot.devices.filter((device) => device.id !== self.id)].map((device) => ({ + ...device, + self: device.id === self.id, + lockedOut: snapshot.locked && !device.accepted, + })), + }; +} + +export function differentDeviceVersions(snapshot: JoinedDevicesSnapshot, ownId: string, ownVersion: string) { + const newer = new Set(); + const older = new Set(); + for (const device of snapshot.devices) { + if (device.id === ownId || !device.pluginVersion) continue; + const comparison = comparePluginVersions(device.pluginVersion, ownVersion); + if (comparison !== undefined && comparison > 0) newer.add(device.pluginVersion); + if (comparison !== undefined && comparison < 0) older.add(device.pluginVersion); + } + return { newer: [...newer].sort(), older: [...older].sort() }; +} diff --git a/src/serviceFeatures/joinedDevices.unit.spec.ts b/src/serviceFeatures/joinedDevices.unit.spec.ts new file mode 100644 index 00000000..a5e4f8c8 --- /dev/null +++ b/src/serviceFeatures/joinedDevices.unit.spec.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { comparePluginVersions, differentDeviceVersions, joinedDevicesView } from "./joinedDevices"; +import type { JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; + +const empty: JoinedDevicesSnapshot = { milestonePresent: false, locked: false, cleaned: false, devices: [] }; + +describe("joined device presentation", () => { + it("always shows this device first, including before its remote record exists", () => { + const view = joinedDevicesView( + { ...empty, devices: [{ id: "another", accepted: true }] }, + { id: "self", platform: "ios", pluginVersion: "1.0.39" } + ); + expect(view.devices.map((device) => device.id)).toEqual(["self", "another"]); + expect(view.devices[0]).toMatchObject({ self: true, accepted: false, lockedOut: false, platform: "ios" }); + }); + it("keeps the remote last-recorded date and acceptance while showing current local versions", () => { + const view = joinedDevicesView( + { + ...empty, + milestonePresent: true, + locked: true, + devices: [ + { id: "other", accepted: true }, + { id: "self", accepted: false, lastConnected: 123, pluginVersion: "1.0.0" }, + ], + }, + { id: "self", pluginVersion: "1.1.0" } + ); + expect(view.devices[0]).toMatchObject({ + self: true, + lockedOut: true, + lastConnected: 123, + pluginVersion: "1.1.0", + }); + expect(view.devices[1].lockedOut).toBe(false); + }); +}); + +describe("version advisories", () => { + it.each([ + ["1.10.0", "1.9.9", 1], + ["1.0.0", "1.0.0-rc.1", 1], + ["1.0.0-beta.2", "1.0.0-beta.10", -1], + ["1.0.0-1", "1.0.0-alpha", -1], + ["1.0.0-alpha", "1.0.0-alpha.1", -1], + ["1.0.0+first", "1.0.0+second", 0], + ["99999999999999999.0.0", "99999999999999998.0.0", 1], + ])("compares %s with %s", (a, b, result) => expect(comparePluginVersions(a, b)).toBe(result)); + it.each(["", "development", "1.0", "01.0.0", "1.0.0-01", "v1.0.0"])( + "does not order unknown version %s", + (version) => { + expect(comparePluginVersions(version, "1.0.0")).toBeUndefined(); + expect(comparePluginVersions("1.0.0", version)).toBeUndefined(); + } + ); + it("excludes this device and unknown versions, and deduplicates versions", () => { + const snapshot = { + ...empty, + devices: [ + { id: "self", accepted: false, pluginVersion: "9.0.0" }, + { id: "a", accepted: true, pluginVersion: "2.0.0" }, + { id: "b", accepted: false, pluginVersion: "2.0.0" }, + { id: "c", accepted: false, pluginVersion: "0.9.0" }, + { id: "d", accepted: true, pluginVersion: "custom" }, + ], + }; + expect(differentDeviceVersions(snapshot, "self", "1.0.0")).toEqual({ newer: ["2.0.0"], older: ["0.9.0"] }); + }); +}); diff --git a/src/serviceFeatures/joinedDevicesCommand.ts b/src/serviceFeatures/joinedDevicesCommand.ts new file mode 100644 index 00000000..c661c560 --- /dev/null +++ b/src/serviceFeatures/joinedDevicesCommand.ts @@ -0,0 +1,5 @@ +import { declareUICommand } from "@vrtmrz/livesync-commonlib/ui-commands"; + +export const OPEN_JOINED_DEVICES = declareUICommand( + "livesync:joined-devices" +); diff --git a/src/serviceFeatures/joinedDevicesDialogTypes.ts b/src/serviceFeatures/joinedDevicesDialogTypes.ts new file mode 100644 index 00000000..59b28ba6 --- /dev/null +++ b/src/serviceFeatures/joinedDevicesDialogTypes.ts @@ -0,0 +1,19 @@ +import type { ForgetJoinedDeviceResult, JoinedDevice } from "@vrtmrz/livesync-commonlib/replication"; + +export interface JoinedDeviceRow extends JoinedDevice { + readonly self: boolean; + readonly lockedOut: boolean; +} + +export interface JoinedDevicesView { + readonly milestonePresent: boolean; + readonly locked: boolean; + readonly cleaned: boolean; + readonly devices: readonly JoinedDeviceRow[]; +} + +export interface JoinedDevicesDialogOptions { + refresh(): Promise; + forget(id: string): Promise; + cancel(): void; +} diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts new file mode 100644 index 00000000..37d50b0f --- /dev/null +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -0,0 +1,212 @@ +import type { NecessaryServices } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import { + NODEINFO_DOCID, + REMOTE_COUCHDB, + REMOTE_MINIO, + type EntryNodeInfo, + type RemoteDBSettings, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { REMOTE_RESOURCE_KINDS, type JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; +import type { ObsidianNoticeGroups } from "@/modules/services/ObsidianNoticeGroups"; +import JoinedDevicesDialog from "@/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte"; +import { + getCouchDBReplicatorConfigurationIdentity, + getObjectStorageReplicatorConfigurationIdentity, +} from "@/common/replicatorConfigurationIdentity"; +import { $msg } from "@/common/translation"; +import { OPEN_JOINED_DEVICES } from "./joinedDevicesCommand"; +import { differentDeviceVersions, joinedDevicesView } from "./joinedDevices"; +import type { JoinedDevicesDialogOptions } from "./joinedDevicesDialogTypes"; + +type JoinedDevicesHost = NecessaryServices< + "API" | "UI" | "setting" | "database" | "replicator" | "replication" | "appLifecycle", + never +>; +const noticeGroup = "joined-device-versions"; + +function remoteIdentity(settings: RemoteDBSettings): string | undefined { + if (settings.remoteType === REMOTE_COUCHDB && settings.couchDB_URI && settings.couchDB_DBNAME) { + return getCouchDBReplicatorConfigurationIdentity(settings); + } + if (settings.remoteType === REMOTE_MINIO && settings.endpoint && settings.bucket) { + return getObjectStorageReplicatorConfigurationIdentity(settings); + } + return undefined; +} + +/** Own the dialogue, advisory Notices, and their per-session state for one host. */ +export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: ObsidianNoticeGroups): void { + const { services } = host; + let disposed = false; + let active: AbortController | undefined; + let checking = false; + let lastCheck: { identity: string; at: number } | undefined; + const notified = new Map>(); + const available = () => + !disposed && + services.database.isDatabaseReady() && + remoteIdentity(services.setting.currentSettings()) !== undefined; + + const notify = (snapshot: JoinedDevicesSnapshot, ownId: string, identity: string) => { + if (disposed || identity !== remoteIdentity(services.setting.currentSettings())) return; + const version = services.API.getPluginVersion(); + const differences = differentDeviceVersions(snapshot, ownId, version); + const seen = notified.get(identity) ?? new Set(); + notified.set(identity, seen); + for (const direction of ["newer", "older"] as const) { + const versions = differences[direction]; + if (versions.length === 0) continue; + const key = JSON.stringify([version, direction, versions]); + if (seen.has(key)) continue; + seen.add(key); + notices.setItem(noticeGroup, direction, { + message: + direction === "newer" + ? $msg("Other joined devices last reported a newer plug-in version: ${versions}.", { + versions: versions.join(", "), + }) + : $msg("Other joined devices last reported an older plug-in version: ${versions}.", { + versions: versions.join(", "), + }), + action: { + label: $msg("Joined devices"), + onSelect: () => { + if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { + void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + } + }, + }, + }); + notices.finish(noticeGroup, { durationMs: 15000 }); + } + }; + + const createOperations = (controller: AbortController): JoinedDevicesDialogOptions => { + const settings = { ...services.setting.currentSettings() }; + const identity = remoteIdentity(settings); + const database = services.database.localDatabase; + const assertCurrent = () => { + if ( + disposed || + controller.signal.aborted || + !identity || + identity !== remoteIdentity(services.setting.currentSettings()) || + !services.database.isDatabaseReady() || + services.database.localDatabase !== database + ) { + throw new Error("Joined devices are no longer available for this dialogue"); + } + }; + const ownId = async () => { + assertCurrent(); + const info = await database.localDatabase.get(NODEINFO_DOCID); + assertCurrent(); + if (typeof info.nodeid !== "string" || !info.nodeid) throw new Error("Device identity is unavailable"); + return info.nodeid; + }; + return { + refresh: async () => { + const id = await ownId(); + const resource = await services.replicator.createRemoteResource( + REMOTE_RESOURCE_KINDS.JOINED_DEVICES, + settings + ); + if (!resource) throw new Error("Joined devices are unavailable"); + try { + assertCurrent(); + const snapshot = await resource.read(); + assertCurrent(); + notify(snapshot, id, identity!); + return joinedDevicesView(snapshot, { + id, + platform: services.API.getPlatform(), + pluginVersion: services.API.getPluginVersion(), + appVersion: services.API.getAppVersion(), + }); + } finally { + await resource.dispose(); + } + }, + forget: async (id) => { + if (id === (await ownId())) throw new Error("This device cannot be forgotten"); + const resource = await services.replicator.createRemoteResource( + REMOTE_RESOURCE_KINDS.JOINED_DEVICES, + settings + ); + if (!resource) throw new Error("Joined devices are unavailable"); + try { + assertCurrent(); + const result = await resource.forget(id); + assertCurrent(); + return result; + } finally { + await resource.dispose(); + } + }, + cancel: () => controller.abort(), + }; + }; + + const unregister = services.UI.commands.register(OPEN_JOINED_DEVICES, async () => { + if (!available()) return "unavailable"; + if (active) return "already-open"; + const controller = new AbortController(); + active = controller; + try { + await services.UI.dialogManager.open<"close", JoinedDevicesDialogOptions>( + JoinedDevicesDialog, + createOperations(controller) + ); + } catch { + // The host may close its dialogues during unload. + } finally { + controller.abort(); + if (active === controller) active = undefined; + } + return "closed"; + }); + + const checkVersions = () => { + if (!available() || checking) return; + const identity = remoteIdentity(services.setting.currentSettings())!; + if (lastCheck?.identity === identity && Date.now() - lastCheck.at < 60000) return; + lastCheck = { identity, at: Date.now() }; + checking = true; + const controller = new AbortController(); + void Promise.resolve() + .then(() => createOperations(controller).refresh()) + .catch(() => { + // Advisory inspection never changes the outcome of synchronisation. + }) + .finally(() => { + controller.abort(); + checking = false; + }); + }; + const stopBeforeReplicate = services.replication.onBeforeReplicate.addHandler(() => { + checkVersions(); + return Promise.resolve(true); + }); + services.appLifecycle.onInitialise.addHandler(() => { + services.API.addCommand({ + id: "joined-devices", + name: $msg("Joined devices"), + icon: "monitor-smartphone", + checkCallback: (checking) => { + if (!available()) return false; + if (!checking) void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + return true; + }, + }); + return Promise.resolve(true); + }); + services.appLifecycle.onUnload.addHandler(() => { + disposed = true; + active?.abort(); + unregister(); + stopBeforeReplicate(); + notices.hide(noticeGroup); + notified.clear(); + return Promise.resolve(true); + }); +} diff --git a/src/serviceFeatures/useJoinedDevices.unit.spec.ts b/src/serviceFeatures/useJoinedDevices.unit.spec.ts new file mode 100644 index 00000000..7806508a --- /dev/null +++ b/src/serviceFeatures/useJoinedDevices.unit.spec.ts @@ -0,0 +1,174 @@ +import { describe, expect, it, vi } from "vitest"; +import { UICommandRegistry } from "@vrtmrz/livesync-commonlib/ui-commands"; +import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings"; +import { REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; +import type { JoinedDevicesDialogOptions } from "./joinedDevicesDialogTypes"; +vi.mock("@/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte", () => ({ default: {} })); +vi.mock("@/common/translation", () => ({ $msg: (message: string) => message })); +import { useJoinedDevicesFeature } from "./useJoinedDevices"; +import { OPEN_JOINED_DEVICES } from "./joinedDevicesCommand"; + +function fixture() { + const settings = { + ...createNewVaultSettings(), + remoteType: REMOTE_COUCHDB, + couchDB_URI: "https://example.test", + couchDB_DBNAME: "vault", + }; + const initialise: Array<() => Promise> = []; + const unload: Array<() => Promise> = []; + const before: Array<() => Promise> = []; + const stopBefore = vi.fn(); + const commands = new UICommandRegistry(); + const read = vi.fn<() => Promise>(async () => ({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [{ id: "other", accepted: true, pluginVersion: "2.0.0" }], + })); + const forget = vi.fn(async (_id: string) => "forgotten" as const); + const dispose = vi.fn(async () => {}); + const createResource = vi.fn(async () => ({ read, forget, dispose })); + let close!: () => void; + const dialog = vi.fn( + (_component: unknown, _options: JoinedDevicesDialogOptions) => + new Promise((resolve) => { + close = resolve; + }) + ); + const notices = { setItem: vi.fn(), finish: vi.fn(), hide: vi.fn() }; + const palette: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = []; + const database = { localDatabase: { get: vi.fn(async () => ({ nodeid: "self" })) } }; + const host = { + services: { + API: { + getPluginVersion: () => "1.0.0", + getAppVersion: () => "1.9.0", + getPlatform: () => "desktop", + addCommand: (command: (typeof palette)[number]) => palette.push(command), + }, + UI: { commands, dialogManager: { open: dialog } }, + setting: { currentSettings: () => settings }, + database: { isDatabaseReady: () => true, localDatabase: database }, + replicator: { createRemoteResource: createResource }, + replication: { + onBeforeReplicate: { + addHandler: (handler: (typeof before)[number]) => { + before.push(handler); + return stopBefore; + }, + }, + }, + appLifecycle: { + onInitialise: { addHandler: (handler: (typeof initialise)[number]) => initialise.push(handler) }, + onUnload: { addHandler: (handler: (typeof unload)[number]) => unload.push(handler) }, + }, + }, + serviceModules: {}, + } as unknown as Parameters[0]; + useJoinedDevicesFeature(host, notices as unknown as Parameters[1]); + return { + settings, + initialise, + unload, + before, + stopBefore, + commands, + read, + forget, + dispose, + createResource, + dialog, + notices, + palette, + database, + close: () => close(), + }; +} + +async function open(data: ReturnType) { + const pending = data.commands.invoke(OPEN_JOINED_DEVICES, undefined); + const options = data.dialog.mock.calls[0][1]; + return { options, pending }; +} + +describe("joined devices service feature", () => { + it("opens one dialogue through the command and disposes each completed read", async () => { + const data = fixture(); + const { options, pending } = await open(data); + expect(await data.commands.invoke(OPEN_JOINED_DEVICES, undefined)).toEqual({ + status: "handled", + value: "already-open", + }); + expect((await options.refresh()).devices[0]).toMatchObject({ id: "self", self: true }); + expect(data.dispose).toHaveBeenCalledTimes(1); + data.close(); + expect(await pending).toEqual({ status: "handled", value: "closed" }); + await expect(options.refresh()).rejects.toThrow(); + }); + it("prevents self removal and fences an old dialogue after a remote switch", async () => { + const data = fixture(); + const { options, pending } = await open(data); + await expect(options.forget("self")).rejects.toThrow(); + data.settings.couchDB_DBNAME = "new-vault"; + await expect(options.forget("other")).rejects.toThrow(); + expect(data.forget).not.toHaveBeenCalled(); + expect(data.createResource).not.toHaveBeenCalled(); + data.close(); + await pending; + }); + it("discards a stale read, releases the resource, and emits no stale Notice", async () => { + const data = fixture(); + let resolve!: (value: JoinedDevicesSnapshot) => void; + data.read.mockReturnValueOnce( + new Promise((done) => { + resolve = done; + }) + ); + const { options, pending } = await open(data); + const reading = options.refresh(); + await vi.waitFor(() => expect(data.read).toHaveBeenCalled()); + data.settings.couchDB_DBNAME = "new-vault"; + resolve({ milestonePresent: true, locked: false, cleaned: false, devices: [] }); + await expect(reading).rejects.toThrow(); + expect(data.dispose).toHaveBeenCalledOnce(); + expect(data.notices.setItem).not.toHaveBeenCalled(); + data.close(); + await pending; + }); + it("deduplicates advisory Notices and fences their actions after configuration changes", async () => { + const data = fixture(); + const { options, pending } = await open(data); + await options.refresh(); + await options.refresh(); + expect(data.notices.setItem).toHaveBeenCalledTimes(1); + const action = data.notices.setItem.mock.calls[0][2].action; + data.close(); + await pending; + data.settings.couchDB_DBNAME = "new-vault"; + action.onSelect(); + expect(data.dialog).toHaveBeenCalledTimes(1); + }); + it("keeps inspection failures out of the replication result and unregisters on unload", async () => { + const data = fixture(); + data.read.mockRejectedValue(new Error("unavailable")); + expect(await data.before[0]()).toBe(true); + await vi.waitFor(() => expect(data.dispose).toHaveBeenCalledOnce()); + expect(await data.before[0]()).toBe(true); + expect(data.read).toHaveBeenCalledTimes(1); + for (const unload of data.unload) await unload(); + expect(await data.commands.invoke(OPEN_JOINED_DEVICES, undefined)).toEqual({ status: "unavailable" }); + expect(data.stopBefore).toHaveBeenCalledOnce(); + expect(data.notices.hide).toHaveBeenCalled(); + }); + it("uses the same UI command from the command palette", async () => { + const data = fixture(); + for (const initialise of data.initialise) await initialise(); + expect(data.palette[0].checkCallback(true)).toBe(true); + expect(data.dialog).not.toHaveBeenCalled(); + data.palette[0].checkCallback(false); + expect(data.dialog).toHaveBeenCalledOnce(); + data.close(); + }); +}); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index b4e854a1..d5522234 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -125,6 +125,8 @@ The underlying `test:e2e:obsidian:` scripts remain available for an im `test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. +`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and forgets one device through the inline confirmation. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. + `test:e2e:obsidian:settings-ui` starts with a pending compatibility review and verifies the dedicated pause summary, its detailed explanation, and the explicit resume action in a temporary real Obsidian session. It captures the desktop summary and the iPhone-sized summary and detail dialogues; the mobile checks cover viewport containment, horizontal overflow, safe-area containment, and the close control's touch target. It confirms that the acknowledged internal version advances only after the review is accepted, and checks that the Change Log contains no acknowledgement control. It then enables Advanced mode and persists one numeric Advanced setting. On Obsidian before 1.13, this covers the imperative settings fallback used by the `SettingSpec` proof. On Obsidian 1.13 or later, it verifies the task-oriented landing order, the separate Synchronisation group containing Remote Configuration and Sync Settings, and the General Settings group containing Appearance, Logging, and Extra menus. It reopens and cancels onboarding from Quick Setup, opens all 14 nested settings pages, searches globally for the Advanced control, captures the landing page, search result, Advanced page, and mobile landing page, and verifies that the value remains after the settings dialogue is closed and reopened. The mobile landing check confirms that Remote Configuration is reachable without initial scrolling. Finally, it selects the Synchronisation Settings page and verifies that the deletion panel still exposes the effective 'Keep empty folder' setting without presenting the legacy `trashInsteadDelete` control, whose value no longer changes Obsidian deletion behaviour. The default runner uses the maintained pre-1.13 Obsidian fixture and therefore exercises the complete compatibility-review, mobile-layout, and imperative-settings path. To exercise only the native settings contract against an additional Obsidian 1.13-or-later installation, supply its executable and companion CLI explicitly: diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts new file mode 100644 index 00000000..d5736139 --- /dev/null +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -0,0 +1,462 @@ +import { DEVICE_ID_PREFERRED, MILESTONE_DOCID, NODEINFO_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + fetchCouchDbDocument, + loadCouchDbConfig, + makeUniqueDatabaseName, + putCouchDbDocument, + type CouchDbDocument, +} from "../runner/couchdb.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + assertEqual, + assertE2eCompatibilityUnpaused, + configureCouchDb, + createE2eCouchDbPluginData, + prepareRemote, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { + assertLocatorHasMinimumTouchTarget, + assertLocatorWithinSafeArea, + assertLocatorWithinViewport, + assertNoHorizontalOverflow, +} from "@vrtmrz/obsidian-test-session"; +import { iPhoneSafeArea, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { + captureObsidianDialogue, + openLiveSyncSettings, + waitForVisibleObsidianDialogue, + withObsidianPage, +} from "../runner/ui.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; +import type { Locator, Page } from "playwright"; + +const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_JOINED_DEVICES_TIMEOUT_MS ?? 10000); +const commandId = "obsidian-livesync:joined-devices"; +const targetId = "joined-devices-target"; +const retainedId = "joined-devices-retained"; +const initialPlatform = "E2E Android tablet"; +const refreshedPlatform = "E2E refreshed platform"; +const connectedAt = Date.UTC(2026, 0, 12, 9, 30, 0); + +type SelfDeviceInfo = { + id: string; + platform: string; + pluginVersion: string; + appVersion: string; +}; + +type DeviceInfo = { + device_name: string; + vault_name: string; + app_version: string; + plugin_version: string; + platform: string; + progress: string; + last_connected: number; +}; + +type JoinedDevicesMilestone = CouchDbDocument & { + accepted_nodes: string[]; + node_info: Record; + locked: boolean; + cleaned: boolean; + node_chunk_info: Record; + tweak_values: Record>; +}; + +type DeviceFixture = { + milestone: JoinedDevicesMilestone; + preferredTweaks: Record; +}; + +function assert(condition: boolean, message: string): void { + if (!condition) throw new Error(message); +} + +function deviceCard(dialogue: Locator, id: string): Locator { + return dialogue.locator(".device-card").filter({ hasText: id }); +} + +async function assertDetail(card: Locator, label: string, expected: string): Promise { + const rows = card.locator(".device-details > div"); + for (let index = 0; index < (await rows.count()); index++) { + const row = rows.nth(index); + if ((await row.locator("dt").innerText()).trim() === label) { + assertEqual((await row.locator("dd").innerText()).trim(), expected, `${label} did not match.`); + return; + } + } + throw new Error(`The device details did not contain '${label}'.`); +} + +async function readSelfDeviceInfo(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + `const nodeInfo=await core.services.database.localDatabase.localDatabase.get(${JSON.stringify(NODEINFO_DOCID)});`, + "return JSON.stringify({", + "id:nodeInfo.nodeid,", + "platform:core.services.API.getPlatform(),", + "pluginVersion:core.services.API.getPluginVersion(),", + "appVersion:core.services.API.getAppVersion(),", + "});", + "})()", + ].join(""), + env + ); +} + +function createFixture(self: SelfDeviceInfo): DeviceFixture { + const preferredTweaks = { customChunkSize: 60, chunkSplitterVersion: "v3-rabin-karp" }; + const milestone: JoinedDevicesMilestone = { + _id: MILESTONE_DOCID, + type: "milestoneinfo", + created: Date.now(), + // The current device is deliberately absent: it must still be listed first and identified as locked out. + accepted_nodes: [targetId, retainedId], + node_info: { + [targetId]: { + device_name: "E2E target device", + vault_name: "Joined devices target fixture", + app_version: "1.99.0-test", + plugin_version: "999.0.0", + platform: initialPlatform, + progress: "ready", + last_connected: connectedAt, + }, + [retainedId]: { + device_name: "E2E retained device", + vault_name: "Joined devices retained fixture", + app_version: "1.88.0-test", + plugin_version: "0.1.0", + platform: "E2E Linux desktop", + progress: "ready", + last_connected: connectedAt - 86400000, + }, + }, + locked: true, + cleaned: true, + node_chunk_info: { + [targetId]: { min: 1, max: 3, current: 3 }, + [retainedId]: { min: 1, max: 2, current: 2 }, + }, + tweak_values: { + [targetId]: { customChunkSize: 48 }, + [retainedId]: { customChunkSize: 60 }, + [DEVICE_ID_PREFERRED]: preferredTweaks, + }, + }; + assert(self.id !== targetId && self.id !== retainedId, "The local device identity collided with a fixture ID."); + return { milestone, preferredTweaks }; +} + +async function openFromRemoteSettings(page: Page): Promise { + const settings = await openLiveSyncSettings(page, uiTimeoutMs); + const remoteConfiguration = await settings.openPage("Remote Configuration"); + await remoteConfiguration.getByRole("button", { name: "Open", exact: true }).click({ timeout: uiTimeoutMs }); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + return dialogue; +} + +async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, page: Page): Promise { + const cards = dialogue.locator(".device-card"); + assertEqual( + await cards.count(), + 3, + "The joined devices dialogue did not show the self, target, and retained records." + ); + const selfCard = cards.nth(0); + assert(Boolean(self.id), "The current device identity was not read from the local NODEINFO_DOCID record."); + assertEqual( + (await selfCard.locator("code.device-id").innerText()).trim(), + self.id, + "The current device was not listed first." + ); + await selfCard.getByText("This device", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await selfCard.getByText("Locked out", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await selfCard.getByText("Not accepted", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await selfCard + .getByText("This device is locked out.", { exact: false }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await assertDetail(selfCard, "Platform", self.platform); + await assertDetail(selfCard, "Plug-in version", self.pluginVersion); + await assertDetail(selfCard, "App version", self.appVersion); + assertEqual( + await selfCard.getByRole("button", { name: "Forget device", exact: true }).count(), + 0, + "The current device exposed a forget control." + ); + + const targetCard = deviceCard(dialogue, targetId); + await assertDetail(targetCard, "Platform", initialPlatform); + await assertDetail(targetCard, "Plug-in version", "999.0.0"); + await assertDetail(targetCard, "App version", "1.99.0-test"); + const targetDate = await page.evaluate((timestamp) => new Date(timestamp).toLocaleString(), connectedAt); + await assertDetail(targetCard, "Last recorded", targetDate); + await targetCard.getByText("Accepted", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + + const retainedCard = deviceCard(dialogue, retainedId); + await assertDetail(retainedCard, "Platform", "E2E Linux desktop"); + await assertDetail(retainedCard, "Plug-in version", "0.1.0"); + await assertDetail(retainedCard, "App version", "1.88.0-test"); + await dialogue + .getByText( + "This list shows device information recorded by the remote. It does not show whether a device is online now.", + { exact: true } + ) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dialogue + .getByText("This remote is locked.", { exact: false }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dialogue + .getByText("This remote has been cleaned.", { exact: false }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); +} + +async function refreshRemotePlatform( + page: Page, + config: Awaited>, + dbName: string +): Promise { + const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; + await putCouchDbDocument(config, dbName, { + ...milestone, + node_info: { + ...milestone.node_info, + [targetId]: { ...milestone.node_info[targetId], platform: refreshedPlatform }, + }, + }); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByRole("button", { name: "Refresh", exact: true }).click({ timeout: uiTimeoutMs }); + await deviceCard(dialogue, targetId).getByText(refreshedPlatform, { exact: true }).waitFor({ + state: "visible", + timeout: uiTimeoutMs, + }); +} + +async function showForgetConfirmation(dialogue: Locator): Promise { + const targetCard = deviceCard(dialogue, targetId); + await targetCard.getByRole("button", { name: "Forget device", exact: true }).click({ timeout: uiTimeoutMs }); + await targetCard.getByText("Forget this device record?", { exact: true }).waitFor({ + state: "visible", + timeout: uiTimeoutMs, + }); +} + +async function confirmForgetTarget(dialogue: Locator): Promise { + const targetCard = deviceCard(dialogue, targetId); + await targetCard.getByRole("button", { name: "Forget device", exact: true }).click({ timeout: uiTimeoutMs }); + await dialogue.getByText("The device record was forgotten.", { exact: true }).waitFor({ + state: "visible", + timeout: uiTimeoutMs, + }); + await targetCard.waitFor({ state: "detached", timeout: uiTimeoutMs }); +} + +async function assertTargetForgotten( + config: Awaited>, + dbName: string, + fixture: DeviceFixture +): Promise { + const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; + assertEqual( + JSON.stringify(milestone.accepted_nodes), + JSON.stringify([retainedId]), + "Forgetting changed the wrong accepted device records." + ); + assertEqual( + JSON.stringify(Object.keys(milestone.node_info)), + JSON.stringify([retainedId]), + "Forgetting changed the wrong device metadata records." + ); + assertEqual( + JSON.stringify(Object.keys(milestone.node_chunk_info)), + JSON.stringify([retainedId]), + "Forgetting changed the wrong chunk metadata records." + ); + assertEqual( + JSON.stringify(Object.keys(milestone.tweak_values).sort()), + JSON.stringify([DEVICE_ID_PREFERRED, retainedId].sort()), + "Forgetting changed the wrong tweak metadata records." + ); + assertEqual(milestone.locked, true, "Forgetting changed the remote lock state."); + assertEqual(milestone.cleaned, true, "Forgetting changed the remote cleaned state."); + assertEqual( + JSON.stringify(milestone.tweak_values[DEVICE_ID_PREFERRED]), + JSON.stringify(fixture.preferredTweaks), + "Forgetting changed the preferred-ID tweak record." + ); + assertEqual( + JSON.stringify(milestone.node_info[retainedId]), + JSON.stringify(fixture.milestone.node_info[retainedId]), + "Forgetting changed the retained device metadata." + ); +} + +async function assertScrollableMobileDialogue(page: Page, container: Locator): Promise { + const dialogue = container.locator(".modal").last(); + const closeButton = container.locator(".modal-close-button, .modal-header-button").last(); + await assertLocatorWithinViewport(page, dialogue, { label: "Joined devices dialogue" }); + await assertNoHorizontalOverflow(page, dialogue, { label: "Joined devices dialogue" }); + await assertLocatorWithinSafeArea(page, dialogue, { + label: "Joined devices dialogue", + safeAreaInsets: iPhoneSafeArea, + }); + const buttons = [closeButton, ...(await dialogue.locator("button:visible").all())]; + for (const button of buttons) { + // Device records form a scrollable list; every control must be reachable after scrolling. + await button.scrollIntoViewIfNeeded(); + const label = (await button.innerText()).trim() || "Close dialogue"; + await assertLocatorWithinViewport(page, button, { label }); + await assertLocatorWithinSafeArea(page, button, { label, safeAreaInsets: iPhoneSafeArea }); + await assertNoHorizontalOverflow(page, button, { label }); + await assertLocatorHasMinimumTouchTarget(page, button, { label }); + } + await container.locator(".modal-content").evaluate((element) => element.scrollTo(0, 0)); +} + +async function dismissNotices(page: Page): Promise { + const notices = page.locator(".notice:visible"); + for (let remaining = 20; remaining > 0 && (await notices.count()) > 0; remaining--) { + const notice = await notices.first().elementHandle(); + if (!notice) continue; + try { + await notice.click({ position: { x: 20, y: 20 }, timeout: uiTimeoutMs }); + await notice.waitForElementState("hidden", { timeout: uiTimeoutMs }); + } finally { + await notice.dispose(); + } + } + assertEqual(await notices.count(), 0, "Notices still obscured the dialogue screenshot."); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + + const couchDb = await loadCouchDbConfig(); + const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "joined-devices"); + const vault = await createTemporaryVault(); + let session: ObsidianLiveSyncSession | undefined; + + try { + await assertCouchDbReachable(couchDb); + await createCouchDbDatabase(couchDb, dbName); + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli.binary, + vault, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + pluginData: createE2eCouchDbPluginData({ + uri: couchDb.uri, + username: couchDb.username, + password: couchDb.password, + dbName, + }), + }); + await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); + await assertE2eCompatibilityUnpaused(cli.binary, session.cliEnv, session.remoteDebuggingPort); + await configureCouchDb(cli.binary, session.cliEnv, { + uri: couchDb.uri, + username: couchDb.username, + password: couchDb.password, + dbName, + }); + await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); + await prepareRemote(cli.binary, session.cliEnv); + + const self = await readSelfDeviceInfo(cli.binary, session.cliEnv); + const fixture = createFixture(self); + const preparedMilestone = await fetchCouchDbDocument(couchDb, dbName, MILESTONE_DOCID); + assert(Boolean(preparedMilestone._rev), "Remote preparation did not create a revisioned milestone document."); + await putCouchDbDocument(couchDb, dbName, { ...fixture.milestone, _rev: preparedMilestone._rev }); + + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { + await page.setViewportSize({ width: 1280, height: 1100 }); + const dialogue = await openFromRemoteSettings(page); + await assertInitialDialogue(dialogue, self, page); + await page + .locator(".notice") + .getByText("Other joined devices last reported a newer plug-in version: 999.0.0.", { exact: true }) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); + }); + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByRole("button", { name: "Close", exact: true }).click({ timeout: uiTimeoutMs }); + await page.locator(".notice").getByRole("button", { name: "Joined devices", exact: true }).first().click(); + const reopened = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await reopened.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dismissNotices(page); + }); + await captureObsidianDialogue( + session.remoteDebuggingPort, + "joined-devices-forget-confirmation.png", + async (page) => { + await refreshRemotePlatform(page, couchDb, dbName); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await showForgetConfirmation(dialogue); + await deviceCard(dialogue, targetId).locator(".forget-confirmation").scrollIntoViewIfNeeded(); + } + ); + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-after-forget.png", async (page) => { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await confirmForgetTarget(dialogue); + await dialogue.locator(".modal-content").evaluate((element) => element.scrollTo(0, 0)); + }); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByRole("button", { name: "Close", exact: true }).click({ timeout: uiTimeoutMs }); + const settings = await openLiveSyncSettings(page, uiTimeoutMs); + await settings.close(); + }); + await assertTargetForgotten(couchDb, dbName, fixture); + + await setObsidianMobileTestMode(session.remoteDebuggingPort, true, uiTimeoutMs); + await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); + await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-mobile.png", async (page) => { + const opened = await page.evaluate((id) => { + const app = ( + window as typeof window & { + app?: { commands?: { executeCommandById?(commandId: string): boolean } }; + } + ).app; + return app?.commands?.executeCommandById?.(id) === true; + }, commandId); + assert(opened, "The Joined devices command did not open the dialogue."); + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await dialogue.getByText(retainedId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); + await assertScrollableMobileDialogue(page, dialogue); + assertEqual( + await dialogue.locator(".device-card").count(), + 2, + "The refreshed mobile list did not retain both remaining devices." + ); + await dismissNotices(page); + }); + + console.log(`Joined devices scenario passed against CouchDB database ${dbName}.`); + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { + await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => { + console.warn(error instanceof Error ? error.message : error); + }); + } + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 7f766c67..9143b830 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -8,6 +8,7 @@ const focusedScenarios = new Set([ "smoke", "onboarding-invitation", "dialog-mounts", + "joined-devices", "database-diagnostics", "database-diagnostics-couchdb", "conflict-dialog-policy", diff --git a/updates.md b/updates.md index ca2e3497..d99fa163 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,14 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +- We can review participating devices through **Joined devices** in Remote Configuration, the command palette, or a version Notice. + - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. + - Advisory Notices help us find devices which last reported newer or older plug-in versions. +- We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. + - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. + ## 1.0.39 20th October, 2026 From e56098eb5728a25303945fd8c42ba15ff3dabae9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 13:53:56 +0000 Subject: [PATCH 35/68] Fix diagnostics checks across Obsidian renderers --- .../scripts/database-diagnostics.ts | 119 +++++++++++------- 1 file changed, 74 insertions(+), 45 deletions(-) diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts index 773c60c2..df54a7ba 100644 --- a/test/e2e-obsidian/scripts/database-diagnostics.ts +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -31,6 +31,18 @@ type DiagnosticPageGlobal = typeof globalThis & { diagnosticDownload?: { name: string; content: string }; }; +async function findVisibleElement(page: Page, selector: string): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + for (const candidate of page.context().pages()) { + const element = candidate.locator(selector); + if (await element.isVisible()) return element; + } + await page.waitForTimeout(50); + } + throw new Error(`No Obsidian renderer contains the visible element: ${selector}`); +} + async function openCommand(page: Page, kind: "usage" | "integrity") { const id = kind === "usage" ? "analyse-database" : "inspect-database-integrity"; const opened = await page.evaluate( @@ -243,11 +255,14 @@ async function assertDiagnosticFileLookup( session: ObsidianLiveSyncSession, mobile = false ): Promise { + const dialoguePage = hatch.page(); + const viewport = page.viewportSize(); + if (viewport) await dialoguePage.setViewportSize(viewport); const setting = hatch.locator(".setting-item").filter({ - has: page.getByText("Find a file from a report pseudonym", { exact: true }), + has: hatch.page().getByText("Find a file from a report pseudonym", { exact: true }), }); await setting.getByRole("button", { name: "Find file", exact: true }).click(); - const dialogue = page.locator(".diagnostic-file-lookup"); + const dialogue = dialoguePage.locator(".diagnostic-file-lookup"); await dialogue.waitFor({ state: "visible", timeout }); const input = dialogue.getByLabel("Report path pseudonym", { exact: true }); await input.fill("file-001"); @@ -259,8 +274,8 @@ async function assertDiagnosticFileLookup( await candidates.first().waitFor({ timeout }); assert.equal(await candidates.count(), 2, "Distinct Metadata IDs claiming the same path must remain selectable"); if (mobile) { - await assertNoHorizontalOverflow(page, dialogue, { label: "Pseudonym file lookup matches" }); - await assertNoHorizontalOverflow(page, dialogue.locator(".lookup-candidates"), { + await assertNoHorizontalOverflow(dialoguePage, dialogue, { label: "Pseudonym file lookup matches" }); + await assertNoHorizontalOverflow(dialoguePage, dialogue.locator(".lookup-candidates"), { label: "Pseudonym file lookup candidate list", }); assert.ok( @@ -270,7 +285,7 @@ async function assertDiagnosticFileLookup( await captureObsidianElement( session.remoteDebuggingPort, "database-diagnostics-file-matches-mobile.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".diagnostic-file-lookup") }) + (page) => findVisibleElement(page, ".modal-container:has(.diagnostic-file-lookup)") ); } const shadow = candidates.filter({ hasText: shadowId }); @@ -294,28 +309,30 @@ async function assertDiagnosticFileLookup( for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) { assert.ok(!json.includes(privateValue), "A follow-up report must preserve the original report's pseudonyms"); } - await installDiagnosticExportInterception(page); + await installDiagnosticExportInterception(dialoguePage); await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); - assert.equal(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), json); + assert.equal(await dialoguePage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), json); await dialogue.getByRole("button", { name: "Save report", exact: true }).click(); - await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { timeout }); - assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { + await dialoguePage.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { + timeout, + }); + assert.deepEqual(await dialoguePage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { name: "diagnostic-file-report.json", content: json, }); - await assertDesktopDiagnosticResultLayout(page, dialogue, "saving a follow-up report"); + await assertDesktopDiagnosticResultLayout(dialoguePage, dialogue, "saving a follow-up report"); if (mobile) { - const modal = page.locator(".modal-container").filter({ has: dialogue }); - await assertMobileDialogueLayout(page, modal, "Pseudonym file follow-up report"); - await assertNoHorizontalOverflow(page, dialogue, { label: "Pseudonym file follow-up content" }); - await assertNoHorizontalOverflow(page, dialogue.locator(".report-token"), { + const modal = dialoguePage.locator(".modal-container").filter({ has: dialogue }); + await assertMobileDialogueLayout(dialoguePage, modal, "Pseudonym file follow-up report"); + await assertNoHorizontalOverflow(dialoguePage, dialogue, { label: "Pseudonym file follow-up content" }); + await assertNoHorizontalOverflow(dialoguePage, dialogue.locator(".report-token"), { label: "Pseudonym file follow-up token", }); } await captureObsidianElement( session.remoteDebuggingPort, mobile ? "database-diagnostics-file-follow-up-mobile.png" : "database-diagnostics-file-follow-up.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".diagnostic-file-lookup") }) + (page) => findVisibleElement(page, ".modal-container:has(.diagnostic-file-lookup)") ); await dialogue.getByRole("radio", { name: "MARKDOWN", exact: true }).check(); const markdown = await dialogue.locator("textarea").inputValue(); @@ -379,10 +396,10 @@ async function assertFollowUpStopsAfterClose(page: Page, hatch: Locator, token: })()`); try { const setting = hatch.locator(".setting-item").filter({ - has: page.getByText("Find a file from a report pseudonym", { exact: true }), + has: hatch.page().getByText("Find a file from a report pseudonym", { exact: true }), }); await setting.getByRole("button", { name: "Find file", exact: true }).click(); - const dialogue = page.locator(".diagnostic-file-lookup"); + const dialogue = hatch.page().locator(".diagnostic-file-lookup"); await dialogue.getByLabel("Report path pseudonym", { exact: true }).fill(token); await dialogue.getByRole("button", { name: "Find file", exact: true }).click(); const candidate = dialogue.locator("[data-diagnostic-candidate]").filter({ hasText: shadowId }); @@ -466,7 +483,7 @@ async function main(): Promise { const activeSession = session; let usageJSON = ""; let usageFixtureRow: { path: string; id: string; revision: string } | undefined; - await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const desktopScreenshot = await withObsidianPage(session.remoteDebuggingPort, async (page) => { await page.setViewportSize({ width: 800, height: 600 }); const choice = await openCommand(page, "usage"); await choice.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).focus(); @@ -560,13 +577,15 @@ async function main(): Promise { hatch = await (await openLiveSyncSettings(page, timeout)).openPage("Hatch"); await assertFollowUpStopsAfterClose(page, hatch, usageFixtureRow.path); await captureObsidianElement(activeSession.remoteDebuggingPort, "database-diagnostics-hatch.png", (page) => - page.locator(".modal.mod-settings") + findVisibleElement(page, ".modal.mod-settings") ); const setting = hatch .locator(".setting-item") - .filter({ has: page.getByText("Inspect database integrity", { exact: true }) }); + .filter({ has: hatch.page().getByText("Inspect database integrity", { exact: true }) }); await setting.getByRole("button", { name: "Inspect", exact: true }).click(); - const progressDialogue = page.locator(".database-diagnostic"); + const progressDialogue = await findVisibleElement(page, ".database-diagnostic"); + const progressPage = progressDialogue.page(); + await progressPage.setViewportSize({ width: 800, height: 600 }); assert.equal( await progressDialogue .getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }) @@ -579,7 +598,7 @@ async function main(): Promise { .isChecked(), true ); - await assertDesktopDiagnosticChoiceLayout(page, progressDialogue); + await assertDesktopDiagnosticChoiceLayout(progressPage, progressDialogue); await installDiagnosticReadGate(cli, activeSession, shadowId, "metadata"); try { await progressDialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); @@ -593,7 +612,7 @@ async function main(): Promise { await captureObsidianElement( activeSession.remoteDebuggingPort, "database-diagnostics-progress.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await progressDialogue.getByRole("button", { name: "Cancel inspection", exact: true }).click(); await evalObsidianJson( @@ -601,7 +620,7 @@ async function main(): Promise { `(async()=>{globalThis.databaseDiagnosticReadGate?.release();return JSON.stringify(true);})()`, activeSession.cliEnv ); - await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await progressPage.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); await assertFormatChoices(progressDialogue, ["MARKDOWN", "JSON"]); await chooseFormat(progressDialogue, "json"); assert.equal(JSON.parse(await progressDialogue.locator("textarea").inputValue()).status, "cancelled"); @@ -612,33 +631,41 @@ async function main(): Promise { } await setting.getByRole("button", { name: "Inspect", exact: true }).click(); - const integrity = page.locator(".database-diagnostic"); + const integrity = await findVisibleElement(page, ".database-diagnostic"); + const integrityPage = integrity.page(); assert.equal( await integrity.getByRole("radio", { name: "Pseudonyms (recommended)", exact: true }).isChecked(), true ); await integrity.getByRole("button", { name: "Begin inspection", exact: true }).click(); - await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await integrityPage.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); await assertFormatChoices(integrity, ["MARKDOWN", "JSON"]); assert.equal(await integrity.getByRole("radio", { name: "MARKDOWN", exact: true }).isChecked(), true); const markdownReport = await integrity.locator("textarea").inputValue(); const markdownDetailsWithCodeBlock = /
\s*Database integrity report \(partial\)<\/summary>\s*(`{3,})text\r?\n[\s\S]*?\1\s*<\/details>/; assert.match(markdownReport, markdownDetailsWithCodeBlock); - await installDiagnosticExportInterception(page); + await installDiagnosticExportInterception(integrityPage); await integrity.getByRole("button", { name: "Copy report", exact: true }).click(); assert.equal( - await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), + await integrityPage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), markdownReport ); await integrity.getByRole("button", { name: "Save report", exact: true }).click(); - await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { - timeout, - }); - assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { - name: "database-integrity-partial.md", - content: markdownReport, - }); + await integrityPage.waitForFunction( + () => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, + undefined, + { + timeout, + } + ); + assert.deepEqual( + await integrityPage.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), + { + name: "database-integrity-partial.md", + content: markdownReport, + } + ); await chooseFormat(integrity, "json"); const reportText = await integrity.locator("textarea").inputValue(); const integrityReport = JSON.parse(reportText); @@ -670,15 +697,17 @@ async function main(): Promise { ); for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) assert.ok(!reportText.includes(privateValue)); + const screenshot = await captureObsidianElement( + activeSession.remoteDebuggingPort, + "database-diagnostics-desktop.png", + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") + ); + await integrity.getByRole("button", { name: "Close", exact: true }).click(); + await integrity.waitFor({ state: "hidden", timeout }); + await settingsNavigator.close(); + return screenshot; }); - const desktopScreenshot = await captureObsidianElement( - session.remoteDebuggingPort, - "database-diagnostics-desktop.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) - ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { - await page.locator(".database-diagnostic").getByRole("button", { name: "Close", exact: true }).click(); - await page.keyboard.press("Escape"); const localOnly = await openCommand(page, "integrity"); await localOnly.getByRole("checkbox", { name: "Include remote database", exact: true }).uncheck(); await localOnly.getByRole("button", { name: "Begin inspection", exact: true }).click(); @@ -780,7 +809,7 @@ async function main(): Promise { const mobileScreenshot = await captureObsidianElement( session.remoteDebuggingPort, "database-diagnostics-mobile.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { await page.setViewportSize({ width: 375, height: 667 }); @@ -828,7 +857,7 @@ async function main(): Promise { const mobileResultScreenshot = await captureObsidianElement( session.remoteDebuggingPort, "database-diagnostics-mobile-result.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await withObsidianPage(session.remoteDebuggingPort, async (page) => { await page.setViewportSize({ width: 375, height: 667 }); @@ -857,7 +886,7 @@ async function main(): Promise { await captureObsidianElement( activeSession.remoteDebuggingPort, "database-diagnostics-mobile-saved.png", - (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + (page) => findVisibleElement(page, ".modal-container:has(.database-diagnostic)") ); await dialogue.getByRole("button", { name: "Close", exact: true }).click(); await dialogue.waitFor({ state: "hidden", timeout }); From 96ad2737a7e846205e052da371e9a17cf8dfe43e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 13:56:54 +0000 Subject: [PATCH 36/68] Adopt Commonlib 0.2.0 with injected device storage --- devs.md | 2 + package-lock.json | 18 +- package.json | 3 +- src/apps/browser/LiveSyncBrowserUIService.ts | 4 +- .../createLiveSyncBrowserServiceHub.ts | 50 +---- ...eateLiveSyncBrowserServiceHub.unit.spec.ts | 43 ++++- src/apps/cli/README.md | 4 +- src/apps/cli/main.bootstrap.unit.spec.ts | 21 ++ src/apps/cli/main.ts | 4 - src/apps/cli/package.json | 2 +- src/apps/cli/services/NodeLocalStorage.ts | 159 +++++++-------- .../services/NodeLocalStorage.unit.spec.ts | 127 +++++++++--- src/apps/cli/services/NodeServiceHub.ts | 19 +- src/apps/cli/services/NodeSettingService.ts | 42 ---- src/apps/webapp/WebAppRuntime.ts | 14 +- src/apps/webpeer/src/WebPeerRuntime.ts | 10 +- src/common/replicatorProviders.ts | 8 + src/common/replicatorProviders.unit.spec.ts | 22 ++- src/common/replicatorResources.unit.spec.ts | 69 +++++++ src/common/replicatorResources/index.ts | 5 + .../replicatorResources/joinedDevices.ts | 40 ++++ src/common/turnSettingsPrivacy.unit.spec.ts | 12 +- .../P2PReplicator/P2PReplicatorPane.svelte | 134 +++++++++++-- .../ReviewHarness/ReviewHarnessView.ts | 3 +- .../ReviewHarness/reviewHarnessController.ts | 27 ++- .../reviewHarnessController.unit.spec.ts | 151 +++++++++++++-- .../features/ModuleObsidianSettingTab.ts | 6 +- .../ModuleObsidianSettingTab.unit.spec.ts | 34 +++- ...iveSyncSettingTab.declarative.unit.spec.ts | 14 +- .../ObsidianLiveSyncSettingTab.ts | 30 ++- .../ObsidianLiveSyncSettingTab.unit.spec.ts | 65 ++++++- .../features/SetupManager.unit.spec.ts | 7 +- .../SetupWizard/dialogs/SetupRemoteP2P.svelte | 44 ++++- src/modules/services/ObsidianServiceHub.ts | 9 +- src/modules/services/ObsidianServices.ts | 7 +- .../services/ObsidianSettingService.ts | 14 -- .../ObsidianSettingService.unit.spec.ts | 39 ++++ src/modules/services/ObsidianUIService.ts | 4 +- src/serviceFeatures/compatibilityReview.ts | 18 +- .../compatibilityReview.unit.spec.ts | 128 +++++++++++- src/serviceFeatures/redFlag.simpleFetch.ts | 37 ++-- .../redFlag.simpleFetch.unit.spec.ts | 84 +++++++- src/serviceFeatures/redFlag.unit.spec.ts | 22 ++- src/serviceFeatures/useReviewHarness.ts | 14 +- .../useReviewHarness.unit.spec.ts | 82 +++++++- test/apps/webapp/WebAppRuntime.unit.spec.ts | 17 +- test/apps/webpeer/WebPeerRuntime.unit.spec.ts | 41 +++- test/contracts/serviceContext.ts | 5 +- test/e2e-obsidian/runner/liveSyncWorkflow.ts | 4 +- test/e2e-obsidian/runner/upgradeWorkflow.ts | 4 +- test/e2e-obsidian/scripts/dialog-mounts.ts | 182 ++++++++++++------ test/e2e-obsidian/scripts/review-harness.ts | 14 +- test/e2e-obsidian/scripts/settings-ui.ts | 26 ++- updates.md | 4 + utils/couchdb/livesync-commonlib.ts | 4 +- utils/flyio/deno.lock | 57 +++++- utils/flyio/generate_setupuri.test.ts | 10 +- utils/livesync-commonlib-version.ts | 2 +- utils/setup/generate_setup_uri.test.ts | 7 +- utils/setup/livesync-commonlib.ts | 12 +- 60 files changed, 1499 insertions(+), 541 deletions(-) delete mode 100644 src/apps/cli/services/NodeSettingService.ts create mode 100644 src/common/replicatorResources/joinedDevices.ts diff --git a/devs.md b/devs.md index a4d3849d..77c31028 100644 --- a/devs.md +++ b/devs.md @@ -309,6 +309,8 @@ Existing legacy Modules continue to register their handlers in `onBindFunction() database and Replicator readiness. See [the declarative settings adapter ADR](docs/adr/2026_08_declarative_settings_adapter.md). - Use `this.services.setting.saveSettingData()` instead of using plugin methods directly +- Inject `@vrtmrz/value-store` at host composition. `setting.deviceValues` uses exact device-local keys; `setting.config` and the Service Hub `config` share the existing Vault and database-suffix scope. Await reads, writes, and deletions before advancing the workflow. Browser and Obsidian hosts retain their Web Storage keys, and the CLI retains its existing flat JSON string map. +- Seed the settings tab with `await settingTab.loadSettings()` before registration. Its editing buffer remains synchronous for Obsidian rendering; storage failures must not replace a previously loaded value with an empty default. ### Database Operations diff --git a/package-lock.json b/package-lock.json index 1d8d4dfb..4cf6a3e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,9 +23,10 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.44", + "@vrtmrz/livesync-commonlib": "0.2.0", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", + "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", "idb": "^8.0.3", "markdown-it": "^14.3.2", @@ -4115,9 +4116,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.44", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.44.tgz", - "integrity": "sha512-GjXNSLTnWLZzdgrhW3sIOsLqGLP4iMgypPMtcBc2Ju2vMq64TRA1Llfoya1KRIeu6m0G7Xu3ZtkWARTRcHkYYg==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.0.tgz", + "integrity": "sha512-K24Pkke+FTjZI7DjIV7qkZk+BmC9VebN3da+2cVD9SWOgoGl7OvDyXTHDafbrmmM4vi4ydaydAEh9glO1xHF6A==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -4128,6 +4129,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@trystero-p2p/nostr": "0.25.3", + "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", "events": "^3.3.0", "fflate": "^0.8.2", @@ -4139,9 +4141,11 @@ "pouchdb-adapter-idb": "^9.0.0", "pouchdb-adapter-indexeddb": "^9.0.0", "pouchdb-adapter-memory": "^9.0.0", + "pouchdb-checkpointer": "9.0.0", "pouchdb-core": "^9.0.0", "pouchdb-errors": "^9.0.0", "pouchdb-find": "^9.0.0", + "pouchdb-generate-replication-id": "9.0.0", "pouchdb-mapreduce": "^9.0.0", "pouchdb-merge": "^9.0.0", "pouchdb-replication": "^9.0.0", @@ -4210,6 +4214,12 @@ "integrity": "sha512-njT2BSFh57imwGDxWXJOcGbZOYFVNGmeXMrS7/RUdBoAeOjWzJB9TxI7WcR1sgQIXKh8TE9hqALU8lue5b8dRw==", "license": "MIT" }, + "node_modules/@vrtmrz/value-store": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@vrtmrz/value-store/-/value-store-0.1.0.tgz", + "integrity": "sha512-M62f1j0bnJkqpgI5sYxnkXUAOSDEwQbYrLf+w58gCCm4sjIXDmFzDJd2duXabRbktBlACt9P99yjN/9gOAPxrA==", + "license": "MIT" + }, "node_modules/abstract-leveldown": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", diff --git a/package.json b/package.json index a6ff13d4..f434524a 100644 --- a/package.json +++ b/package.json @@ -193,9 +193,10 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.44", + "@vrtmrz/livesync-commonlib": "0.2.0", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", + "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", "idb": "^8.0.3", "markdown-it": "^14.3.2", diff --git a/src/apps/browser/LiveSyncBrowserUIService.ts b/src/apps/browser/LiveSyncBrowserUIService.ts index ea23cf5a..c0ab9d00 100644 --- a/src/apps/browser/LiveSyncBrowserUIService.ts +++ b/src/apps/browser/LiveSyncBrowserUIService.ts @@ -1,6 +1,6 @@ import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; import type { AppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService"; -import type { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService"; +import type { IConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import type { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import type { ReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/base/ReplicatorService"; import type { InjectableAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAPIService"; @@ -11,7 +11,7 @@ import { BrowserSvelteDialogManager } from "./BrowserSvelteDialogManager"; export interface LiveSyncBrowserUIServiceDependencies { API: InjectableAPIService; appLifecycle: AppLifecycleService; - config: ConfigService; + config: IConfigService; control: ControlService; replicator: ReplicatorService; } diff --git a/src/apps/browser/createLiveSyncBrowserServiceHub.ts b/src/apps/browser/createLiveSyncBrowserServiceHub.ts index 76c671c0..3197114a 100644 --- a/src/apps/browser/createLiveSyncBrowserServiceHub.ts +++ b/src/apps/browser/createLiveSyncBrowserServiceHub.ts @@ -1,11 +1,10 @@ import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { KeyValueDatabaseFactory } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase"; import { PouchDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/pouchdb-browser"; -import { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService"; import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService"; import { KeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/KeyValueDBService"; -import type { ISettingService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import type { ValueDeleter, ValueStore } from "@vrtmrz/value-store"; import { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAppLifecycleService"; import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService"; @@ -24,10 +23,7 @@ import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/ import { setLang, translateLiveSyncMessage } from "@/common/translation"; import { BrowserConfirm } from "./BrowserConfirm"; import { createBrowserKeyValueDatabaseFactory } from "./BrowserKeyValueDatabase"; -import { - LiveSyncBrowserAPIService, - type LiveSyncBrowserAPIServiceOptions, -} from "./LiveSyncBrowserAPIService"; +import { LiveSyncBrowserAPIService, type LiveSyncBrowserAPIServiceOptions } from "./LiveSyncBrowserAPIService"; import { LiveSyncBrowserUIService } from "./LiveSyncBrowserUIService"; export interface LiveSyncBrowserSettingsPersistence { @@ -43,6 +39,7 @@ export interface LiveSyncBrowserRestartPolicy { } export interface LiveSyncBrowserServiceHubOptions { + deviceValues: ValueStore & ValueDeleter; context?: T; getSystemVaultName?: () => string; settings?: LiveSyncBrowserSettingsPersistence; @@ -51,35 +48,12 @@ export interface LiveSyncBrowserServiceHubOptions { API?: Omit; } -class LiveSyncBrowserAppLifecycleService< - T extends ServiceContext, -> extends InjectableAppLifecycleService {} +class LiveSyncBrowserAppLifecycleService extends InjectableAppLifecycleService {} class LiveSyncBrowserDatabaseService extends DatabaseService {} class LiveSyncBrowserKeyValueDBService extends KeyValueDBService {} -class LiveSyncBrowserConfigService extends ConfigService { - constructor( - context: T, - private readonly setting: ISettingService - ) { - super(context); - } - - getSmallConfig(key: string): string | null { - return this.setting.getSmallConfig(key); - } - - setSmallConfig(key: string, value: string): void { - this.setting.setSmallConfig(key, value); - } - - deleteSmallConfig(key: string): void { - this.setting.deleteSmallConfig(key); - } -} - /** LiveSync-owned service composition shared by WebApp and WebPeer. */ export class LiveSyncBrowserServiceHub extends InjectableServiceHub { constructor(options: LiveSyncBrowserServiceHubOptions) { @@ -98,17 +72,14 @@ export class LiveSyncBrowserServiceHub extends Injecta const setting = new InjectableSettingService(context, { APIService: API, onDisplayLanguageChanged: setLang, + deviceValues: options.deviceValues, }); const settingsPersistence = options.settings; setting.loadData.setHandler( - settingsPersistence - ? () => settingsPersistence.load() - : () => Promise.resolve(undefined) + settingsPersistence ? () => settingsPersistence.load() : () => Promise.resolve(undefined) ); setting.saveData.setHandler( - settingsPersistence - ? (settings) => settingsPersistence.save(settings) - : () => Promise.resolve() + settingsPersistence ? (settings) => settingsPersistence.save(settings) : () => Promise.resolve() ); const appLifecycle = new LiveSyncBrowserAppLifecycleService(context, { @@ -142,7 +113,7 @@ export class LiveSyncBrowserServiceHub extends Injecta setting, API, }); - const config = new LiveSyncBrowserConfigService(context, setting); + const config = setting.config; const replicator = new InjectableReplicatorService(context, { settingService: setting, appLifecycleService: appLifecycle, @@ -163,8 +134,7 @@ export class LiveSyncBrowserServiceHub extends Injecta databaseService: database, }); const keyValueDB = new LiveSyncBrowserKeyValueDBService(context, { - openKeyValueDatabase: - options.openKeyValueDatabase ?? createBrowserKeyValueDatabaseFactory(), + openKeyValueDatabase: options.openKeyValueDatabase ?? createBrowserKeyValueDatabaseFactory(), appLifecycle, databaseEvents, vault, @@ -209,7 +179,7 @@ export class LiveSyncBrowserServiceHub extends Injecta } export function createLiveSyncBrowserServiceHub( - options: LiveSyncBrowserServiceHubOptions = {} + options: LiveSyncBrowserServiceHubOptions ): LiveSyncBrowserServiceHub { return new LiveSyncBrowserServiceHub(options); } diff --git a/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts b/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts index 06f90938..558f4fb9 100644 --- a/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts +++ b/src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts @@ -1,5 +1,6 @@ import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { MemoryValueStore } from "@vrtmrz/value-store"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -21,7 +22,8 @@ describe("LiveSync browser service context contract", () => { const context = createServiceContext({ translate: (key) => `webapp:${key}`, }); - const hub = createLiveSyncBrowserServiceHub({ context }); + const deviceValues = new MemoryValueStore(); + const hub = createLiveSyncBrowserServiceHub({ context, deviceValues }); expect(observeServiceContext(context, "moduleLocalDatabase.logWaitingForReady")).toEqual({ translation: "webapp:moduleLocalDatabase.logWaitingForReady", @@ -32,20 +34,17 @@ describe("LiveSync browser service context contract", () => { expect(SERVICE_CONTEXT_MEMBERS.filter((member) => !composition.servicesUsingExpectedContext[member])).toEqual( [] ); + expect(hub.setting.deviceValues).toBe(deviceValues); + expect(hub.config).toBe(hub.setting.config); }); it("binds browser identity, persistence, restart policy, and native Fetch while composing the hub", async () => { - const deviceLocalSettings = new Map(); - vi.stubGlobal("localStorage", { - getItem: (key: string) => deviceLocalSettings.get(key) ?? null, - setItem: (key: string, value: string) => deviceLocalSettings.set(key, value), - removeItem: (key: string) => deviceLocalSettings.delete(key), - clear: () => deviceLocalSettings.clear(), - }); + const deviceValues = new MemoryValueStore(); const savedSettings: (typeof DEFAULT_SETTINGS)[] = []; const restartCalls: string[] = []; const nativeFetch = vi.fn(async () => new Response("ok")); const options: LiveSyncBrowserServiceHubOptions> = { + deviceValues, getSystemVaultName: () => "browser-vault", settings: { load: async () => ({ @@ -69,6 +68,8 @@ describe("LiveSync browser service context contract", () => { const hub = createLiveSyncBrowserServiceHub(options); + expect(hub.setting.deviceValues).toBe(deviceValues); + expect(hub.config).toBe(hub.setting.config); expect(hub.API.getSystemVaultName()).toBe("browser-vault"); expect(hub.API.getPlatform()).toBe("browser"); const response = await hub.API.nativeFetch("https://example.invalid/"); @@ -86,4 +87,30 @@ describe("LiveSync browser service context contract", () => { expect(hub.appLifecycle.isReloadingScheduled()).toBe(true); expect(restartCalls).toEqual(["schedule", "perform", "ask:restart now"]); }); + + it("keeps each browser composition bound to its own device value store", async () => { + const firstValues = new MemoryValueStore(); + const secondValues = new MemoryValueStore(); + const firstHub = createLiveSyncBrowserServiceHub({ + deviceValues: firstValues, + getSystemVaultName: () => "first-vault", + }); + const secondHub = createLiveSyncBrowserServiceHub({ + deviceValues: secondValues, + getSystemVaultName: () => "second-vault", + }); + + await firstHub.config.setValue("probe", "first"); + await secondHub.config.setValue("probe", "second"); + + expect(firstHub.setting.deviceValues).toBe(firstValues); + expect(secondHub.setting.deviceValues).toBe(secondValues); + expect(firstHub.config).toBe(firstHub.setting.config); + expect(secondHub.config).toBe(secondHub.setting.config); + expect(firstHub.config).not.toBe(secondHub.config); + await expect(firstHub.config.getValue("probe")).resolves.toBe("first"); + await expect(secondHub.config.getValue("probe")).resolves.toBe("second"); + await expect(firstValues.getValue("first-vault-probe")).resolves.toBe("first"); + await expect(secondValues.getValue("second-vault-probe")).resolves.toBe("second"); + }); }); diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 17674fc3..fb2a7827 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -719,8 +719,8 @@ src/apps/cli/ │ └── ServiceFileAccessImpl.ts ├── services/ │ ├── NodeKeyValueDBService.ts -│ ├── NodeServiceHub.ts -│ └── NodeSettingService.ts +│ ├── NodeLocalStorage.ts +│ └── NodeServiceHub.ts ├── test/ │ ├── test-e2e-two-vaults-common.sh │ ├── test-e2e-two-vaults-matrix.sh diff --git a/src/apps/cli/main.bootstrap.unit.spec.ts b/src/apps/cli/main.bootstrap.unit.spec.ts index 3fcc3913..033771a1 100644 --- a/src/apps/cli/main.bootstrap.unit.spec.ts +++ b/src/apps/cli/main.bootstrap.unit.spec.ts @@ -88,6 +88,27 @@ describe("CLI database preparation", () => { await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`); } + it("loads legacy device values and persists configuration through the service-owned store", async () => { + const localStoragePath = join(directory, ".livesync", "runtime", "local-storage.json"); + await mkdir(join(directory, ".livesync", "runtime"), { recursive: true }); + await writeFile(localStoragePath, JSON.stringify({ ["__proto__"]: "legacy value" })); + let legacyValue: string | null | undefined; + const runner = vi.fn(async (_options, { core }) => { + legacyValue = await core.services.setting.deviceValues.getValue("__proto__"); + expect(core.services.config).toBe(core.services.setting.config); + await core.services.config.setValue("bootstrap-probe", "config value"); + return false; + }); + + await start("ls", runner); + + expect(legacyValue).toBe("legacy value"); + const stored = JSON.parse(await readFile(localStoragePath, "utf-8")) as Record; + expect(Object.prototype.hasOwnProperty.call(stored, "__proto__")).toBe(true); + expect(stored["__proto__"]).toBe("legacy value"); + expect(Object.values(stored)).toContain("config value"); + }); + it.each([REMOTE_P2P, REMOTE_COUCHDB])( "restores configured P2P automation before the host connects (%s)", async (remoteType) => { diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 028bd465..cea0ec35 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -1,6 +1,5 @@ import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation"; import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub"; -import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage"; import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore"; import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules"; import { @@ -78,7 +77,6 @@ const VAULT_SYNC_MODES: Readonly>> }, }; -ensureGlobalNodeLocalStorage(); defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG; /** Injectable command boundary used by CLI integration probes. */ @@ -400,8 +398,6 @@ export async function main( const settingsPath = options.settingsPath ? path.resolve(options.settingsPath) : path.join(databasePath, SETTINGS_FILE); - configureNodeLocalStorage(path.join(databasePath, ".livesync", "runtime", "local-storage.json")); - // Resolve vault path: mirror positional argument takes priority, // then --vault flag, otherwise fall back to databasePath. const vaultPath = diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 538c0d99..367e9fce 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -12,7 +12,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", diff --git a/src/apps/cli/services/NodeLocalStorage.ts b/src/apps/cli/services/NodeLocalStorage.ts index d8149c38..c6997c60 100644 --- a/src/apps/cli/services/NodeLocalStorage.ts +++ b/src/apps/cli/services/NodeLocalStorage.ts @@ -1,115 +1,86 @@ +import { createDeletableValueStore, sanitiseValueStoreError, ValueStoreError } from "@vrtmrz/value-store"; +import type { ValueDeleter, ValueStore } from "@vrtmrz/value-store"; import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node"; -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; -type LocalStorageShape = { - getItem(key: string): string | null; - setItem(key: string, value: string): void; - removeItem(key: string): void; - clear(): void; -}; +type NodeLocalStorage = ValueStore & ValueDeleter; -class PersistentNodeLocalStorage { - private storagePath: string | undefined; - private localStore: Record = {}; +class JsonFileValueBackend { + private values: Map; - configure(storagePath: string) { - if (this.storagePath === storagePath) { - return; - } - this.storagePath = storagePath; - this.loadFromFile(); + constructor(private readonly storagePath: string) { + this.values = this.loadValues(); } - private loadFromFile() { - if (!this.storagePath) { - this.localStore = {}; + get(key: string): string | null { + return this.values.get(key) ?? null; + } + + set(key: string, value: string): void { + const nextValues = new Map(this.values); + nextValues.set(key, value); + this.writeValues(nextValues); + this.values = nextValues; + } + + delete(key: string): void { + if (!this.values.has(key)) { return; } + const nextValues = new Map(this.values); + nextValues.delete(key); + this.writeValues(nextValues); + this.values = nextValues; + } + + private loadValues(): Map { + let contents: string; try { - const loaded = JSON.parse(nodeFs.readFileSync(this.storagePath, "utf-8")) as Record; - this.localStore = { ...loaded }; + contents = nodeFs.readFileSync(this.storagePath, "utf-8"); + } catch (error) { + if (isMissingFileError(error)) { + return new Map(); + } + throw sanitiseValueStoreError(error); + } + + let document: unknown; + try { + document = JSON.parse(contents); } catch { - this.localStore = {}; + throw new ValueStoreError("invalid-data"); } + if (document === null || typeof document !== "object" || Array.isArray(document)) { + throw new ValueStoreError("invalid-data"); + } + + const values = new Map(); + for (const [key, value] of Object.entries(document)) { + if (typeof value !== "string") { + throw new ValueStoreError("invalid-data"); + } + values.set(key, value); + } + return values; } - private flushToFile() { - if (!this.storagePath) { - return; - } + private writeValues(values: ReadonlyMap): void { nodeFs.mkdirSync(nodePath.dirname(this.storagePath), { recursive: true }); - nodeFs.writeFileSync(this.storagePath, JSON.stringify(this.localStore, null, 2), "utf-8"); - } - - getItem(key: string): string | null { - return this.localStore[key] ?? null; - } - - setItem(key: string, value: string) { - this.localStore[key] = value; - this.flushToFile(); - } - - removeItem(key: string) { - if (!(key in this.localStore)) { - return; - } - delete this.localStore[key]; - this.flushToFile(); - } - - clear() { - this.localStore = {}; - this.flushToFile(); + nodeFs.writeFileSync(this.storagePath, JSON.stringify(Object.fromEntries(values), null, 2), "utf-8"); } } -const persistentNodeLocalStorage = new PersistentNodeLocalStorage(); - -function createNodeLocalStorageShim(): LocalStorageShape { - return { - getItem(key: string) { - return persistentNodeLocalStorage.getItem(key); - }, - setItem(key: string, value: string) { - persistentNodeLocalStorage.setItem(key, value); - }, - removeItem(key: string) { - persistentNodeLocalStorage.removeItem(key); - }, - clear() { - persistentNodeLocalStorage.clear(); - }, - }; -} - -export function ensureGlobalNodeLocalStorage() { - const g = compatGlobal as unknown as Record; - if ( - !("localStorage" in g) || - typeof (g.localStorage as Record | undefined)?.getItem !== "function" - ) { - g.localStorage = createNodeLocalStorageShim(); +function isMissingFileError(error: unknown): boolean { + if (typeof error !== "object" || error === null) { + return false; + } + try { + const code = Object.getOwnPropertyDescriptor(error, "code"); + return code !== undefined && Object.prototype.hasOwnProperty.call(code, "value") && code.value === "ENOENT"; + } catch { + return false; } } -export function configureNodeLocalStorage(storagePath: string) { - persistentNodeLocalStorage.configure(storagePath); - ensureGlobalNodeLocalStorage(); -} - -export function getNodeLocalStorageItem(key: string): string { - return persistentNodeLocalStorage.getItem(key) ?? ""; -} - -export function setNodeLocalStorageItem(key: string, value: string) { - persistentNodeLocalStorage.setItem(key, value); -} - -export function deleteNodeLocalStorageItem(key: string) { - persistentNodeLocalStorage.removeItem(key); -} - -export function clearNodeLocalStorage() { - persistentNodeLocalStorage.clear(); +export function createNodeLocalStorage(storagePath: string): NodeLocalStorage { + return createDeletableValueStore(new JsonFileValueBackend(storagePath)); } diff --git a/src/apps/cli/services/NodeLocalStorage.unit.spec.ts b/src/apps/cli/services/NodeLocalStorage.unit.spec.ts index fde5aae4..db35b755 100644 --- a/src/apps/cli/services/NodeLocalStorage.unit.spec.ts +++ b/src/apps/cli/services/NodeLocalStorage.unit.spec.ts @@ -1,58 +1,127 @@ import { fs, os, path } from "@vrtmrz/livesync-commonlib/node"; +import { ValueStoreError } from "@vrtmrz/value-store"; import { afterEach, describe, expect, it } from "vitest"; -import { - clearNodeLocalStorage, - configureNodeLocalStorage, - ensureGlobalNodeLocalStorage, - getNodeLocalStorageItem, - setNodeLocalStorageItem, -} from "./NodeLocalStorage"; +import { createNodeLocalStorage } from "./NodeLocalStorage"; describe("NodeLocalStorage", () => { const tempDirs: string[] = []; afterEach(() => { - clearNodeLocalStorage(); for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true, force: true }); } }); - it("persists values to the configured file", () => { - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); - tempDirs.push(tempDir); - const storagePath = path.join(tempDir, "runtime", "local-storage.json"); - - configureNodeLocalStorage(storagePath); - setNodeLocalStorageItem("checkpoint", "42"); - - const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record; - expect(saved.checkpoint).toBe("42"); - }); - - it("reloads persisted values when configured again", () => { + it("loads and preserves legacy flat string data, including empty values and exact keys", async () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); tempDirs.push(tempDir); const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const specialKey = "__proto__"; + const specialValue = "line one\né›Ē"; fs.mkdirSync(path.dirname(storagePath), { recursive: true }); - fs.writeFileSync(storagePath, JSON.stringify({ persisted: "value" }, null, 2), "utf-8"); + fs.writeFileSync(storagePath, JSON.stringify({ [specialKey]: specialValue, empty: "" }), "utf-8"); - configureNodeLocalStorage(storagePath); + const store = createNodeLocalStorage(storagePath); - expect(getNodeLocalStorageItem("persisted")).toBe("value"); + await expect(store.getValue(specialKey)).resolves.toBe(specialValue); + await expect(store.getValue("empty", "default")).resolves.toBe(""); + const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record; + expect(saved[specialKey]).toBe(specialValue); + expect(saved.empty).toBe(""); }); - it("installs a global localStorage shim backed by the same store", () => { + it("persists strings and objects, reloads them, and deletes keys", async () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); tempDirs.push(tempDir); const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const store = createNodeLocalStorage(storagePath); + const object = { enabled: false, entries: ["value", null, 0] }; - configureNodeLocalStorage(storagePath); - ensureGlobalNodeLocalStorage(); + await store.setValue("empty", ""); + await store.setValue("text", "stored value"); + await store.setObject("object", object); - globalThis.localStorage.setItem("shared", "state"); + const reloaded = createNodeLocalStorage(storagePath); + await expect(reloaded.getValue("empty", "default")).resolves.toBe(""); + await expect(reloaded.getValue("text")).resolves.toBe("stored value"); + await expect(reloaded.getObject("object")).resolves.toEqual(object); - expect(getNodeLocalStorageItem("shared")).toBe("state"); + await reloaded.delete("text"); + await expect(reloaded.getValue("text")).resolves.toBeNull(); + const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record; + expect(saved.text).toBeUndefined(); + expect(saved.object).toBe(JSON.stringify(object)); + }); + + it("treats a missing file as empty and creates its parent on the first write", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const store = createNodeLocalStorage(storagePath); + + await expect(store.getValue("absent")).resolves.toBeNull(); + expect(fs.existsSync(storagePath)).toBe(false); + await store.setValue("created", "value"); + expect(fs.existsSync(storagePath)).toBe(true); + }); + + it("keeps separate store instances isolated by their file paths", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const first = createNodeLocalStorage(path.join(tempDir, "first", "local-storage.json")); + const second = createNodeLocalStorage(path.join(tempDir, "second", "local-storage.json")); + + await first.setValue("shared-key", "first"); + await second.setValue("shared-key", "second"); + + await expect(first.getValue("shared-key")).resolves.toBe("first"); + await expect(second.getValue("shared-key")).resolves.toBe("second"); + }); + + it("surfaces malformed and unreadable files as sanitised value-store errors", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const malformedPath = path.join(tempDir, "malformed.json"); + fs.writeFileSync(malformedPath, "{ malformed", "utf-8"); + + let malformedError: unknown; + try { + createNodeLocalStorage(malformedPath); + } catch (error) { + malformedError = error; + } + expect(malformedError).toBeInstanceOf(ValueStoreError); + expect(malformedError).toMatchObject({ code: "invalid-data" }); + expect(fs.readFileSync(malformedPath, "utf-8")).toBe("{ malformed"); + + const unreadablePath = path.join(tempDir, "unreadable.json"); + fs.mkdirSync(unreadablePath); + expect(() => createNodeLocalStorage(unreadablePath)).toThrowError(ValueStoreError); + }); + + it("does not change its visible values when a write fails", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const storagePath = path.join(tempDir, "runtime", "local-storage.json"); + const store = createNodeLocalStorage(storagePath); + await store.setValue("retained", "before failure"); + + fs.rmSync(storagePath); + fs.mkdirSync(storagePath); + + await expect(store.setValue("new", "write fails")).rejects.toBeInstanceOf(ValueStoreError); + await expect(store.getValue("retained")).resolves.toBe("before failure"); + await expect(store.getValue("new")).resolves.toBeNull(); + }); + + it("does not install or modify a global localStorage value", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-")); + tempDirs.push(tempDir); + const before = Object.getOwnPropertyDescriptor(globalThis, "localStorage"); + + createNodeLocalStorage(path.join(tempDir, "runtime", "local-storage.json")); + + expect(Object.getOwnPropertyDescriptor(globalThis, "localStorage")).toEqual(before); }); }); diff --git a/src/apps/cli/services/NodeServiceHub.ts b/src/apps/cli/services/NodeServiceHub.ts index bed9cb16..484861c2 100644 --- a/src/apps/cli/services/NodeServiceHub.ts +++ b/src/apps/cli/services/NodeServiceHub.ts @@ -1,6 +1,5 @@ import type { AppLifecycleServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService"; import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; -import { ConfigServiceBrowserCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/browser/ConfigServiceBrowserCompat"; import type { ComponentHasResult, SvelteDialogManager, @@ -11,6 +10,7 @@ import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService"; import { InjectableDatabaseEventService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableDatabaseEventService"; import { InjectableFileProcessingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableFileProcessingService"; +import { InjectableSettingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableSettingService"; import { PathServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectablePathService"; import { InjectableRemoteService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableRemoteService"; import { InjectableReplicationService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableReplicationService"; @@ -21,7 +21,7 @@ import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/ import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import { HeadlessAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/headless/HeadlessAPIService"; import { NodeKeyValueDBService } from "./NodeKeyValueDBService"; -import { NodeSettingService } from "./NodeSettingService"; +import { createNodeLocalStorage } from "./NodeLocalStorage"; import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { path as nodePath } from "@vrtmrz/livesync-commonlib/node"; @@ -96,11 +96,11 @@ export class NodeServiceHub extends InjectableServ const conflict = new InjectableConflictService(context); const fileProcessing = new InjectableFileProcessingService(context); - const setting = new NodeSettingService( - context, - { APIService: API, onDisplayLanguageChanged: setLang }, - localStoragePath - ); + const setting = new InjectableSettingService(context, { + APIService: API, + onDisplayLanguageChanged: setLang, + deviceValues: createNodeLocalStorage(localStoragePath), + }); const appLifecycle = new NodeAppLifecycleService(context, { settingService: setting, @@ -132,10 +132,7 @@ export class NodeServiceHub extends InjectableServ setting, }); - const config = new ConfigServiceBrowserCompat(context, { - settingService: setting, - APIService: API, - }); + const config = setting.config; const replicator = new InjectableReplicatorService(context, { settingService: setting, diff --git a/src/apps/cli/services/NodeSettingService.ts b/src/apps/cli/services/NodeSettingService.ts deleted file mode 100644 index 771cb41c..00000000 --- a/src/apps/cli/services/NodeSettingService.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { EVENT_SETTING_SAVED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; -import { EVENT_REQUEST_RELOAD_SETTING_TAB } from "@/common/events"; -import { handlers } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; -import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; -import { SettingService, type SettingServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService"; -import { - configureNodeLocalStorage, - deleteNodeLocalStorageItem, - getNodeLocalStorageItem, - setNodeLocalStorageItem, -} from "./NodeLocalStorage"; - -export class NodeSettingService extends SettingService { - constructor(context: T, dependencies: SettingServiceDependencies, storagePath: string) { - super(context, dependencies); - configureNodeLocalStorage(storagePath); - this.onSettingSaved.addHandler((settings) => { - this.context.events.emitEvent(EVENT_SETTING_SAVED, settings); - return Promise.resolve(true); - }); - this.onSettingLoaded.addHandler((settings) => { - this.context.events.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB); - return Promise.resolve(true); - }); - } - - protected setItem(key: string, value: string) { - setNodeLocalStorageItem(key, value); - } - - protected getItem(key: string): string { - return getNodeLocalStorageItem(key); - } - - protected deleteItem(key: string): void { - deleteNodeLocalStorageItem(key); - } - - public saveData = handlers<{ saveData: (data: ObsidianLiveSyncSettings) => Promise }>().binder("saveData"); - public loadData = handlers<{ loadData: () => Promise }>().binder("loadData"); -} diff --git a/src/apps/webapp/WebAppRuntime.ts b/src/apps/webapp/WebAppRuntime.ts index 48405a4c..fbee662b 100644 --- a/src/apps/webapp/WebAppRuntime.ts +++ b/src/apps/webapp/WebAppRuntime.ts @@ -21,6 +21,8 @@ import { useCheckRemoteSize } from "@vrtmrz/livesync-commonlib/compat/serviceFea import { useRemoteConfiguration } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/remoteConfig"; import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature"; import type { UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult"; +import type { ValueStore, ValueDeleter } from "@vrtmrz/value-store"; +import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { createLiveSyncBrowserServiceHub, @@ -56,6 +58,7 @@ const DEFAULT_SETTINGS: Partial = { export type WebAppRuntimeStatusKind = "info" | "warning" | "error" | "success"; export interface WebAppRuntimeOptions { + deviceValues?: ValueStore & ValueDeleter; reportStatus?: (kind: WebAppRuntimeStatusKind, message: string) => void; scheduleReload?: (delayMilliseconds: number) => void; } @@ -71,7 +74,10 @@ export class WebAppRuntime { private paneHost: P2PReplicatorPaneHost | null = null; private restartScheduled = false; - constructor(rootHandle: FileSystemDirectoryHandle, options: WebAppRuntimeOptions = {}) { + constructor( + rootHandle: FileSystemDirectoryHandle, + private readonly options: WebAppRuntimeOptions = {} + ) { this.rootHandle = rootHandle; this.reportStatus = options.reportStatus ?? (() => {}); this.scheduleReload = @@ -89,6 +95,8 @@ export class WebAppRuntime { private createServiceOptions(): LiveSyncBrowserServiceHubOptions { return { + deviceValues: + this.options.deviceValues ?? createWebStorageValueStore(compatGlobal.localStorage, { prefix: "" }), getSystemVaultName: () => this.rootHandle.name || "livesync-webapp", settings: { save: async (data) => { @@ -219,7 +227,9 @@ export class WebAppRuntime { useCheckRemoteSize(core); useRemoteConfiguration(core); this.p2p = useP2PReplicatorFeature(core, undefined, undefined, { - prepareP2PSettings: useP2PSettingsPreparation(core.services.API.webCompatFetch.bind(core.services.API)), + prepareP2PSettings: useP2PSettingsPreparation( + core.services.API.webCompatFetch.bind(core.services.API) + ), }); this.paneHost = { services: core.services, diff --git a/src/apps/webpeer/src/WebPeerRuntime.ts b/src/apps/webpeer/src/WebPeerRuntime.ts index b4f69758..fd29947a 100644 --- a/src/apps/webpeer/src/WebPeerRuntime.ts +++ b/src/apps/webpeer/src/WebPeerRuntime.ts @@ -7,6 +7,8 @@ import { P2PLogCollector } from "@vrtmrz/livesync-commonlib/compat/replication/t import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature"; import { ServiceContext, type LiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; import type { P2PServiceViews } from "@vrtmrz/livesync-commonlib/p2p"; +import type { ValueStore, ValueDeleter } from "@vrtmrz/value-store"; +import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage"; import { unique } from "octagonal-wheels/collection"; import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase"; @@ -20,6 +22,7 @@ import { translateLiveSyncMessage } from "@/common/translation"; import { WEBPEER_STORE_NAME, createWebPeerPersistence } from "./WebPeerPersistence"; export interface WebPeerRuntimeOptions { + deviceValues?: ValueStore & ValueDeleter; context?: ServiceContext; store?: SimpleStore; deviceName?: string; @@ -62,6 +65,7 @@ export class WebPeerRuntime { this.context = options.context ?? new ServiceContext({ translate: translateLiveSyncMessage }); this.services = createLiveSyncBrowserServiceHub({ context: this.context, + deviceValues: options.deviceValues ?? createWebStorageValueStore(compatGlobal.localStorage, { prefix: "" }), getSystemVaultName: () => options.systemVaultName ?? WEBPEER_STORE_NAME, settings: persistence.settings, restart: { @@ -103,7 +107,7 @@ export class WebPeerRuntime { await this.services.setting.loadSettings(); const deviceName = this.options.deviceName?.trim(); if (deviceName) { - this.services.config.setSmallConfig(SETTING_KEY_P2P_DEVICE_NAME, deviceName); + await this.services.config.setValue(SETTING_KEY_P2P_DEVICE_NAME, deviceName); } const opened = await this.services.database.openDatabase({ replicator: this.services.replicator, @@ -192,7 +196,7 @@ export class WebPeerRuntime { ); } - getDeviceName(): string { - return this.services.config.getSmallConfig(SETTING_KEY_P2P_DEVICE_NAME) || this.services.vault.getVaultName(); + async getDeviceName(): Promise { + return (await this.services.config.getValue(SETTING_KEY_P2P_DEVICE_NAME)) || this.services.vault.getVaultName(); } } diff --git a/src/common/replicatorProviders.ts b/src/common/replicatorProviders.ts index 845d8c2e..f3cc40b0 100644 --- a/src/common/replicatorProviders.ts +++ b/src/common/replicatorProviders.ts @@ -28,6 +28,8 @@ import { getObjectStorageReplicatorConfigurationIdentity, } from "./replicatorConfigurationIdentity"; import { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, createCouchDBConnectionProbeFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, @@ -115,6 +117,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getCouchDBReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncCouchDBReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createCouchDBJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability(createCouchDBConnectionProbeFactory(host)), [REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK]: supportedCapability( createCouchDBPreferredTweakProbeFactory(host) @@ -141,6 +146,9 @@ export function createCentralReplicatorProviderDefinitions( configurationIdentity: getObjectStorageReplicatorConfigurationIdentity, create: () => Promise.resolve(new LiveSyncJournalReplicator(host)), remoteResources: { + [REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability( + createObjectStorageJoinedDevicesResourceFactory(host) + ), [REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability( createObjectStorageConnectionProbeFactory(host) ), diff --git a/src/common/replicatorProviders.unit.spec.ts b/src/common/replicatorProviders.unit.spec.ts index 54a2c441..97371072 100644 --- a/src/common/replicatorProviders.unit.spec.ts +++ b/src/common/replicatorProviders.unit.spec.ts @@ -44,7 +44,16 @@ describe("central Replicator provider definitions", () => { it("keeps the retained remote-resource catalogue bounded", () => { expect .soft(Object.values(REMOTE_RESOURCE_KINDS).sort()) - .toEqual(["connection", "preferred-tweak", "security-seed", "synchronisation-information"].sort()); + .toEqual( + [ + "connection", + "initial-fetch-fast-forward", + "joined-devices", + "preferred-tweak", + "security-seed", + "synchronisation-information", + ].sort() + ); }); it("lets Journal prepare its own fresh Security Seed while CouchDB uses central preparation", () => { @@ -106,8 +115,15 @@ describe("central Replicator provider definitions", () => { const couchAdministration = definitions.get(REMOTE_COUCHDB)?.centralRemoteAdministration; const objectAdministration = definitions.get(REMOTE_MINIO)?.centralRemoteAdministration; - expect(Object.keys(couchResources ?? {}).sort()).toEqual(Object.values(REMOTE_RESOURCE_KINDS).sort()); - expect(Object.keys(objectResources ?? {}).sort()).toEqual(Object.values(REMOTE_RESOURCE_KINDS).sort()); + const requiredResources = Object.values(REMOTE_RESOURCE_KINDS).filter( + (kind) => kind !== REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD + ); + expect(Object.keys(couchResources ?? {}).sort()).toEqual(requiredResources.sort()); + expect(Object.keys(objectResources ?? {}).sort()).toEqual(requiredResources.sort()); + expect(couchResources?.[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); + expect(objectResources?.[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); + expect(couchResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]).toBeUndefined(); + expect(objectResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]).toBeUndefined(); expect(couchResources?.[REMOTE_RESOURCE_KINDS.CONNECTION].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(couchResources?.[REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); expect(couchResources?.[REMOTE_RESOURCE_KINDS.SECURITY_SEED].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED); diff --git a/src/common/replicatorResources.unit.spec.ts b/src/common/replicatorResources.unit.spec.ts index 2215a48a..0dffd7d0 100644 --- a/src/common/replicatorResources.unit.spec.ts +++ b/src/common/replicatorResources.unit.spec.ts @@ -12,6 +12,8 @@ const mocks = vi.hoisted(() => ({ getRemoteStatus: ReturnType; getRemotePreferredTweakValues: ReturnType; getReplicationPBKDF2Salt: ReturnType; + getJoinedDevices: ReturnType; + forgetJoinedDevice: ReturnType; closeReplication: ReturnType; }>, objectStorage: [] as Array<{ @@ -20,6 +22,8 @@ const mocks = vi.hoisted(() => ({ getRemoteStatus: ReturnType; getRemotePreferredTweakValues: ReturnType; getReplicationPBKDF2Salt: ReturnType; + getJoinedDevices: ReturnType; + forgetJoinedDevice: ReturnType; closeReplication: ReturnType; }>, checkSyncInfo: vi.fn(async () => true), @@ -42,6 +46,8 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicato getRemoteStatus = vi.fn(); getRemotePreferredTweakValues = vi.fn(); getReplicationPBKDF2Salt = vi.fn(); + getJoinedDevices = vi.fn(); + forgetJoinedDevice = vi.fn(); closeReplication = vi.fn(); constructor(host: unknown) { @@ -58,6 +64,8 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe getRemoteStatus = vi.fn(); getRemotePreferredTweakValues = vi.fn(); getReplicationPBKDF2Salt = vi.fn(); + getJoinedDevices = vi.fn(); + forgetJoinedDevice = vi.fn(); closeReplication = vi.fn(); constructor(host: unknown) { @@ -69,10 +77,12 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe import { createCouchDBConnectionProbeFactory, + createCouchDBJoinedDevicesResourceFactory, createCouchDBPreferredTweakProbeFactory, createCouchDBSecuritySeedResourceFactory, createCouchDBSynchronisationInformationResourceFactory, createObjectStorageConnectionProbeFactory, + createObjectStorageJoinedDevicesResourceFactory, createObjectStoragePreferredTweakProbeFactory, createObjectStorageSecuritySeedResourceFactory, } from "./replicatorResources"; @@ -226,6 +236,65 @@ describe("replicator probe factories", () => { expect(preferredReplicator.getRemotePreferredTweakValues).toHaveBeenCalledWith(snapshot); }); + it.each([ + { + kind: "CouchDB", + remoteType: REMOTE_COUCHDB, + createFactory: createCouchDBJoinedDevicesResourceFactory, + instances: () => mocks.couchDB, + }, + { + kind: "Object Storage", + remoteType: REMOTE_MINIO, + createFactory: createObjectStorageJoinedDevicesResourceFactory, + instances: () => mocks.objectStorage, + }, + ])( + "keeps $kind joined-device operations on an owned settings snapshot", + async ({ remoteType, createFactory, instances }) => { + const host = { name: "host" }; + const source = createSettings({ remoteType }); + const snapshot = { ...source }; + const factory = createFactory(host as never); + const resource = await factory(source); + const nextResource = await factory(source); + const [replicator, nextReplicator] = instances(); + expect(instances()).toHaveLength(2); + expect(replicator.host).toBe(host); + expect(replicator.getJoinedDevices).not.toHaveBeenCalled(); + expect(replicator.forgetJoinedDevice).not.toHaveBeenCalled(); + const devices = { + milestonePresent: true, + locked: false, + cleaned: false, + devices: [{ id: "other", accepted: true }], + }; + replicator.getJoinedDevices.mockResolvedValue(devices); + replicator.forgetJoinedDevice.mockResolvedValue("conflict"); + + source.couchDB_DBNAME = "changed-vault"; + source.bucket = "changed-bucket"; + await expect(resource.read()).resolves.toBe(devices); + await expect(resource.forget("other")).resolves.toBe("conflict"); + expect(replicator.getJoinedDevices).toHaveBeenCalledWith(snapshot); + expect(replicator.getJoinedDevices.mock.calls[0][0]).not.toBe(source); + expect(replicator.forgetJoinedDevice).toHaveBeenCalledWith(snapshot, "other"); + + const failure = new Error("remote unavailable"); + replicator.getJoinedDevices.mockRejectedValue(failure); + await expect(resource.read()).rejects.toBe(failure); + replicator.forgetJoinedDevice.mockRejectedValue(failure); + await expect(resource.forget("other")).rejects.toBe(failure); + const disposal = resource.dispose(); + expect(resource.dispose()).toBe(disposal); + await disposal; + expect(replicator.closeReplication).toHaveBeenCalledOnce(); + expect(nextReplicator.closeReplication).not.toHaveBeenCalled(); + await nextResource.dispose(); + expect(nextReplicator.closeReplication).toHaveBeenCalledOnce(); + } + ); + it("shares one successful asynchronous disposal promise for every probe kind", async () => { const couchProbe = await createCouchDBPreferredTweakProbeFactory({} as never)(createSettings()); const objectProbe = await createObjectStoragePreferredTweakProbeFactory({} as never)( diff --git a/src/common/replicatorResources/index.ts b/src/common/replicatorResources/index.ts index 9888e85c..f2f359f1 100644 --- a/src/common/replicatorResources/index.ts +++ b/src/common/replicatorResources/index.ts @@ -14,3 +14,8 @@ export { type SecuritySeedResourceHost, } from "./securitySeed"; export { createCouchDBSynchronisationInformationResourceFactory } from "./synchronisationInformation"; + +export { + createCouchDBJoinedDevicesResourceFactory, + createObjectStorageJoinedDevicesResourceFactory, +} from "./joinedDevices"; diff --git a/src/common/replicatorResources/joinedDevices.ts b/src/common/replicatorResources/joinedDevices.ts new file mode 100644 index 00000000..a9acf1c1 --- /dev/null +++ b/src/common/replicatorResources/joinedDevices.ts @@ -0,0 +1,40 @@ +import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { + JoinedDevicesResource, + JoinedDevicesResourceFactory, + JoinedDevicesSnapshot, + ForgetJoinedDeviceResult, +} from "@vrtmrz/livesync-commonlib/replication"; +import { + LiveSyncCouchDBReplicator, + type LiveSyncCouchDBReplicatorEnv, +} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; +import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator"; +import { createReplicatorDisposer, snapshotRemoteSettings, type ResourceReplicator } from "./shared"; + +interface JoinedDevicesReplicator extends ResourceReplicator { + getJoinedDevices(setting: RemoteDBSettings): Promise; + forgetJoinedDevice(setting: RemoteDBSettings, deviceId: string): Promise; +} + +function createResource(replicator: JoinedDevicesReplicator, snapshot: RemoteDBSettings): JoinedDevicesResource { + return { + read: () => replicator.getJoinedDevices(snapshot), + forget: (deviceId) => replicator.forgetJoinedDevice(snapshot, deviceId), + dispose: createReplicatorDisposer(replicator), + }; +} + +export function createCouchDBJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncCouchDBReplicator(host), snapshotRemoteSettings(setting))); +} + +export function createObjectStorageJoinedDevicesResourceFactory( + host: LiveSyncCouchDBReplicatorEnv +): JoinedDevicesResourceFactory { + return (setting) => + Promise.resolve(createResource(new LiveSyncJournalReplicator(host), snapshotRemoteSettings(setting))); +} diff --git a/src/common/turnSettingsPrivacy.unit.spec.ts b/src/common/turnSettingsPrivacy.unit.spec.ts index c0243c79..451db93b 100644 --- a/src/common/turnSettingsPrivacy.unit.spec.ts +++ b/src/common/turnSettingsPrivacy.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { describe, expect, it } from "vitest"; import { DEFAULT_SETTINGS, @@ -18,17 +19,7 @@ import { } from "./turnSettingsPrivacy"; class MemorySettingService extends SettingService { - readonly items = new Map(); saved?: ObsidianLiveSyncSettings; - protected setItem(key: string, value: string) { - this.items.set(key, value); - } - protected getItem(key: string) { - return this.items.get(key) ?? ""; - } - protected deleteItem(key: string) { - this.items.delete(key); - } protected saveData(settings: ObsidianLiveSyncSettings) { this.saved = structuredClone(settings); return Promise.resolve(); @@ -70,6 +61,7 @@ describe("managed TURN settings privacy", () => { const originalURI = ConnectionStringParser.serialize({ type: "p2p", settings: current }); current.remoteConfigurations.managed.uri = originalURI; const service = new MemorySettingService(new ServiceContext(), { + deviceValues: new MemoryValueStore(), APIService: { getSystemVaultName: () => "test-vault", getAppID: () => "test-app", diff --git a/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte b/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte index f8bf13ab..39ad12fc 100644 --- a/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte +++ b/src/features/P2PSync/P2PReplicator/P2PReplicatorPane.svelte @@ -1,5 +1,6 @@
+ {#each Object.values(storageErrors) as storageError} +

{storageError}

+ {/each}

{_msg("Peer to Peer Replicator")}

{_msg("P2P.Note.Summary")} @@ -345,6 +440,7 @@ type="text" placeholder="iphone-16" bind:value={eDeviceName} + disabled={!deviceNameReady} autocomplete="off" /> @@ -383,7 +479,7 @@ --> - diff --git a/src/features/ReviewHarness/ReviewHarnessView.ts b/src/features/ReviewHarness/ReviewHarnessView.ts index b116e9cf..2d1ae0f0 100644 --- a/src/features/ReviewHarness/ReviewHarnessView.ts +++ b/src/features/ReviewHarness/ReviewHarnessView.ts @@ -1,4 +1,5 @@ import { ItemView, Notice, Setting, type WorkspaceLeaf } from "@/deps"; +import { LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger"; import { REVIEW_HARNESS_SCENARIOS, type ReviewHarnessScenarioId, @@ -101,7 +102,7 @@ export class ReviewHarnessView extends ItemView { true ); this.addActionButton(actions, "Restart and return", "review-harness-restart", () => - this.controller.prepareCompatibilityReviewRestart() + this.controller.prepareCompatibilityReviewRestart().catch((error) => Logger(error, LOG_LEVEL_NOTICE)) ); } } diff --git a/src/features/ReviewHarness/reviewHarnessController.ts b/src/features/ReviewHarness/reviewHarnessController.ts index 2bde2f58..1cec69aa 100644 --- a/src/features/ReviewHarness/reviewHarnessController.ts +++ b/src/features/ReviewHarness/reviewHarnessController.ts @@ -23,9 +23,9 @@ export interface ReviewHarnessRuntime { runVaultRoundTrip(): Promise; runIdBenchmark(): Promise; runFileContentHashBenchmark(): Promise; - readContinuation(): string | null; - writeContinuation(value: string): void; - deleteContinuation(): void; + readContinuation(): Promise; + writeContinuation(value: string): Promise; + deleteContinuation(): Promise; restart(): void; reportError(error: unknown): void; copyText(value: string): Promise; @@ -68,6 +68,7 @@ export class ReviewHarnessController { private current: ReviewHarnessScenarioId | null = null; private resumedRequestId: string | null = null; private continuationError: string | null = null; + private continuationConsumption: Promise | undefined; constructor(private readonly runtime: ReviewHarnessRuntime) {} @@ -102,10 +103,20 @@ export class ReviewHarnessController { }; } - consumeContinuation(): void { - const serialised = this.runtime.readContinuation(); + consumeContinuation(): Promise { + if (this.continuationConsumption) return this.continuationConsumption; + const consumption = this.consumeContinuationFromStore(); + const trackedConsumption = consumption.finally(() => { + if (this.continuationConsumption === trackedConsumption) this.continuationConsumption = undefined; + }); + this.continuationConsumption = trackedConsumption; + return trackedConsumption; + } + + private async consumeContinuationFromStore(): Promise { + const serialised = await this.runtime.readContinuation(); if (!serialised) return; - this.runtime.deleteContinuation(); + await this.runtime.deleteContinuation(); const parsed = parsePendingReviewRun(serialised); if (!parsed.pendingRun) { this.continuationError = parsed.error ?? "The Review Harness continuation was invalid."; @@ -232,7 +243,7 @@ export class ReviewHarnessController { this.record("scenario-failed", id); } - prepareCompatibilityReviewRestart(): void { + async prepareCompatibilityReviewRestart(): Promise { const requestedAt = this.runtime.now().toISOString(); const pending = { formatVersion: 1, @@ -241,7 +252,7 @@ export class ReviewHarnessController { stage: "awaiting-restart", requestedAt, } as const; - this.runtime.writeContinuation(JSON.stringify(pending)); + await this.runtime.writeContinuation(JSON.stringify(pending)); this.results["compatibility-review"] = { status: "waiting-for-user", detail: "Restart requested. The one-shot continuation will be removed before the review resumes.", diff --git a/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts index cce28869..2b00cd23 100644 --- a/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts +++ b/src/features/ReviewHarness/reviewHarnessController.unit.spec.ts @@ -90,14 +90,14 @@ function createRuntime(): ReviewHarnessRuntime & { detail: "File content hash measurements completed.", observations: ["SHA-256, 4 KiB: median=1.00 ms; range=1.00–1.00 ms"], })), - readContinuation() { + async readContinuation() { return this.continuation; }, - writeContinuation(value) { + async writeContinuation(value) { this.events.push("write"); this.continuation = value; }, - deleteContinuation() { + async deleteContinuation() { this.events.push("delete"); this.continuation = null; }, @@ -251,7 +251,7 @@ describe("ReviewHarnessController", () => { expect(controller.snapshot().running).toBe(false); }); - it("deletes a one-shot continuation before exposing the resumed guided step", () => { + it("deletes a one-shot continuation before exposing the resumed guided step", async () => { const runtime = createRuntime(); runtime.continuation = JSON.stringify({ formatVersion: 1, @@ -262,7 +262,7 @@ describe("ReviewHarnessController", () => { }); const controller = new ReviewHarnessController(runtime); - controller.consumeContinuation(); + await controller.consumeContinuation(); expect(runtime.events).toEqual(["delete"]); expect(controller.snapshot().results["compatibility-review"]).toMatchObject({ @@ -271,7 +271,7 @@ describe("ReviewHarnessController", () => { expect(controller.snapshot().resumedRequestId).toBe("compatibility-review-2026-07-18T11:59:00.000Z"); }); - it("does not copy rejected continuation values into the report", () => { + it("does not copy rejected continuation values into the report", async () => { const runtime = createRuntime(); runtime.continuation = JSON.stringify({ formatVersion: 1, @@ -282,7 +282,7 @@ describe("ReviewHarnessController", () => { }); const controller = new ReviewHarnessController(runtime); - controller.consumeContinuation(); + await controller.consumeContinuation(); expect(controller.snapshot().continuationError).toContain("private-vault"); expect(controller.snapshot().results["compatibility-review"].detail).toBe( @@ -292,7 +292,7 @@ describe("ReviewHarnessController", () => { expect(controller.createReport()).not.toContain("secret-note.md"); }); - it("does not accept an injected request ID which could enter the report transcript", () => { + it("does not accept an injected request ID which could enter the report transcript", async () => { const runtime = createRuntime(); runtime.continuation = JSON.stringify({ formatVersion: 1, @@ -303,18 +303,18 @@ describe("ReviewHarnessController", () => { }); const controller = new ReviewHarnessController(runtime); - controller.consumeContinuation(); + await controller.consumeContinuation(); expect(controller.snapshot().results["compatibility-review"].status).toBe("failed"); expect(controller.createReport()).not.toContain("private-vault"); expect(controller.createReport()).not.toContain("secret-note.md"); }); - it("persists the fixed continuation before requesting restart", () => { + it("persists the fixed continuation before requesting restart", async () => { const runtime = createRuntime(); const controller = new ReviewHarnessController(runtime); - controller.prepareCompatibilityReviewRestart(); + await controller.prepareCompatibilityReviewRestart(); expect(runtime.events).toEqual(["write", "restart"]); expect(JSON.parse(runtime.continuation!)).toMatchObject({ @@ -324,6 +324,135 @@ describe("ReviewHarnessController", () => { }); }); + it("waits for the continuation save before requesting restart", async () => { + const runtime = createRuntime(); + let markSaveStarted!: () => void; + const saveStarted = new Promise((resolve) => { + markSaveStarted = resolve; + }); + let finishSave!: () => void; + const saving = new Promise((resolve) => { + finishSave = resolve; + }); + runtime.writeContinuation = vi.fn(async (value) => { + runtime.events.push("write-started"); + markSaveStarted(); + await saving; + runtime.continuation = value; + runtime.events.push("write-saved"); + }); + const controller = new ReviewHarnessController(runtime); + + const preparing = controller.prepareCompatibilityReviewRestart(); + await saveStarted; + + expect(runtime.events).toEqual(["write-started"]); + expect(runtime.restart).not.toHaveBeenCalled(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("idle"); + + finishSave(); + await preparing; + + expect(runtime.events).toEqual(["write-started", "write-saved", "restart"]); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + + it("keeps the guided review retryable when saving its continuation fails", async () => { + const runtime = createRuntime(); + const controller = new ReviewHarnessController(runtime); + await controller.runScenario("compatibility-review"); + runtime.writeContinuation = vi.fn(async () => { + runtime.events.push("write-failed"); + throw new Error("continuation save failed"); + }); + + await expect(controller.prepareCompatibilityReviewRestart()).rejects.toThrow("continuation save failed"); + + expect(runtime.events).toEqual(["write-failed"]); + expect(runtime.restart).not.toHaveBeenCalled(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + + it("waits for one continuation deletion before exposing it and coalesces concurrent consumption", async () => { + const runtime = createRuntime(); + runtime.continuation = JSON.stringify({ + formatVersion: 1, + requestId: "compatibility-review-2026-07-18T11:59:00.000Z", + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt: "2026-07-18T11:59:00.000Z", + }); + let markDeleteStarted!: () => void; + const deleteStarted = new Promise((resolve) => { + markDeleteStarted = resolve; + }); + let finishDelete!: () => void; + const deleting = new Promise((resolve) => { + finishDelete = resolve; + }); + runtime.readContinuation = vi.fn(async () => { + runtime.events.push("read"); + return runtime.continuation; + }); + runtime.deleteContinuation = vi.fn(async () => { + runtime.events.push("delete-started"); + markDeleteStarted(); + await deleting; + runtime.continuation = null; + runtime.events.push("delete-finished"); + }); + const controller = new ReviewHarnessController(runtime); + + const firstConsumption = controller.consumeContinuation(); + const duplicateConsumption = controller.consumeContinuation(); + await deleteStarted; + + expect(duplicateConsumption).toBe(firstConsumption); + expect(runtime.events).toEqual(["read", "delete-started"]); + expect(controller.snapshot().resumedRequestId).toBeNull(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("idle"); + + finishDelete(); + await firstConsumption; + + expect(runtime.events).toEqual(["read", "delete-started", "delete-finished"]); + expect(runtime.readContinuation).toHaveBeenCalledOnce(); + expect(runtime.deleteContinuation).toHaveBeenCalledOnce(); + expect(controller.snapshot().resumedRequestId).toBe("compatibility-review-2026-07-18T11:59:00.000Z"); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + + it("keeps a failed continuation deletion available for retry", async () => { + const runtime = createRuntime(); + runtime.continuation = JSON.stringify({ + formatVersion: 1, + requestId: "compatibility-review-2026-07-18T11:59:00.000Z", + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt: "2026-07-18T11:59:00.000Z", + }); + runtime.deleteContinuation = vi + .fn() + .mockRejectedValueOnce(new Error("continuation delete failed")) + .mockImplementation(async () => { + runtime.events.push("delete"); + runtime.continuation = null; + }); + const controller = new ReviewHarnessController(runtime); + + await expect(controller.consumeContinuation()).rejects.toThrow("continuation delete failed"); + + expect(runtime.continuation).not.toBeNull(); + expect(controller.snapshot().resumedRequestId).toBeNull(); + expect(controller.snapshot().results["compatibility-review"].status).toBe("idle"); + + await controller.consumeContinuation(); + + expect(runtime.continuation).toBeNull(); + expect(controller.snapshot().resumedRequestId).toBe("compatibility-review-2026-07-18T11:59:00.000Z"); + expect(controller.snapshot().results["compatibility-review"].status).toBe("waiting-for-user"); + }); + it("passes only after the actual compatibility review clears its pause", async () => { const runtime = createRuntime(); const controller = new ReviewHarnessController(runtime); diff --git a/src/modules/features/ModuleObsidianSettingTab.ts b/src/modules/features/ModuleObsidianSettingTab.ts index 727eee81..ccb51bbd 100644 --- a/src/modules/features/ModuleObsidianSettingTab.ts +++ b/src/modules/features/ModuleObsidianSettingTab.ts @@ -8,13 +8,13 @@ import { openObsidianSettings } from "@/common/obsidianSettings.ts"; export class ModuleObsidianSettingDialogue extends AbstractObsidianModule { settingTab!: ObsidianLiveSyncSettingTab; - _everyOnloadAfterLoadSettings(): Promise { + async _everyOnloadAfterLoadSettings(): Promise { this.settingTab = new ObsidianLiveSyncSettingTab(this.app, this.plugin); - this.settingTab.reloadAllSettings(true); + await this.settingTab.loadSettings(); this.plugin.addSettingTab(this.settingTab); eventHub.onEvent(EVENT_REQUEST_OPEN_SETTINGS, () => this.openSetting()); - return Promise.resolve(true); + return true; } openSetting() { diff --git a/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts b/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts index b1840ea0..87ddc9eb 100644 --- a/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts +++ b/src/modules/features/ModuleObsidianSettingTab.unit.spec.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const settingTabState = vi.hoisted(() => ({ callOrder: [] as string[], - reloadAllSettings: vi.fn<(skipUpdate?: boolean) => void>(), + loadSettings: vi.fn<() => Promise>(), })); const eventHubState = vi.hoisted(() => ({ @@ -11,9 +11,9 @@ const eventHubState = vi.hoisted(() => ({ vi.mock("./SettingDialogue/ObsidianLiveSyncSettingTab.ts", () => ({ ObsidianLiveSyncSettingTab: class ObsidianLiveSyncSettingTab { - reloadAllSettings(skipUpdate?: boolean) { - settingTabState.callOrder.push(`reload:${String(skipUpdate)}`); - settingTabState.reloadAllSettings(skipUpdate); + async loadSettings() { + await settingTabState.loadSettings(); + settingTabState.callOrder.push("settings-loaded"); } }, })); @@ -65,7 +65,7 @@ function createModuleHarness() { describe("ModuleObsidianSettingDialogue startup lifecycle", () => { beforeEach(() => { settingTabState.callOrder.length = 0; - settingTabState.reloadAllSettings.mockClear(); + settingTabState.loadSettings.mockReset().mockResolvedValue(undefined); eventHubState.onEvent.mockClear(); }); @@ -79,13 +79,29 @@ describe("ModuleObsidianSettingDialogue startup lifecycle", () => { }); it("seeds the setting editor without requesting a render before registration", async () => { - const { initialisationHandler, settingsLoadedHandler } = createModuleHarness(); + const { initialisationHandler, settingsLoadedHandler, plugin } = createModuleHarness(); + let finishRead!: () => void; + settingTabState.loadSettings.mockReturnValueOnce( + new Promise((resolve) => { + finishRead = resolve; + }) + ); const handler = settingsLoadedHandler() ?? initialisationHandler(); expect(handler).toBeTypeOf("function"); - await handler!(); + const loading = handler!(); + expect(plugin.addSettingTab).not.toHaveBeenCalled(); + finishRead(); + await loading; - expect(settingTabState.reloadAllSettings).toHaveBeenCalledWith(true); - expect(settingTabState.callOrder).toEqual(["reload:true", "add-setting-tab"]); + expect(settingTabState.loadSettings).toHaveBeenCalledOnce(); + expect(settingTabState.callOrder).toEqual(["settings-loaded", "add-setting-tab"]); + }); + it("does not register an editor when reading device values fails", async () => { + const { settingsLoadedHandler, plugin } = createModuleHarness(); + const failure = new Error("device storage unavailable"); + settingTabState.loadSettings.mockRejectedValueOnce(failure); + await expect(settingsLoadedHandler()!()).rejects.toBe(failure); + expect(plugin.addSettingTab).not.toHaveBeenCalled(); }); }); diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts index d45582bb..da894f79 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.declarative.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { SettingDefinitionGroup, SettingDefinitionItem, SettingDefinitionPage } from "obsidian"; @@ -158,6 +159,7 @@ function createSettingsTab(options: SettingsTabOptions = {}): ObsidianLiveSyncSe }, services: { setting: { + deviceValues: new MemoryValueStore(), getDeviceAndVaultName: vi.fn(() => ""), saveSettingData: vi.fn(async () => undefined), }, @@ -457,7 +459,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { expect(runtime.pageCleanup).toHaveBeenCalledOnce(); }); - it("rebuilds the catalogue when an externally loaded setting changes page visibility", () => { + it("rebuilds the catalogue when an externally loaded setting changes page visibility", async () => { const tab = createSettingsTab(); const changeLog = findPage(tab, "Change Log"); if (!changeLog.page) { @@ -466,7 +468,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { changeLog.page().display(); tab.core.settings.usePowerUserMode = !tab.editingSettings.usePowerUserMode; - tab.requestReload(); + await tab.requestReload(); expect(tab.update).toHaveBeenCalledOnce(); }); @@ -480,7 +482,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { expect(tab.update).toHaveBeenCalledOnce(); }); - it("rebuilds translated catalogue names when the display language changes externally", () => { + it("rebuilds translated catalogue names when the display language changes externally", async () => { const tab = createSettingsTab(); const changeLog = findPage(tab, "Change Log"); if (!changeLog.page) { @@ -489,12 +491,12 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { changeLog.page().display(); tab.core.settings.displayLanguage = "ja"; - tab.requestReload(); + await tab.requestReload(); expect(tab.update).toHaveBeenCalledOnce(); }); - it("rebuilds the catalogue after accepting an external page-visibility setting over a dirty value", () => { + it("rebuilds the catalogue after accepting an external page-visibility setting over a dirty value", async () => { const tab = createSettingsTab(); const changeLog = findPage(tab, "Change Log"); if (!changeLog.page) { @@ -505,7 +507,7 @@ describe("ObsidianLiveSyncSettingTab native page lifecycle", () => { tab.editingSettings.usePowerUserMode = true; tab.core.settings.usePowerUserMode = true; - tab.requestReload(); + await tab.requestReload(); const configureAnchor = vi.mocked(tab.core.confirm.askInPopup).mock.calls[0]?.[2]; expect(configureAnchor).toBeTypeOf("function"); let acceptExternalSetting: (() => void) | undefined; diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 424213f8..a3c72a31 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -52,7 +52,6 @@ import { type PageFunctions, type UpdateFunction, } from "./SettingPane.ts"; -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { JournalSyncCore } from "@vrtmrz/livesync-commonlib/compat/replication/journal/JournalSyncCore"; import { MinioStorageAdapter } from "@vrtmrz/livesync-commonlib/compat/replication/journal/objectstore/MinioStorageAdapter"; import { closeObsidianSettings } from "@/common/obsidianSettings.ts"; @@ -100,6 +99,9 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { } selectedScreen = ""; + private localConfigurationPassphrase = ""; + private localSettingsReadVersion = 0; + _editingSettings?: AllSettings; // Buffered Settings for editing get editingSettings(): AllSettings { @@ -162,8 +164,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { async saveLocalSetting(key: keyof typeof OnDialogSettingsDefault) { if (key == "deviceAndVaultName") { this.services.setting.setDeviceAndVaultName(this.editingSettings?.[key] ?? ""); - this.services.setting.saveDeviceAndVaultName(); - return await Promise.resolve(); + await this.services.setting.saveDeviceAndVaultName(); } } /** @@ -225,6 +226,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { appliedConfigurationEncryption.configPassphrase ); + if (appliedConfigurationEncryption.configPassphraseStore === "LOCALSTORAGE") { + this.localSettingsReadVersion++; + this.localConfigurationPassphrase = appliedConfigurationEncryption.configPassphrase; + } this.copySettingValue(this.initialSettings, appliedConfigurationEncryption, "configPassphraseStore"); this.copySettingValue(this.initialSettings, appliedConfigurationEncryption, "configPassphrase"); } @@ -285,9 +290,21 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { } } + private async readLocalSettings(): Promise { + const version = ++this.localSettingsReadVersion; + const passphrase = await this.services.setting.deviceValues.getValue("ls-setting-passphrase", ""); + if (version !== this.localSettingsReadVersion) return false; + this.localConfigurationPassphrase = passphrase; + return true; + } + + async loadSettings(): Promise { + if (await this.readLocalSettings()) this.reloadAllSettings(true); + } + reloadAllLocalSettings() { const ret = { ...OnDialogSettingsDefault }; - ret.configPassphrase = compatGlobal.localStorage.getItem("ls-setting-passphrase") || ""; + ret.configPassphrase = this.localConfigurationPassphrase; ret.preset = ""; ret.deviceAndVaultName = this.services.setting.getDeviceAndVaultName(); return ret; @@ -402,7 +419,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { this.plugin = plugin; Setting.env = this; eventHub.onEvent(EVENT_REQUEST_RELOAD_SETTING_TAB, () => { - this.requestReload(); + void this.requestReload().catch((error: unknown) => Logger(error, LOG_LEVEL_NOTICE)); }); this.addOnSaved("displayLanguage", () => this.requestCatalogueRefresh()); this.addOnSaved("showStatusOnEditor", () => eventHub.emitEvent(EVENT_ON_UNRESOLVED_ERROR)); @@ -545,7 +562,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { ); } - requestReload() { + async requestReload(): Promise { + if (!(await this.readLocalSettings())) return; const nativeTabIsShown = this.supportsDeclarativeSettings() && this.containerEl !== undefined && this.containerEl.isShown(); if (this.isShown || nativeTabIsShown) { diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts index 1349edfc..df8f6b23 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.unit.spec.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { REMOTE_RESOURCE_KINDS } from "@vrtmrz/livesync-commonlib/replication"; -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +import { MemoryValueStore } from "@vrtmrz/value-store"; const settingsInitialisationMocks = vi.hoisted(() => ({ applySettingsWithInitialisationChoice: vi.fn(), @@ -76,8 +76,6 @@ beforeEach(() => { settingsInitialisationMocks.applySettingsWithInitialisationChoice.mockReset(); loggerMocks.Logger.mockClear(); vi.mocked($msg).mockClear(); - vi.mocked(compatGlobal.localStorage.getItem).mockReset().mockReturnValue(null); - vi.mocked(compatGlobal.localStorage.setItem).mockReset(); }); describe("ObsidianLiveSyncSettingTab passphrase verification", () => { @@ -280,6 +278,7 @@ describe("ObsidianLiveSyncSettingTab pending-setting initialisation", () => { }, services: { setting: { + deviceValues: new MemoryValueStore(), saveSettingData, getDeviceAndVaultName: vi.fn(() => ""), suspendAllSync, @@ -665,6 +664,7 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { }, services: { setting: { + deviceValues: new MemoryValueStore(), saveSettingData, getDeviceAndVaultName: vi.fn(() => ""), }, @@ -692,6 +692,54 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { return { tab, saveSettingData }; } + it("awaits the stored passphrase before initialising the editing buffer", async () => { + const { tab } = createSettingsTab(); + tab.resetEditingSettings(); + let resolveRead!: (value: string) => void; + vi.spyOn(tab.services.setting.deviceValues, "getValue").mockReturnValueOnce( + new Promise((resolve) => { + resolveRead = resolve; + }) + ); + const loading = tab.loadSettings(); + expect(tab.initialSettings).toBeUndefined(); + resolveRead("stored passphrase"); + await loading; + expect(tab.editingSettings.configPassphrase).toBe("stored passphrase"); + expect(tab.initialSettings!.configPassphrase).toBe("stored passphrase"); + }); + + it("preserves the loaded settings when device storage cannot be read", async () => { + const { tab } = createSettingsTab(); + await tab.services.setting.deviceValues.setValue("ls-setting-passphrase", "stored passphrase"); + await tab.loadSettings(); + const failure = new Error("device storage unavailable"); + vi.spyOn(tab.services.setting.deviceValues, "getValue").mockRejectedValueOnce(failure); + await expect(tab.requestReload()).rejects.toBe(failure); + expect(tab.editingSettings.configPassphrase).toBe("stored passphrase"); + expect(tab.initialSettings!.configPassphrase).toBe("stored passphrase"); + }); + + it("does not let a delayed read overwrite a successfully saved passphrase", async () => { + const { tab } = createSettingsTab(); + await tab.loadSettings(); + let resolveRead!: (value: string) => void; + vi.spyOn(tab.services.setting.deviceValues, "getValue").mockReturnValueOnce( + new Promise((resolve) => { + resolveRead = resolve; + }) + ); + const loading = tab.loadSettings(); + Object.assign(tab.services.setting, { changeConfigurationEncryption: vi.fn(async () => undefined) }); + tab.editingSettings.configPassphraseStore = "LOCALSTORAGE"; + tab.editingSettings.configPassphrase = "new passphrase"; + await tab.saveSettings(["configPassphraseStore", "configPassphrase"]); + resolveRead("old passphrase"); + await loading; + tab.reloadAllSettings(); + expect(tab.initialSettings!.configPassphrase).toBe("new passphrase"); + }); + it("loads the imperative fallback without a SettingPage runtime export", () => { const { tab } = createSettingsTab(); @@ -779,7 +827,7 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { it("saves a custom configuration passphrase edit through the setting service", async () => { const { tab } = createSettingsTab(); const changeConfigurationEncryption = vi.fn(async () => { - vi.mocked(compatGlobal.localStorage.getItem).mockReturnValue("new custom passphrase"); + await tab.services.setting.deviceValues.setValue("ls-setting-passphrase", "new custom passphrase"); }); Object.assign(tab.core.services.setting, { changeConfigurationEncryption }); const store = "LOCALSTORAGE"; @@ -787,13 +835,13 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { tab.initialSettings!.configPassphraseStore = store; tab.editingSettings.configPassphrase = "new custom passphrase"; tab.initialSettings!.configPassphrase = "old custom passphrase"; - vi.mocked(compatGlobal.localStorage.setItem).mockClear(); + const setValue = vi.spyOn(tab.services.setting.deviceValues, "setValue"); await tab.saveSettings(["configPassphrase"]); expect(changeConfigurationEncryption).toHaveBeenCalledOnce(); expect(changeConfigurationEncryption).toHaveBeenCalledWith(store, "new custom passphrase"); - expect(compatGlobal.localStorage.setItem).not.toHaveBeenCalled(); + expect(setValue).toHaveBeenCalledOnce(); expect(tab.initialSettings!.configPassphrase).toBe("new custom passphrase"); }); @@ -811,7 +859,6 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { tab.initialSettings!.configPassphraseStore = "LOCALSTORAGE"; tab.initialSettings!.configPassphrase = "previous passphrase"; tab.editingSettings.configPassphrase = "passphrase A"; - vi.mocked(compatGlobal.localStorage.getItem).mockReturnValue("passphrase A"); const apply = tab.saveSettings(["configPassphrase"]); expect(changeConfigurationEncryption).toHaveBeenCalledWith("LOCALSTORAGE", "passphrase A"); @@ -840,13 +887,13 @@ describe("ObsidianLiveSyncSettingTab declarative settings boundary", () => { tab.editingSettings.configPassphraseStore = editedStore; tab.initialSettings!.configPassphrase = "old custom passphrase"; tab.editingSettings.configPassphrase = "new custom passphrase"; - vi.mocked(compatGlobal.localStorage.setItem).mockClear(); + const setValue = vi.spyOn(tab.services.setting.deviceValues, "setValue"); await expect(tab.saveSettings(["configPassphraseStore", "configPassphrase"])).rejects.toBe(failure); expect(changeConfigurationEncryption).toHaveBeenCalledOnce(); expect(changeConfigurationEncryption).toHaveBeenCalledWith(editedStore, "new custom passphrase"); - expect(compatGlobal.localStorage.setItem).not.toHaveBeenCalled(); + expect(setValue).not.toHaveBeenCalled(); expect(tab.core.settings.configPassphraseStore).toBe(savedStore); expect(tab.initialSettings!.configPassphraseStore).toBe(savedStore); expect(tab.initialSettings!.configPassphrase).toBe("old custom passphrase"); diff --git a/src/modules/features/SetupManager.unit.spec.ts b/src/modules/features/SetupManager.unit.spec.ts index c8dfe4ec..68320e2c 100644 --- a/src/modules/features/SetupManager.unit.spec.ts +++ b/src/modules/features/SetupManager.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_SETTINGS, @@ -37,11 +38,6 @@ import { decodeSettingsFromQRCodeData } from "@vrtmrz/livesync-commonlib/compat/ import { SetupManager, UserMode } from "./SetupManager"; class TestSettingService extends SettingService { - protected setItem(_key: string, _value: string): void {} - protected getItem(_key: string): string { - return ""; - } - protected deleteItem(_key: string): void {} protected saveData(_setting: ObsidianLiveSyncSettings): Promise { return Promise.resolve(); } @@ -65,6 +61,7 @@ function createLegacyRemoteSetting(): ObsidianLiveSyncSettings { function createSetupManager() { const setting = new TestSettingService(new ServiceContext(), { + deviceValues: new MemoryValueStore(), APIService: { getSystemVaultName: vi.fn(() => "vault"), getAppID: vi.fn(() => "app"), diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte index 2f89b845..210633b5 100644 --- a/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte +++ b/src/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte @@ -52,26 +52,48 @@ const context = getDialogContext(); let error = $state(""); let connectionPathResetNotice = $state(false); + let peerNameReady = $state(false); + let mounted = false; const hasValidTurnServer = $derived(hasP2PTurnConfiguration(syncSetting)); type Props = GuestDialogProps; const { setResult, getInitialData }: Props = $props(); let connectionProbe: SetupRemoteP2PInitialData["connectionProbe"] | undefined; onMount(() => { + mounted = true; const initialData = getInitialData?.(); connectionProbe = initialData?.connectionProbe; const initialSettings = initialData?.settings; if (initialSettings) { syncSetting = pickP2PSyncSettings(initialSettings); } - const initialPeerName = (initialSettings?.P2P_DevicePeerName ?? "").trim(); + const savedPeerName = initialSettings?.P2P_DevicePeerName; + const initialPeerName = typeof savedPeerName === "string" ? savedPeerName.trim() : ""; if (initialPeerName !== "") { - return; - } - const cachedPeerName = context.services.config.getSmallConfig(SETTING_KEY_P2P_DEVICE_NAME); - if (cachedPeerName) { - syncSetting.P2P_DevicePeerName = cachedPeerName as string; + peerNameReady = true; + return () => { + mounted = false; + }; } + void context.services.config.getValue(SETTING_KEY_P2P_DEVICE_NAME).then( + (cachedPeerName) => { + if (mounted) { + if (typeof cachedPeerName === "string" && cachedPeerName) { + syncSetting.P2P_DevicePeerName = cachedPeerName; + } + peerNameReady = true; + } + }, + (loadError: unknown) => { + if (mounted) { + error = loadError instanceof Error ? loadError.message : String(loadError); + peerNameReady = true; + } + } + ); + return () => { + mounted = false; + }; }); function generateSetting() { const connSetting: P2PSyncSetting = { @@ -193,6 +215,7 @@ } async function checkAndCommit() { + if (!peerNameReady) return; error = ""; try { error = (await checkConnection()) || ""; @@ -207,6 +230,7 @@ } } function commit() { + if (!peerNameReady) return; error = validateManagedTurnSettings(syncSetting) ?? ""; if (error) return; const setting = pickP2PSyncSettings(generateSetting()); @@ -217,6 +241,7 @@ } const canProceed = $derived.by(() => { return ( + peerNameReady && syncSetting.P2P_relays.trim() !== "" && syncSetting.P2P_roomID.trim() !== "" && syncSetting.P2P_passphrase.trim() !== "" && @@ -291,6 +316,7 @@ autocorrect="off" autocapitalize="off" spellcheck="false" + disabled={!peerNameReady} bind:value={syncSetting.P2P_DevicePeerName} /> @@ -385,7 +411,11 @@ disabled={!canProceed} commit={() => checkAndCommit()} /> - commit()} /> + commit()} + /> cancel()} /> {/if} diff --git a/src/modules/services/ObsidianServiceHub.ts b/src/modules/services/ObsidianServiceHub.ts index 3613ea27..b361986f 100644 --- a/src/modules/services/ObsidianServiceHub.ts +++ b/src/modules/services/ObsidianServiceHub.ts @@ -1,4 +1,6 @@ import { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; +import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage"; import { ObsidianServiceContext } from "@/modules/services/ObsidianServiceContext"; import type { ServiceInstances } from "@vrtmrz/livesync-commonlib/compat/services/ServiceHub"; import type ObsidianLiveSyncPlugin from "@/main"; @@ -11,7 +13,6 @@ import { ObsidianTweakValueService, ObsidianTestService, ObsidianDatabaseEventService, - ObsidianConfigService, ObsidianKeyValueDBService, ObsidianControlService, } from "./ObsidianServices"; @@ -46,6 +47,7 @@ export class ObsidianServiceHub extends InjectableServiceHub( - task: () => T | PromiseLike, - options?: ActivityOptions - ): Promise { + async runBoundedLocalApplicationActivity(task: () => T | PromiseLike, options?: ActivityOptions): Promise { this.boundedLocalApplicationActivityCount.value++; try { return this.dependencies.activityRunner @@ -84,7 +80,6 @@ export class ObsidianConflictService extends InjectableConflictService {} // InjectableTestService export class ObsidianTestService extends InjectableTestService {} -export class ObsidianConfigService extends ConfigServiceBrowserCompat {} export class ObsidianKeyValueDBService extends KeyValueDBService {} diff --git a/src/modules/services/ObsidianSettingService.ts b/src/modules/services/ObsidianSettingService.ts index a934ebd1..87b93b59 100644 --- a/src/modules/services/ObsidianSettingService.ts +++ b/src/modules/services/ObsidianSettingService.ts @@ -1,4 +1,3 @@ -import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { EVENT_REQUEST_RELOAD_SETTING_TAB, @@ -27,19 +26,6 @@ export class ObsidianSettingService extends Se return Promise.resolve(true); }); } - protected setItem(key: string, value: string) { - // TODO: Implement nativeLocalStorage. - return compatGlobal.localStorage.setItem(key, value); - } - protected getItem(key: string): string { - // TODO: Implement nativeLocalStorage. - return compatGlobal.localStorage.getItem(key) ?? ""; - } - protected deleteItem(key: string): void { - // TODO: Implement nativeLocalStorage. - compatGlobal.localStorage.removeItem(key); - } - protected override async saveData(data: ObsidianLiveSyncSettings): Promise { return await this.context.liveSyncPlugin.saveData(data); } diff --git a/src/modules/services/ObsidianSettingService.unit.spec.ts b/src/modules/services/ObsidianSettingService.unit.spec.ts index 9175c931..6df8404d 100644 --- a/src/modules/services/ObsidianSettingService.unit.spec.ts +++ b/src/modules/services/ObsidianSettingService.unit.spec.ts @@ -1,6 +1,23 @@ import { describe, expect, it } from "vitest"; +import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { IAPIService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import { MemoryValueStore } from "@vrtmrz/value-store"; +import type { ObsidianServiceContext } from "@/modules/services/ObsidianServiceContext"; import { normaliseObsidianSettingsData } from "./ObsidianSettingService.ts"; +import { ObsidianSettingService } from "./ObsidianSettingService.ts"; + +function createSettingService( + vaultName: string, + deviceValues: MemoryValueStore +): ObsidianSettingService { + const context = createServiceContext() as unknown as ObsidianServiceContext; + const APIService = { + addLog: () => undefined, + getSystemVaultName: () => vaultName, + } as unknown as IAPIService; + return new ObsidianSettingService(context, { APIService, deviceValues }); +} describe("normaliseObsidianSettingsData", () => { it("maps Obsidian's missing data value to Commonlib's new-Vault input", () => { @@ -12,4 +29,26 @@ describe("normaliseObsidianSettingsData", () => { expect(normaliseObsidianSettingsData(settings)).toBe(settings); }); + + it("keeps injected device values independent from each Vault-scoped configuration store", async () => { + const firstValues = new MemoryValueStore(); + const secondValues = new MemoryValueStore(); + const firstSetting = createSettingService("first-vault", firstValues); + const secondSetting = createSettingService("second-vault", secondValues); + + await firstSetting.deviceValues.setValue("shared-key", "device value"); + await firstSetting.config.setValue("shared-key", "first config"); + await secondSetting.config.setValue("shared-key", "second config"); + + expect(firstSetting.deviceValues).toBe(firstValues); + expect(secondSetting.deviceValues).toBe(secondValues); + expect(firstSetting.deviceValues).not.toBe(secondSetting.deviceValues); + expect(firstSetting.config).not.toBe(firstValues); + expect(firstSetting.config).not.toBe(secondSetting.config); + await expect(firstSetting.deviceValues.getValue("shared-key")).resolves.toBe("device value"); + await expect(firstSetting.config.getValue("shared-key")).resolves.toBe("first config"); + await expect(secondSetting.config.getValue("shared-key")).resolves.toBe("second config"); + await expect(firstValues.getValue("first-vault-shared-key")).resolves.toBe("first config"); + await expect(secondValues.getValue("second-vault-shared-key")).resolves.toBe("second config"); + }); }); diff --git a/src/modules/services/ObsidianUIService.ts b/src/modules/services/ObsidianUIService.ts index c5abf23f..08e6e572 100644 --- a/src/modules/services/ObsidianUIService.ts +++ b/src/modules/services/ObsidianUIService.ts @@ -1,4 +1,4 @@ -import type { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService"; +import type { IConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import type { AppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService"; import type { ReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/base/ReplicatorService"; import { UIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/base/UIService"; @@ -8,7 +8,7 @@ import DialogToCopy from "@/modules/services/LiveSyncUI/dialogues/DialogueToCopy import type { IAPIService, IControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; export type ObsidianUIServiceDependencies = { appLifecycle: AppLifecycleService; - config: ConfigService; + config: IConfigService; replicator: ReplicatorService; APIService: IAPIService; control: IControlService; diff --git a/src/serviceFeatures/compatibilityReview.ts b/src/serviceFeatures/compatibilityReview.ts index b92aeec6..7c79f98f 100644 --- a/src/serviceFeatures/compatibilityReview.ts +++ b/src/serviceFeatures/compatibilityReview.ts @@ -44,17 +44,17 @@ export class CompatibilityReviewController { return this._initialised; } - private readAcknowledgedVersion(): string | null { + private async readAcknowledgedVersion(): Promise { const setting = this.core.services.setting; - const currentMarker = setting.getSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY); + const currentMarker = await this.core.services.config.getValue(DATABASE_COMPATIBILITY_VERSION_KEY, ""); if (currentMarker) return currentMarker; const legacyKey = legacyDatabaseCompatibilityVersionKey(this.core.services.vault.getVaultName()); - const legacyMarker = setting.getDeviceLocalConfig(legacyKey); + const legacyMarker = await setting.deviceValues.getValue(legacyKey, ""); if (!legacyMarker) return null; - setting.setSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY, legacyMarker); - setting.deleteDeviceLocalConfig(legacyKey); + await this.core.services.config.setValue(DATABASE_COMPATIBILITY_VERSION_KEY, legacyMarker); + await setting.deviceValues.delete(legacyKey); return legacyMarker; } @@ -75,7 +75,7 @@ export class CompatibilityReviewController { return true; } - const acknowledgedVersion = this.readAcknowledgedVersion(); + const acknowledgedVersion = await this.readAcknowledgedVersion(); const evaluation = evaluateCompatibilityPause({ acknowledgedVersion, currentVersion: this.currentVersion, @@ -85,7 +85,7 @@ export class CompatibilityReviewController { this.pause = evaluation.pause; if (evaluation.initialiseAcknowledgedVersion) { - setting.setSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); + await this.core.services.config.setValue(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); this._initialised = true; return true; } @@ -115,9 +115,9 @@ export class CompatibilityReviewController { throw error; } - setting.setSmallConfig(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); + await this.core.services.config.setValue(DATABASE_COMPATIBILITY_VERSION_KEY, `${this.currentVersion}`); const legacyKey = legacyDatabaseCompatibilityVersionKey(this.core.services.vault.getVaultName()); - setting.deleteDeviceLocalConfig(legacyKey); + await setting.deviceValues.delete(legacyKey); this.pause = undefined; this.ui.clearReminder(); await this.core.services.control.applySettings(); diff --git a/src/serviceFeatures/compatibilityReview.unit.spec.ts b/src/serviceFeatures/compatibilityReview.unit.spec.ts index 760718cc..44a1b05b 100644 --- a/src/serviceFeatures/compatibilityReview.unit.spec.ts +++ b/src/serviceFeatures/compatibilityReview.unit.spec.ts @@ -33,13 +33,32 @@ function createFixture( } = {} ) { const local = new Map(); + const deviceValues = new Map(); if (options.marker !== undefined && options.marker !== null) { local.set(DATABASE_COMPATIBILITY_VERSION_KEY, options.marker); } const legacyKey = legacyDatabaseCompatibilityVersionKey("Test Vault"); if (options.legacyMarker !== undefined && options.legacyMarker !== null) { - local.set(legacyKey, options.legacyMarker); + deviceValues.set(legacyKey, options.legacyMarker); } + const config = { + getValue: vi.fn(async (key: string, defaultValue = "") => local.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + local.set(key, value); + }), + delete: vi.fn(async (key: string) => { + local.delete(key); + }), + }; + const deviceValueStore = { + getValue: vi.fn(async (key: string, defaultValue = "") => deviceValues.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + deviceValues.set(key, value); + }), + delete: vi.fn(async (key: string) => { + deviceValues.delete(key); + }), + }; const settings = { versionUpFlash: options.versionUpFlash ?? "", isConfigured: options.isConfigured ?? true, @@ -49,15 +68,13 @@ function createFixture( const setting = { currentSettings: () => settings, getSettingsMigrationState: () => migrationState(options.migration), - getSmallConfig: (key: string) => local.get(key) ?? "", - setSmallConfig: (key: string, value: string) => local.set(key, value), - getDeviceLocalConfig: (key: string) => local.get(key) ?? null, - deleteDeviceLocalConfig: (key: string) => local.delete(key), + deviceValues: deviceValueStore, saveSettingData, }; const core = { services: { setting, + config, vault: { getVaultName: () => "Test Vault" }, control: { applySettings }, }, @@ -69,7 +86,18 @@ function createFixture( clearReminder: vi.fn(), }; const controller = new CompatibilityReviewController(core, ui, 12); - return { controller, ui, local, legacyKey, settings, saveSettingData, applySettings }; + return { + controller, + ui, + local, + deviceValues, + config, + deviceValueStore, + legacyKey, + settings, + saveSettingData, + applySettings, + }; } describe("compatibility review controller", () => { @@ -196,10 +224,96 @@ describe("compatibility review controller", () => { await fixture.controller.initialise(); expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); - expect(fixture.local.has(fixture.legacyKey)).toBe(false); + expect(fixture.deviceValues.has(fixture.legacyKey)).toBe(false); expect(fixture.controller.pendingPause).toBeDefined(); }); + it("saves the scoped marker before deleting the legacy device value", async () => { + const fixture = createFixture({ legacyMarker: "11" }); + let finishSave!: () => void; + const saving = new Promise((resolve) => { + finishSave = resolve; + }); + fixture.config.setValue.mockImplementation(async (key, value) => { + await saving; + fixture.local.set(key, value); + }); + + const initialising = fixture.controller.initialise(); + await Promise.resolve(); + await Promise.resolve(); + + expect(fixture.deviceValues.get(fixture.legacyKey)).toBe("11"); + expect(fixture.deviceValueStore.delete).not.toHaveBeenCalled(); + expect(fixture.controller.initialised).toBe(false); + + finishSave(); + await initialising; + + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); + expect(fixture.deviceValues.has(fixture.legacyKey)).toBe(false); + expect(fixture.controller.initialised).toBe(true); + }); + + it("does not treat a failed marker read as an absent value", async () => { + const fixture = createFixture({ legacyMarker: "11" }); + fixture.config.getValue.mockRejectedValueOnce(new Error("marker read failed")); + + await expect(fixture.controller.initialise()).rejects.toThrow("marker read failed"); + + expect(fixture.deviceValueStore.getValue).not.toHaveBeenCalled(); + expect(fixture.deviceValues.get(fixture.legacyKey)).toBe("11"); + expect(fixture.controller.initialised).toBe(false); + }); + + it("keeps the compatibility pause blocked until its async marker write succeeds", async () => { + const fixture = createFixture({ marker: "11" }); + vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); + await fixture.controller.initialise(); + let finishSave!: () => void; + const saving = new Promise((resolve) => { + finishSave = resolve; + }); + fixture.config.setValue.mockImplementation(async (key, value) => { + await saving; + fixture.local.set(key, value); + }); + + const reviewing = fixture.controller.openReview(); + await Promise.resolve(); + await Promise.resolve(); + + expect(fixture.controller.pendingPause).toBeDefined(); + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); + expect(fixture.applySettings).not.toHaveBeenCalled(); + + finishSave(); + await reviewing; + + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("12"); + expect(fixture.controller.pendingPause).toBeUndefined(); + expect(fixture.applySettings).toHaveBeenCalledOnce(); + }); + + it("keeps a storage-failed acknowledgement blocked and retryable", async () => { + const fixture = createFixture({ marker: "11" }); + vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); + await fixture.controller.initialise(); + fixture.config.setValue.mockRejectedValueOnce(new Error("marker write failed")); + + await expect(fixture.controller.openReview()).rejects.toThrow("marker write failed"); + + expect(fixture.controller.pendingPause).toBeDefined(); + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("11"); + expect(fixture.applySettings).not.toHaveBeenCalled(); + + await fixture.controller.openReview(); + + expect(fixture.controller.pendingPause).toBeUndefined(); + expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("12"); + expect(fixture.applySettings).toHaveBeenCalledOnce(); + }); + it("restores the runtime gate if persisting an acknowledgement fails", async () => { const fixture = createFixture({ marker: "11" }); vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); diff --git a/src/serviceFeatures/redFlag.simpleFetch.ts b/src/serviceFeatures/redFlag.simpleFetch.ts index 4ce84e29..950e1618 100644 --- a/src/serviceFeatures/redFlag.simpleFetch.ts +++ b/src/serviceFeatures/redFlag.simpleFetch.ts @@ -63,28 +63,27 @@ function buildSimpleFetchResult(stage1: string, stage2?: string) { return undefined; } -function rememberSimpleFetchMode(host: NecessaryServices<"setting", never>, stage1: string, stage2?: string) { - host.services.setting.setSmallConfig(SIMPLE_FETCH_MODE_KEY, JSON.stringify({ stage1, stage2 })); +async function rememberSimpleFetchMode(host: NecessaryServices<"setting", never>, stage1: string, stage2?: string) { + await host.services.setting.config.setValue(SIMPLE_FETCH_MODE_KEY, JSON.stringify({ stage1, stage2 })); } -function getRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { - const saved = host.services.setting.getSmallConfig(SIMPLE_FETCH_MODE_KEY); +async function getRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { + const saved = await host.services.setting.config.getValue(SIMPLE_FETCH_MODE_KEY, ""); if (!saved) return undefined; + let remembered: ReturnType = undefined; try { const { stage1, stage2 } = JSON.parse(saved) as { stage1?: string; stage2?: string }; - if (stage1) { - const remembered = buildSimpleFetchResult(stage1, stage2); - if (remembered) return remembered; - } + if (stage1) remembered = buildSimpleFetchResult(stage1, stage2); } catch { // Clear below; the saved choice is optional and can be rebuilt by asking again. } - host.services.setting.deleteSmallConfig(SIMPLE_FETCH_MODE_KEY); + if (remembered) return remembered; + await host.services.setting.config.delete(SIMPLE_FETCH_MODE_KEY); return undefined; } -function clearRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { - host.services.setting.deleteSmallConfig(SIMPLE_FETCH_MODE_KEY); +async function clearRememberedSimpleFetchMode(host: NecessaryServices<"setting", never>) { + await host.services.setting.config.delete(SIMPLE_FETCH_MODE_KEY); } async function confirmTranslatedChoice( @@ -104,7 +103,7 @@ async function confirmTranslatedChoice( export async function askSimpleFetchMode( host: NecessaryServices<"UI" | "setting", never> ): Promise<{ mode: string; options: Partial } | "cancelled" | "aborted"> { - const remembered = getRememberedSimpleFetchMode(host); + const remembered = await getRememberedSimpleFetchMode(host); if (remembered) return remembered; const msg = $msg( @@ -169,7 +168,7 @@ Firstly, how shall we handle the data retrieved from this remote source? if (stage2 === STAGE2_ABORT) { return "aborted"; } - rememberSimpleFetchMode(host, stage1, stage2); + await rememberSimpleFetchMode(host, stage1, stage2); return buildSimpleFetchResult(stage1, stage2)!; } @@ -198,7 +197,7 @@ Firstly, how shall we handle the data retrieved from this remote source? if (stage2 === STAGE2_ABORT) { return "aborted"; } - rememberSimpleFetchMode(host, stage1, stage2); + await rememberSimpleFetchMode(host, stage1, stage2); return buildSimpleFetchResult(stage1, stage2)!; } @@ -234,18 +233,18 @@ export async function askAndPerformFastSetupOnScheduledFetchAll( "Remediation mode is active, so the detailed fetch flow is used instead of Simple Fetch.", LOG_LEVEL_NOTICE ); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); return undefined; } const result = await askSimpleFetchMode(host); if (result === "cancelled") { log("Fetch cancelled by user.", LOG_LEVEL_NOTICE); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); return await cancelScheduledInitialisation(host, cleanupFlag); } if (result === "aborted") { log("Fetch exited by user.", LOG_LEVEL_NOTICE); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); host.services.appLifecycle.performRestart(); return false; } @@ -256,7 +255,7 @@ export async function askAndPerformFastSetupOnScheduledFetchAll( const settings = host.services.setting.currentSettings(); if (!(await adjustSettingToRemoteIfNeeded(host, log, { preventFetchingConfig: false }, settings))) { log("Fetch initialisation cancelled by user.", LOG_LEVEL_NOTICE); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); return await cancelScheduledInitialisation(host, cleanupFlag); } @@ -295,7 +294,7 @@ export async function askAndPerformFastSetupOnScheduledFetchAll( } await host.serviceModules.rebuilder.finishRebuild(); await cleanupFlag(); - clearRememberedSimpleFetchMode(host); + await clearRememberedSimpleFetchMode(host); log("Simple fetch and scan operation completed.", LOG_LEVEL_NOTICE); return true; }; diff --git a/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts b/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts index 73b3d9c0..bc583d8f 100644 --- a/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts +++ b/src/serviceFeatures/redFlag.simpleFetch.unit.spec.ts @@ -10,19 +10,26 @@ const KEEP_LOCAL = "Keep local files even if not on remote"; function createHost(savedChoice?: string) { const smallConfig = new Map(); if (savedChoice) smallConfig.set("simple-fetch-mode", savedChoice); + const config = { + getValue: vi.fn(async (key: string, defaultValue = "") => smallConfig.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + smallConfig.set(key, value); + }), + delete: vi.fn(async (key: string) => { + smallConfig.delete(key); + }), + }; const confirmWithMessage = vi.fn<(title: string, message: string, buttons: string[], defaultAction: string) => Promise>(); const host = { services: { UI: { confirm: { confirmWithMessage } }, setting: { - getSmallConfig: (key: string) => smallConfig.get(key) ?? "", - setSmallConfig: (key: string, value: string) => smallConfig.set(key, value), - deleteSmallConfig: (key: string) => smallConfig.delete(key), + config, }, }, } as unknown as NecessaryServices<"UI" | "setting", never>; - return { host, confirmWithMessage, smallConfig }; + return { host, confirmWithMessage, smallConfig, config }; } afterEach(() => setLang("def")); @@ -50,6 +57,75 @@ describe("Fast Setup translated choices", () => { }); }); + it("waits for the saved choice before returning it to the fetch workflow", async () => { + const { host, confirmWithMessage, smallConfig, config } = createHost(); + let markSaveStarted!: () => void; + const saveStarted = new Promise((resolve) => { + markSaveStarted = resolve; + }); + let finishSave!: () => void; + const save = new Promise((resolve) => { + finishSave = resolve; + }); + config.setValue.mockImplementation(async (key, value) => { + markSaveStarted(); + await save; + smallConfig.set(key, value); + }); + confirmWithMessage.mockImplementation(async (_title, _message, buttons) => buttons[1]); + + const asking = askSimpleFetchMode(host); + await saveStarted; + + expect(smallConfig.has("simple-fetch-mode")).toBe(false); + expect(confirmWithMessage).toHaveBeenCalledTimes(2); + + finishSave(); + await expect(asking).resolves.toMatchObject({ mode: "remote-only" }); + expect(JSON.parse(smallConfig.get("simple-fetch-mode")!)).toEqual({ + stage1: REMOTE_WINS, + stage2: KEEP_LOCAL, + }); + }); + + it("propagates a stored-choice read failure instead of treating it as missing", async () => { + const { host, confirmWithMessage, config } = createHost(); + config.getValue.mockRejectedValueOnce(new Error("choice read failed")); + + await expect(askSimpleFetchMode(host)).rejects.toThrow("choice read failed"); + + expect(confirmWithMessage).not.toHaveBeenCalled(); + }); + + it("waits for an invalid optional choice to be deleted before asking again", async () => { + const { host, confirmWithMessage, smallConfig, config } = createHost("not-json"); + let markDeleteStarted!: () => void; + const deleteStarted = new Promise((resolve) => { + markDeleteStarted = resolve; + }); + let finishDelete!: () => void; + const deletion = new Promise((resolve) => { + finishDelete = resolve; + }); + config.delete.mockImplementation(async (key) => { + markDeleteStarted(); + await deletion; + smallConfig.delete(key); + }); + confirmWithMessage.mockImplementation(async (_title, _message, buttons) => buttons[3]); + + const asking = askSimpleFetchMode(host); + await deleteStarted; + + expect(confirmWithMessage).not.toHaveBeenCalled(); + expect(smallConfig.has("simple-fetch-mode")).toBe(true); + + finishDelete(); + await expect(asking).resolves.toBe("cancelled"); + expect(smallConfig.has("simple-fetch-mode")).toBe(false); + expect(confirmWithMessage).toHaveBeenCalledOnce(); + }); + it("restores choices saved by the previous English implementation while displaying Spanish", async () => { const savedChoice = JSON.stringify({ stage1: REMOTE_WINS, stage2: KEEP_LOCAL }); const { host, confirmWithMessage, smallConfig } = createHost(savedChoice); diff --git a/src/serviceFeatures/redFlag.unit.spec.ts b/src/serviceFeatures/redFlag.unit.spec.ts index 9c1a2bf8..b21cb99a 100644 --- a/src/serviceFeatures/redFlag.unit.spec.ts +++ b/src/serviceFeatures/redFlag.unit.spec.ts @@ -107,13 +107,15 @@ const createSettingServiceMock = () => { }), suspendAllSync: vi.fn(() => Promise.resolve()), suspendExtraSync: vi.fn(() => Promise.resolve()), - getSmallConfig: vi.fn((key: string) => smallConfig.get(key) ?? ""), - setSmallConfig: vi.fn((key: string, value: string) => { - smallConfig.set(key, value); - }), - deleteSmallConfig: vi.fn((key: string) => { - smallConfig.delete(key); - }), + config: { + getValue: vi.fn(async (key: string, defaultValue = "") => smallConfig.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + smallConfig.set(key, value); + }), + delete: vi.fn(async (key: string) => { + smallConfig.delete(key); + }), + }, }; }; @@ -785,7 +787,7 @@ describe("Red Flag Feature", () => { ); expect(host.mocks.ui.confirm.confirmWithMessage).not.toHaveBeenCalled(); - expect(host.mocks.setting.deleteSmallConfig).toHaveBeenCalledWith("simple-fetch-mode"); + expect(host.mocks.setting.config.delete).toHaveBeenCalledWith("simple-fetch-mode"); expect(cleanupFlag).not.toHaveBeenCalled(); }); @@ -830,7 +832,7 @@ describe("Red Flag Feature", () => { }); expect(host.mocks.rebuilder.finishRebuild).not.toHaveBeenCalled(); expect(cleanupFlag).not.toHaveBeenCalled(); - expect(host.mocks.setting.deleteSmallConfig).not.toHaveBeenCalledWith("simple-fetch-mode"); + expect(host.mocks.setting.config.delete).not.toHaveBeenCalledWith("simple-fetch-mode"); }); it("re-suspends both reflection directions when finalisation fails after releasing them", async () => { @@ -892,7 +894,7 @@ describe("Red Flag Feature", () => { await askAndPerformFastSetupOnScheduledFetchAll(host as any, log, cleanupFlag); - expect(host.mocks.setting.deleteSmallConfig).toHaveBeenCalledWith("simple-fetch-mode"); + expect(host.mocks.setting.config.delete).toHaveBeenCalledWith("simple-fetch-mode"); }); it("should return false and cleanup when quick flow is cancelled", async () => { diff --git a/src/serviceFeatures/useReviewHarness.ts b/src/serviceFeatures/useReviewHarness.ts index 99e2cd5a..64efb03e 100644 --- a/src/serviceFeatures/useReviewHarness.ts +++ b/src/serviceFeatures/useReviewHarness.ts @@ -69,9 +69,9 @@ export function useReviewHarness( await createFileContentHashBenchmarkOperations(activeWindow.performance), activeWindow.performance ), - readContinuation: () => services.setting.getSmallConfig(REVIEW_HARNESS_STATE_KEY), - writeContinuation: (value) => services.setting.setSmallConfig(REVIEW_HARNESS_STATE_KEY, value), - deleteContinuation: () => services.setting.deleteSmallConfig(REVIEW_HARNESS_STATE_KEY), + readContinuation: () => services.config.getValue(REVIEW_HARNESS_STATE_KEY, ""), + writeContinuation: (value) => services.config.setValue(REVIEW_HARNESS_STATE_KEY, value), + deleteContinuation: () => services.config.delete(REVIEW_HARNESS_STATE_KEY), restart: () => services.appLifecycle.performRestart(), reportError: (error) => services.API.addLog(error, LOG_LEVEL_NOTICE), copyText: async (value) => { @@ -94,14 +94,14 @@ export function useReviewHarness( let registered = false; let openAfterLayout = false; - services.appLifecycle.onSettingLoaded.addHandler(() => { + services.appLifecycle.onSettingLoaded.addHandler(async () => { if (!continuationConsumed) { + await controller.consumeContinuation(); continuationConsumed = true; - controller.consumeContinuation(); } const snapshot = controller.snapshot(); openAfterLayout = snapshot.resumedRequestId !== null || snapshot.continuationError !== null; - if (!services.setting.currentSettings().enableDebugTools || registered) return Promise.resolve(true); + if (!services.setting.currentSettings().enableDebugTools || registered) return true; registered = true; services.API.registerWindow( @@ -115,7 +115,7 @@ export function useReviewHarness( void services.API.showWindow(VIEW_TYPE_REVIEW_HARNESS); }, }); - return Promise.resolve(true); + return true; }); services.appLifecycle.onLayoutReady.addHandler(() => { diff --git a/src/serviceFeatures/useReviewHarness.unit.spec.ts b/src/serviceFeatures/useReviewHarness.unit.spec.ts index 35281081..c73070a7 100644 --- a/src/serviceFeatures/useReviewHarness.unit.spec.ts +++ b/src/serviceFeatures/useReviewHarness.unit.spec.ts @@ -31,6 +31,15 @@ function createFixture(options: { enableDebugTools?: boolean; continuation?: str const layoutReadyHandlers: Array<() => Promise> = []; const local = new Map(); if (options.continuation) local.set(REVIEW_HARNESS_STATE_KEY, options.continuation); + const config = { + getValue: vi.fn(async (key: string, defaultValue = "") => local.get(key) ?? defaultValue), + setValue: vi.fn(async (key: string, value: string) => { + local.set(key, value); + }), + delete: vi.fn(async (key: string) => { + local.delete(key); + }), + }; const settings = { enableDebugTools: options.enableDebugTools ?? true, liveSync: false, @@ -63,10 +72,8 @@ function createFixture(options: { enableDebugTools?: boolean; continuation?: str requiresSyncReview: true, reviewReasons: [], }), - getSmallConfig: (key: string) => local.get(key) ?? "", - setSmallConfig: (key: string, value: string) => local.set(key, value), - deleteSmallConfig: (key: string) => local.delete(key), }, + config, appLifecycle: { onSettingLoaded: { addHandler: vi.fn((handler: () => Promise) => settingLoadedHandlers.push(handler)), @@ -106,6 +113,7 @@ function createFixture(options: { enableDebugTools?: boolean; continuation?: str api, settings, local, + config, settingLoadedHandlers, layoutReadyHandlers, compatibilityReview, @@ -144,6 +152,74 @@ describe("Review Harness composition", () => { expect(fixture.api.showWindow).toHaveBeenCalledWith(VIEW_TYPE_REVIEW_HARNESS); }); + it("holds the setting-loaded lifecycle until continuation deletion has completed", async () => { + const requestedAt = "2026-07-18T11:59:00.000Z"; + const continuation = JSON.stringify({ + formatVersion: 1, + requestId: `compatibility-review-${requestedAt}`, + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt, + }); + const fixture = createFixture({ continuation }); + let markDeleteStarted!: () => void; + const deleteStarted = new Promise((resolve) => { + markDeleteStarted = resolve; + }); + let finishDelete!: () => void; + const deleting = new Promise((resolve) => { + finishDelete = resolve; + }); + fixture.config.delete.mockImplementation(async (key) => { + markDeleteStarted(); + await deleting; + fixture.local.delete(key); + }); + + const loading = fixture.settingLoadedHandlers[0](); + await deleteStarted; + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(true); + expect(fixture.controller.snapshot().resumedRequestId).toBeNull(); + expect(fixture.api.registerWindow).not.toHaveBeenCalled(); + + finishDelete(); + await loading; + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(false); + expect(fixture.controller.snapshot().resumedRequestId).toBe(`compatibility-review-${requestedAt}`); + expect(fixture.api.registerWindow).toHaveBeenCalledOnce(); + }); + + it("can retry one-shot continuation consumption after a deletion failure", async () => { + const requestedAt = "2026-07-18T11:59:00.000Z"; + const continuation = JSON.stringify({ + formatVersion: 1, + requestId: `compatibility-review-${requestedAt}`, + scenarioId: "compatibility-review", + stage: "awaiting-restart", + requestedAt, + }); + const fixture = createFixture({ continuation }); + fixture.config.delete + .mockRejectedValueOnce(new Error("continuation delete failed")) + .mockImplementation(async (key) => { + fixture.local.delete(key); + }); + + await expect(fixture.settingLoadedHandlers[0]()).rejects.toThrow("continuation delete failed"); + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(true); + expect(fixture.controller.snapshot().resumedRequestId).toBeNull(); + expect(fixture.api.registerWindow).not.toHaveBeenCalled(); + + await fixture.settingLoadedHandlers[0](); + + expect(fixture.local.has(REVIEW_HARNESS_STATE_KEY)).toBe(false); + expect(fixture.controller.snapshot().resumedRequestId).toBe(`compatibility-review-${requestedAt}`); + expect(fixture.api.registerWindow).toHaveBeenCalledOnce(); + }); + it("uses the actual compatibility controller as the guided review boundary", async () => { const fixture = createFixture(); diff --git a/test/apps/webapp/WebAppRuntime.unit.spec.ts b/test/apps/webapp/WebAppRuntime.unit.spec.ts index 34d2cffa..e5aae6c3 100644 --- a/test/apps/webapp/WebAppRuntime.unit.spec.ts +++ b/test/apps/webapp/WebAppRuntime.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { LOG_LEVEL_INFO, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { LiveSyncBrowserServiceHubOptions } from "@/apps/browser/createLiveSyncBrowserServiceHub"; @@ -168,7 +169,7 @@ describe("WebAppRuntime lifecycle", () => { it("owns core start, directory scan, status reporting, and shutdown", async () => { const reportStatus = vi.fn(); - const runtime = new WebAppRuntime(createRootHandle(), { reportStatus }); + const runtime = new WebAppRuntime(createRootHandle(), { reportStatus, deviceValues: new MemoryValueStore() }); await runtime.start(); @@ -193,7 +194,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("finalises application readiness after importing local files for optional P2P", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.currentSettings.mockReturnValue(unconfiguredSettings); @@ -239,7 +240,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not publish readiness after a partial local-file import", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.collectFilesOnStorage.mockResolvedValue({ @@ -269,7 +270,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not publish readiness when database initialisation rejects manual preparation", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.onDatabaseInitialised.mockResolvedValue(false); @@ -282,7 +283,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not publish readiness when pending file events cannot be committed", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); runtimeMocks.commitPendingFileEvents.mockResolvedValue(false); @@ -295,7 +296,7 @@ describe("WebAppRuntime lifecycle", () => { }); it("does not repeat readiness finalisation after a successful manual scan", async () => { - const runtime = new WebAppRuntime(createRootHandle()); + const runtime = new WebAppRuntime(createRootHandle(), { deviceValues: new MemoryValueStore() }); await runtime.start(); vi.clearAllMocks(); @@ -311,7 +312,7 @@ describe("WebAppRuntime lifecycle", () => { it("rejects a failed core start after cleaning up the partial runtime", async () => { runtimeMocks.onLoad.mockResolvedValue(false); const reportStatus = vi.fn(); - const runtime = new WebAppRuntime(createRootHandle(), { reportStatus }); + const runtime = new WebAppRuntime(createRootHandle(), { reportStatus, deviceValues: new MemoryValueStore() }); await expect(runtime.start()).rejects.toThrow("Failed to initialise LiveSync"); @@ -325,7 +326,7 @@ describe("WebAppRuntime lifecycle", () => { it("shuts down once before scheduling a host reload", async () => { const scheduleReload = vi.fn(); - const runtime = new WebAppRuntime(createRootHandle(), { scheduleReload }); + const runtime = new WebAppRuntime(createRootHandle(), { scheduleReload, deviceValues: new MemoryValueStore() }); await runtime.start(); runtimeMocks.options?.restart?.schedule(); diff --git a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts index 2d8f678c..693b1e3d 100644 --- a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts +++ b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts @@ -1,3 +1,4 @@ +import { MemoryValueStore } from "@vrtmrz/value-store"; import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { DEFAULT_SETTINGS, SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase"; @@ -27,6 +28,7 @@ describe("WebPeer runtime composition", () => { translate: (key) => `webpeer:${key}`, }); const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), context, store: createMemoryStore(), }); @@ -43,6 +45,7 @@ describe("WebPeer runtime composition", () => { it("loads settings and opens the local database before announcing layout readiness", async () => { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); const loadSettings = vi.spyOn(runtime.services.setting, "loadSettings").mockResolvedValue(undefined); @@ -68,6 +71,7 @@ describe("WebPeer runtime composition", () => { vi.useFakeTimers(); try { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); vi.spyOn(runtime.services.setting, "loadSettings").mockResolvedValue(undefined); @@ -77,9 +81,7 @@ describe("WebPeer runtime composition", () => { P2P_Enabled: true, }); vi.spyOn(runtime.services.database, "openDatabase").mockResolvedValue(true); - const onResumed = vi - .spyOn(runtime.services.appLifecycle, "onResumed") - .mockResolvedValue(true); + const onResumed = vi.spyOn(runtime.services.appLifecycle, "onResumed").mockResolvedValue(true); const open = vi.spyOn(runtime.p2p.transportLifecycle, "connect").mockResolvedValue(undefined); await runtime.start(); @@ -94,9 +96,11 @@ describe("WebPeer runtime composition", () => { it("rejects start when the browser-local database cannot be opened", async () => { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); vi.spyOn(runtime.services.setting, "loadSettings").mockResolvedValue(undefined); + vi.spyOn(runtime.services.setting, "currentSettings").mockReturnValue({ ...DEFAULT_SETTINGS }); vi.spyOn(runtime.services.database, "openDatabase").mockResolvedValue(false); await expect(runtime.start()).rejects.toThrow("WebPeer local database could not be opened"); @@ -104,6 +108,7 @@ describe("WebPeer runtime composition", () => { it("isolates a specialised runtime and applies its device name before opening the database", async () => { const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), store: createMemoryStore(), deviceName: " p2p-check-browser-desktop-abc ", systemVaultName: "p2p-check-vault", @@ -114,15 +119,39 @@ describe("WebPeer runtime composition", () => { P2P_AutoStart: false, P2P_Enabled: false, }); - const setSmallConfig = vi.spyOn(runtime.services.config, "setSmallConfig").mockImplementation(() => undefined); + let finishWrite!: () => void; + const setValue = vi.spyOn(runtime.services.config, "setValue").mockImplementation( + () => + new Promise((resolve) => { + finishWrite = resolve; + }) + ); const openDatabase = vi.spyOn(runtime.services.database, "openDatabase").mockImplementation(async () => { - expect(setSmallConfig).toHaveBeenCalledWith(SETTING_KEY_P2P_DEVICE_NAME, "p2p-check-browser-desktop-abc"); + expect(setValue).toHaveBeenCalledWith(SETTING_KEY_P2P_DEVICE_NAME, "p2p-check-browser-desktop-abc"); return true; }); - await runtime.start(); + const starting = runtime.start(); + await vi.waitFor(() => expect(setValue).toHaveBeenCalledOnce()); + expect(openDatabase).not.toHaveBeenCalled(); + finishWrite(); + await starting; expect(runtime.services.API.getSystemVaultName()).toBe("p2p-check-vault"); expect(openDatabase).toHaveBeenCalledOnce(); }); + + it("reads the current device name after the pane updates the store", async () => { + const runtime = new WebPeerRuntime({ + deviceValues: new MemoryValueStore(), + store: createMemoryStore(), + systemVaultName: "webpeer-vault", + }); + vi.spyOn(runtime.services.setting, "currentSettings").mockReturnValue({ ...DEFAULT_SETTINGS }); + expect(await runtime.getDeviceName()).toBe("webpeer-vault"); + await runtime.services.config.setValue(SETTING_KEY_P2P_DEVICE_NAME, "first-name"); + expect(await runtime.getDeviceName()).toBe("first-name"); + await runtime.services.config.setValue(SETTING_KEY_P2P_DEVICE_NAME, "renamed-device"); + expect(await runtime.getDeviceName()).toBe("renamed-device"); + }); }); diff --git a/test/contracts/serviceContext.ts b/test/contracts/serviceContext.ts index ddfb6ce4..ec712e09 100644 --- a/test/contracts/serviceContext.ts +++ b/test/contracts/serviceContext.ts @@ -17,13 +17,12 @@ export const SERVICE_CONTEXT_MEMBERS = [ "vault", "test", "UI", - "config", "keyValueDB", "control", -] as const satisfies readonly Exclude[]; +] as const satisfies readonly Exclude[]; export type ServiceContextMember = (typeof SERVICE_CONTEXT_MEMBERS)[number]; -type MissingServiceContextMember = Exclude, ServiceContextMember>; +type MissingServiceContextMember = Exclude, ServiceContextMember>; const serviceContextMembersAreExhaustive: [MissingServiceContextMember] extends [never] ? true : never = true; void serviceContextMembersAreExhaustive; diff --git a/test/e2e-obsidian/runner/liveSyncWorkflow.ts b/test/e2e-obsidian/runner/liveSyncWorkflow.ts index cde2d049..e256fc10 100644 --- a/test/e2e-obsidian/runner/liveSyncWorkflow.ts +++ b/test/e2e-obsidian/runner/liveSyncWorkflow.ts @@ -111,7 +111,7 @@ export async function readE2eCompatibilityMarker( return await evalObsidianJson( cliBinary, [ - "(()=>{", + "(async()=>{", "const core=app.plugins.plugins['obsidian-livesync'].core;", "const setting=core.services.setting;", "const settings=setting.currentSettings();", @@ -122,7 +122,7 @@ export async function readE2eCompatibilityMarker( "return JSON.stringify({", "vaultName,additionalSuffix,expectedStorageKey,", "rawStorageValue:localStorage.getItem(expectedStorageKey),", - "serviceValue:setting.getSmallConfig(markerKey),", + "serviceValue:await setting.config.getValue(markerKey,''),", "versionUpFlash:settings.versionUpFlash,", "});", "})()", diff --git a/test/e2e-obsidian/runner/upgradeWorkflow.ts b/test/e2e-obsidian/runner/upgradeWorkflow.ts index 76b6c7aa..1af10d32 100644 --- a/test/e2e-obsidian/runner/upgradeWorkflow.ts +++ b/test/e2e-obsidian/runner/upgradeWorkflow.ts @@ -288,7 +288,7 @@ export async function readRuntimeUpgradeState( "localDatabaseDocumentCount:databaseInfo.doc_count,", "nodeId:nodeInfo?.nodeid??replicator?.nodeid??'',", "legacyCompatibilityMarker:localStorage.getItem(`obsidian-live-sync-ver${vaultName}`),", - "compatibilityMarker:setting.getSmallConfig('database-compatibility-version')??'',", + "compatibilityMarker:await (setting.config?.getValue?.('database-compatibility-version','')??Promise.resolve(setting.getSmallConfig?.('database-compatibility-version')??'')),", "compatibilityStorageEntries:Object.fromEntries(Array.from({length:localStorage.length},(_,index)=>localStorage.key(index))", ".filter((key)=>key!==null)", ".filter(key=>key.startsWith('obsidian-live-sync-ver')||key.endsWith('-database-compatibility-version'))", @@ -335,7 +335,7 @@ export async function readRuntimeSettingsUpgradeState( "const migrationState=setting.getSettingsMigrationState?.();", "return JSON.stringify({", "pluginVersion:app.plugins.manifests['obsidian-livesync']?.version??'unknown',", - "compatibilityMarker:setting.getSmallConfig?.('database-compatibility-version')??'',", + "compatibilityMarker:await (setting.config?.getValue?.('database-compatibility-version','')??Promise.resolve(setting.getSmallConfig?.('database-compatibility-version')??'')),", "migrationState,", "settings:{", "isConfigured:settings.isConfigured,settingVersion:settings.settingVersion,", diff --git a/test/e2e-obsidian/scripts/dialog-mounts.ts b/test/e2e-obsidian/scripts/dialog-mounts.ts index 5f3e6307..2456d9cd 100644 --- a/test/e2e-obsidian/scripts/dialog-mounts.ts +++ b/test/e2e-obsidian/scripts/dialog-mounts.ts @@ -1,7 +1,9 @@ import { readFile } from "node:fs/promises"; import { join } from "node:path"; +import type { Page } from "playwright"; import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/setup-uri"; import { decryptString } from "@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption"; +import { SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { $msg } from "../../../src/common/translation.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { createE2eCouchDbPluginData, waitForLiveSyncCoreReady } from "../runner/liveSyncWorkflow.ts"; @@ -13,18 +15,34 @@ import { captureObsidianPage, obsidianRemoteDebuggingPort, openLiveSyncSettings, - withObsidianPage, + withObsidianPage as withObsidianSessionPage, } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; const dialogRunStateKey = "__livesyncE2EDialogMount"; +const savedP2PDeviceName = "e2e-p2p-saved-device"; const repairRunStateKey = "__livesyncE2ETroubleshootingRepair"; const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_DIALOG_TIMEOUT_MS ?? 10000); +async function withObsidianPage(port: number, operation: (page: Page) => Promise): Promise { + return await withObsidianSessionPage(port, async (page) => { + for (const candidate of page.context().pages()) { + const hasPlugin = await candidate + .evaluate( + () => (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"] !== undefined + ) + .catch(() => false); + if (hasPlugin) return await operation(candidate); + } + throw new Error("The LiveSync Vault renderer is unavailable."); + }); +} + type DialogueMode = "desktop" | "mobile"; type DialogueRunState = { done: boolean; + deviceNameRead?: boolean; error?: string; expected?: unknown; kind: string; @@ -46,6 +64,14 @@ type LiveSyncTestPlugin = { createAllChunks(force: boolean): Promise; }; modules: SetupManagerHandle[]; + services: { + setting: { + config: { + getValue(key: string, defaultValue?: string): Promise; + setValue(key: string, value: string): Promise; + }; + }; + }; settings: Record; }; }; @@ -91,39 +117,63 @@ async function openRemoteSelectionDialogue(): Promise { async function openP2PRemoteSelectionDialogueForInspection(): Promise { await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - await page.evaluate((stateKey) => { - const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; - if (plugin === undefined) throw new Error("Self-hosted LiveSync is not loaded"); - const manager = plugin.core.modules.find((module) => module.constructor.name === "SetupManager"); - if ( - typeof manager?.onSelectServer !== "function" || - typeof manager.onConfirmApplySettingsFromWizard !== "function" - ) { - throw new Error("Could not find the P2P setup workflow"); + await page.evaluate( + async ({ stateKey, deviceNameKey, deviceName }) => { + const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; + if (plugin === undefined) throw new Error("Self-hosted LiveSync is not loaded"); + const initialPeerName = plugin.core.settings.P2P_DevicePeerName; + if (typeof initialPeerName === "string" && initialPeerName.trim() !== "") { + throw new Error("The P2P setup fixture unexpectedly has an initial device name."); + } + const manager = plugin.core.modules.find((module) => module.constructor.name === "SetupManager"); + if ( + typeof manager?.onSelectServer !== "function" || + typeof manager.onConfirmApplySettingsFromWizard !== "function" + ) { + throw new Error("Could not find the P2P setup workflow"); + } + const config = plugin.core.services.setting.config; + await config.setValue(deviceNameKey, deviceName); + const originalGetValue = config.getValue; + const originalConfirm = manager.onConfirmApplySettingsFromWizard; + const state: DialogueRunState = { + kind: "p2p-compatibility-settings", + done: false, + deviceNameRead: false, + }; + (globalThis as unknown as Record)[stateKey] = state; + config.getValue = async (key, defaultValue) => { + const value = await originalGetValue.call(config, key, defaultValue); + if (key === deviceNameKey && value === deviceName) state.deviceNameRead = true; + return value; + }; + manager.onConfirmApplySettingsFromWizard = async (settings: unknown) => { + state.expected = settings; + return true; + }; + void manager + .onSelectServer(plugin.core.settings, "unknown") + .then( + (result) => { + state.result = result; + state.done = true; + }, + (error: unknown) => { + state.error = error instanceof Error ? error.message : String(error); + state.done = true; + } + ) + .finally(() => { + manager.onConfirmApplySettingsFromWizard = originalConfirm; + config.getValue = originalGetValue; + }); + }, + { + stateKey: dialogRunStateKey, + deviceNameKey: SETTING_KEY_P2P_DEVICE_NAME, + deviceName: savedP2PDeviceName, } - const originalConfirm = manager.onConfirmApplySettingsFromWizard; - const state: DialogueRunState = { kind: "p2p-compatibility-settings", done: false }; - (globalThis as unknown as Record)[stateKey] = state; - manager.onConfirmApplySettingsFromWizard = async (settings: unknown) => { - state.expected = settings; - return true; - }; - void manager - .onSelectServer(plugin.core.settings, "unknown") - .then( - (result) => { - state.result = result; - state.done = true; - }, - (error: unknown) => { - state.error = error instanceof Error ? error.message : String(error); - state.done = true; - } - ) - .finally(() => { - manager.onConfirmApplySettingsFromWizard = originalConfirm; - }); - }, dialogRunStateKey); + ); }); } @@ -388,7 +438,7 @@ async function verifyRemoteSelectionDialogue(mode: DialogueMode): Promise !element.isConnected, previousDialogue, { timeout: uiTimeoutMs }); await modal.waitFor({ state: "visible", timeout: uiTimeoutMs }); } @@ -495,9 +545,21 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { has: page.locator(".modal-title").filter({ hasText: "P2P Configuration" }), }); await modal.waitFor({ state: "visible", timeout: uiTimeoutMs }); + const peerName = modal.locator('input[name="p2p-device-peer-id"]'); + await peerName.waitFor({ state: "visible", timeout: uiTimeoutMs }); + await page.waitForFunction( + (expectedName) => { + const input = document.querySelector('input[name="p2p-device-peer-id"]'); + return input instanceof HTMLInputElement && !input.disabled && input.value === expectedName; + }, + savedP2PDeviceName, + { timeout: uiTimeoutMs } + ); + if ((await peerName.inputValue()) !== savedP2PDeviceName) { + throw new Error("The saved P2P device name was not loaded before submitting the setup dialogue."); + } await modal.locator('input[name="p2p-room-id"]').fill("e2e-p2p-compatibility"); await modal.locator('input[name="p2p-password"]').fill("e2e-passphrase"); - await modal.locator('input[name="p2p-device-peer-id"]').fill("e2e-device"); const compatibility = modal.locator("details").filter({ has: page.locator("summary").filter({ hasText: "Connection compatibility" }), @@ -565,8 +627,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { { timeout: uiTimeoutMs } ); const resetNotice = compatibility.locator(".sls-info-note-notice").filter({ - hasText: - "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.", + hasText: "TURN relay only requires TURN configuration. Connection path has been restored to Automatic.", }); await resetNotice.waitFor({ state: "visible", timeout: uiTimeoutMs }); await resetNotice @@ -594,10 +655,16 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { }); const state = await assertDialogueRunCompleted(); + if (state.deviceNameRead !== true) { + throw new Error("The saved P2P device name was not read through setting.config."); + } if (state.result !== true || typeof state.expected !== "object" || state.expected === null) { throw new Error("The P2P compatibility settings were not passed to the setup workflow."); } const settings = state.expected as Record; + if (settings.P2P_DevicePeerName !== savedP2PDeviceName) { + throw new Error("The saved P2P device name was not passed through the setup workflow."); + } if (settings.P2P_maxWirePayloadBytes !== 800 || settings.P2P_connectionPath !== "relay") { throw new Error( `The P2P compatibility choices did not round-trip through the setup dialogue: ${JSON.stringify({ @@ -672,15 +739,14 @@ async function verifyGenerateSetupUriDialogue(mode: DialogueMode): Promise - (globalThis as ObsidianTestGlobal).app?.commands?.executeCommandById(commandId) === true, + (commandId) => (globalThis as ObsidianTestGlobal).app?.commands?.executeCommandById(commandId) === true, "obsidian-livesync:livesync-copysetupuri" ); }); @@ -714,7 +780,8 @@ async function verifyGenerateSetupUriDialogue(mode: DialogueMode): Promise { if (!(await toggle.evaluate((element) => element.classList.contains("is-enabled")))) { throw new Error("The automatic compatible-setting policy was displayed as disabled while still undefined."); } + await settingsNavigator.close(); }); } @@ -1094,9 +1162,10 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { ); await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - const liveSyncSettings = page.locator(".sls-setting"); + const settingsNavigator = await openLiveSyncSettings(page, uiTimeoutMs); + const liveSyncSettings = await settingsNavigator.openPage("Hatch"); const logSetting = liveSyncSettings.locator(".setting-item").filter({ - has: page.getByText("Write logs into the file", { exact: true }), + has: settingsNavigator.page.getByText("Write logs into the file", { exact: true }), }); await logSetting.locator(".checkbox-container").click({ timeout: uiTimeoutMs }); await page.waitForFunction( @@ -1130,10 +1199,10 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { // Saving a toggle refreshes the settings pane. Resolve the visible control again so the // second action does not target the detached pre-save element. - const refreshedLogToggle = page - .locator(".sls-setting:visible .setting-item:visible") + const refreshedLogToggle = liveSyncSettings + .locator(".setting-item:visible") .filter({ - has: page.getByText("Write logs into the file", { exact: true }), + has: settingsNavigator.page.getByText("Write logs into the file", { exact: true }), }) .locator(".checkbox-container:visible") .last(); @@ -1179,13 +1248,9 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { } }; }, repairRunStateKey); - await page - .locator(".sls-setting:visible") - .last() - .getByRole("button", { name: "Recreate current chunks", exact: true }) - .click({ - timeout: uiTimeoutMs, - }); + await liveSyncSettings.getByRole("button", { name: "Recreate current chunks", exact: true }).click({ + timeout: uiTimeoutMs, + }); await page.waitForFunction( (stateKey) => (globalThis as unknown as Record)[stateKey]?.done === true, @@ -1200,13 +1265,9 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { throw new Error(`Recreate missing chunks failed: ${repairState.error}`); } - await page - .locator(".sls-setting:visible") - .last() - .getByRole("button", { name: "Begin inspection", exact: true }) - .click({ - timeout: uiTimeoutMs, - }); + await liveSyncSettings.getByRole("button", { name: "Begin inspection", exact: true }).click({ + timeout: uiTimeoutMs, + }); await page .locator(".notice") .filter({ hasText: /^done$/u }) @@ -1214,6 +1275,7 @@ async function verifyHatchSurfacesAndSafeActions(): Promise { state: "visible", timeout: uiTimeoutMs, }); + await settingsNavigator.close(); }); return screenshotPath; diff --git a/test/e2e-obsidian/scripts/review-harness.ts b/test/e2e-obsidian/scripts/review-harness.ts index 49e22c67..64631db9 100644 --- a/test/e2e-obsidian/scripts/review-harness.ts +++ b/test/e2e-obsidian/scripts/review-harness.ts @@ -372,13 +372,21 @@ async function restartAndResumeHarness(): Promise { await harness .locator('[data-testid="review-harness-resumed"]') .waitFor({ state: "visible", timeout: uiTimeoutMs }); - const continuationRemoved = await page.evaluate((stateKey) => { + const continuationRemoved = await page.evaluate(async (stateKey) => { const plugin = (globalThis as ReviewHarnessTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; if (typeof plugin !== "object" || plugin === null || !("core" in plugin)) { throw new Error("Self-hosted LiveSync is unavailable after restart."); } - const core = (plugin as { core: { services: { setting: { getSmallConfig(key: string): string } } } }).core; - return core.services.setting.getSmallConfig(stateKey) === ""; + const core = ( + plugin as { + core: { + services: { + setting: { config: { getValue(key: string, defaultValue: string): Promise } }; + }; + }; + } + ).core; + return (await core.services.setting.config.getValue(stateKey, "")) === ""; }, REVIEW_HARNESS_STATE_KEY); if (!continuationRemoved) throw new Error("The one-shot continuation was not removed before use."); await assertNoHorizontalOverflow(page, harness, { label: "resumed Review Harness" }); diff --git a/test/e2e-obsidian/scripts/settings-ui.ts b/test/e2e-obsidian/scripts/settings-ui.ts index c4be1651..0b930234 100644 --- a/test/e2e-obsidian/scripts/settings-ui.ts +++ b/test/e2e-obsidian/scripts/settings-ui.ts @@ -58,8 +58,9 @@ type LiveSyncTestPlugin = { usePowerUserMode: boolean; useEdgeCaseMode: boolean; hashCacheMaxCount: number; + customChunkSize: number; }; - getSmallConfig(key: string): string | null; + config: { getValue(key: string, defaultValue: string): Promise }; }; }; }; @@ -484,12 +485,12 @@ async function verifyCompatibilityReview(): Promise { }); await withObsidianPage(port, async (page) => { - const markerBeforeAcknowledgement = await page.evaluate(() => { + const markerBeforeAcknowledgement = await page.evaluate(async () => { const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; if (plugin === undefined) throw new Error("Self-hosted LiveSync is unavailable"); - return plugin.core.services.setting.getSmallConfig("database-compatibility-version"); + return await plugin.core.services.setting.config.getValue("database-compatibility-version", ""); }); - if (markerBeforeAcknowledgement !== null && markerBeforeAcknowledgement !== "") { + if (markerBeforeAcknowledgement !== "") { throw new Error( `The database version was marked as acknowledged before review: ${markerBeforeAcknowledgement}` ); @@ -579,12 +580,12 @@ async function verifyCompatibilityReview(): Promise { .click(); await summary.waitFor({ state: "hidden", timeout: uiTimeoutMs }); await page.waitForFunction( - (expectedVersion) => { + async (expectedVersion) => { const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; if (plugin === undefined) return false; const setting = plugin.core.services.setting; return ( - setting.getSmallConfig("database-compatibility-version") === expectedVersion && + (await setting.config.getValue("database-compatibility-version", "")) === expectedVersion && setting.currentSettings().versionUpFlash === "" ); }, @@ -611,10 +612,16 @@ async function verifyConfigDoctorFollowsCompatibilityReview(): Promise { state: "visible", timeout: uiTimeoutMs, }); - await doctor.getByText("Enhance chunk size", { exact: true }).waitFor({ - state: "visible", - timeout: uiTimeoutMs, + const chunkSize = await page.evaluate(() => { + const plugin = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins["obsidian-livesync"]; + return plugin?.core.services.setting.currentSettings().customChunkSize; }); + if (chunkSize !== 0) { + throw new Error("Compatibility review must retain the fixture's zero chunk-size value."); + } + if ((await doctor.getByText("Enhance chunk size", { exact: true }).count()) !== 0) { + throw new Error("Config Doctor must accept the recommended zero chunk-size value."); + } if ((await doctor.getByText("Data Compression", { exact: true }).count()) !== 0) { throw new Error("Config Doctor still treats supported Data Compression as a problem."); } @@ -978,6 +985,7 @@ function createSettingsPluginData(settingsOnlyRun: boolean): Record Date: Fri, 23 Oct 2026 14:40:01 +0000 Subject: [PATCH 37/68] Close WebPeer runtimes after composition tests --- test/apps/webpeer/WebPeerRuntime.unit.spec.ts | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts index 693b1e3d..73362b39 100644 --- a/test/apps/webpeer/WebPeerRuntime.unit.spec.ts +++ b/test/apps/webpeer/WebPeerRuntime.unit.spec.ts @@ -3,9 +3,9 @@ import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { DEFAULT_SETTINGS, SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase"; import { EVENT_LAYOUT_READY } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { WebPeerRuntime } from "@/apps/webpeer/src/WebPeerRuntime"; +import { WebPeerRuntime, type WebPeerRuntimeOptions } from "@/apps/webpeer/src/WebPeerRuntime"; function createMemoryStore(): SimpleStore { const values = new Map(); @@ -23,11 +23,30 @@ function createMemoryStore(): SimpleStore { } describe("WebPeer runtime composition", () => { + const runtimes = new Set(); + + function createRuntime(options: WebPeerRuntimeOptions): WebPeerRuntime { + const runtime = new WebPeerRuntime(options); + runtimes.add(runtime); + return runtime; + } + + afterEach(async () => { + try { + for (const runtime of runtimes) { + await runtime.shutdown(); + } + } finally { + runtimes.clear(); + vi.restoreAllMocks(); + } + }); + it("preserves one context and live P2P result across the service graph and pane host", () => { const context = createServiceContext({ translate: (key) => `webpeer:${key}`, }); - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), context, store: createMemoryStore(), @@ -44,7 +63,7 @@ describe("WebPeer runtime composition", () => { }); it("loads settings and opens the local database before announcing layout readiness", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); @@ -70,7 +89,7 @@ describe("WebPeer runtime composition", () => { it("delegates automatic P2P startup to the resumed lifecycle handler", async () => { vi.useFakeTimers(); try { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); @@ -95,7 +114,7 @@ describe("WebPeer runtime composition", () => { }); it("rejects start when the browser-local database cannot be opened", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), }); @@ -107,7 +126,7 @@ describe("WebPeer runtime composition", () => { }); it("isolates a specialised runtime and applies its device name before opening the database", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), deviceName: " p2p-check-browser-desktop-abc ", @@ -142,7 +161,7 @@ describe("WebPeer runtime composition", () => { }); it("reads the current device name after the pane updates the store", async () => { - const runtime = new WebPeerRuntime({ + const runtime = createRuntime({ deviceValues: new MemoryValueStore(), store: createMemoryStore(), systemVaultName: "webpeer-vault", From fc2b35e0dc8538a639ba7efd8bcfd97001fc769d Mon Sep 17 00:00:00 2001 From: s-qin Date: Fri, 23 Oct 2026 23:54:13 +0800 Subject: [PATCH 38/68] perf(startup): inspect storage sizes without loading healthy files --- .../startupLifecycle/incompleteDocuments.ts | 33 ++++++++++++----- .../incompleteDocuments.unit.spec.ts | 36 +++++++++++++++++++ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts index 9836d682..0b4ee3de 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts @@ -31,7 +31,7 @@ export interface IncompleteDocumentsDependencies { readonly localDatabase: Pick; readonly getPath: StartupPathReader; readonly isTargetFile: (path: string) => Promise; - readonly storageAccess: Pick; + readonly storageAccess: Pick; readonly fileHandler: Pick; readonly keyValueDB: Pick; readonly noticeGroups: NoticeGroups; @@ -94,16 +94,11 @@ export async function checkIncompleteDocuments( } const isConflicted = metaDoc?._conflicts && metaDoc._conflicts.length > 0; - let storageFileContent; - try { - storageFileContent = await dependencies.storageAccess.readHiddenFileBinary(path); - } catch (e) { - Logger(`Failed to read file ${path}: Possibly unprocessed or missing`); - Logger(e, LOG_LEVEL_VERBOSE); + const storageStat = await dependencies.storageAccess.statHidden(path); + if (!storageStat || storageStat.type !== "file") { continue; } - // const storageFileBlob = createBlob(storageFileContent); - const sizeOnStorage = storageFileContent.byteLength; + let sizeOnStorage = storageStat.size; const recordedSize = doc.size; const docBlob = readAsBlob(doc); const actualSize = docBlob.size; @@ -113,6 +108,15 @@ export async function checkIncompleteDocuments( sizeOnStorage !== recordedSize || isConflicted ) { + let storageFileContent; + try { + storageFileContent = await dependencies.storageAccess.readHiddenFileBinary(path); + } catch (e) { + Logger(`Failed to read file ${path}: Possibly unprocessed or missing`); + Logger(e, LOG_LEVEL_VERBOSE); + continue; + } + sizeOnStorage = storageFileContent.byteLength; const contentMatched = await isDocContentSame(doc.data, storageFileContent); errorFiles.push({ path, @@ -125,6 +129,17 @@ export async function checkIncompleteDocuments( Logger( `Size mismatch for ${path}: ${recordedSize} (DB Recorded) , ${actualSize} (DB Stored) , ${sizeOnStorage} (Storage Stored), ${contentMatched ? "Content Matched" : "Content Mismatched"} ${isConflicted ? "Conflicted" : "Not Conflicted"}` ); + } else { + const currentStat = await dependencies.storageAccess.statHidden(path); + if ( + !currentStat || + currentStat.type !== "file" || + currentStat.size !== storageStat.size || + currentStat.mtime !== storageStat.mtime || + currentStat.ctime !== storageStat.ctime + ) { + throw new Error("Storage changed during the incomplete document check"); + } } } if (errorFiles.length == 0) { diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts index f40305c4..2f30f12b 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts @@ -99,6 +99,7 @@ function createDependencies(findAllNormalDocs: FindAllNormalDocs = noDocuments, isTargetFile: vi.fn(async () => true), storageAccess: { readHiddenFileBinary, + statHidden: vi.fn(async (path: string) => getFixture(path)?.stub.stat ?? null), getFileStub, }, fileHandler: { storeFileToDB }, @@ -157,6 +158,41 @@ describe("checkIncompleteDocuments", () => { expect(noticeGroups.finish).toHaveBeenCalledWith("startup-integrity-check"); }); + it("does not read healthy storage contents only to determine their size", async () => { + const healthy = documentFixture("healthy.md", { storageContent: "abc" }); + const fixture = createDependencies(documentsFrom([healthy]), [healthy]); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).not.toHaveBeenCalled(); + expect(fixture.askSelectStringDialogue).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).toHaveBeenCalledWith("checkIncompleteDocs", true); + }); + + it("does not mark a scan complete when storage changes during the size check", async () => { + const healthy = documentFixture("changing.md", { storageContent: "abc" }); + const fixture = createDependencies(documentsFrom([healthy]), [healthy]); + vi.mocked(fixture.dependencies.storageAccess.statHidden) + .mockResolvedValueOnce(healthy.stub.stat) + .mockResolvedValueOnce({ ...healthy.stub.stat, mtime: 3 }); + + await expect(checkIncompleteDocuments(fixture.dependencies)).rejects.toThrow("Storage changed"); + + expect(fixture.readHiddenFileBinary).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); + }); + + it("still reads conflicted storage for the existing repair dialogue", async () => { + const conflicted = documentFixture("conflicted.md", { storageContent: "abc", conflicts: ["2-other"] }); + const fixture = createDependencies(documentsFrom([conflicted]), [conflicted]); + fixture.askSelectStringDialogue.mockImplementation(selectButton(0)); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("conflicted.md"); + expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); + }); + it("repairs a recoverable document when FIX is selected", async () => { const recoverable = documentFixture("recoverable.md"); const fixture = createDependencies(documentsFrom([recoverable]), [recoverable]); From 39ef429d7bcefdc062320b43200b40c683b9f54c Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 16:07:35 +0000 Subject: [PATCH 39/68] Clarify device review guidance and follow dialogue windows --- docs/settings.md | 2 +- .../messages/LiveSyncProvisionalMessages.ts | 13 ++-- .../SettingDialogue/PaneRemoteConfig.ts | 2 +- .../dialogues/JoinedDevicesDialog.svelte | 2 +- src/serviceFeatures/useJoinedDevices.ts | 22 ++++-- test/e2e-obsidian/scripts/joined-devices.ts | 76 ++++++++++++++----- updates.md | 4 +- 7 files changed, 81 insertions(+), 40 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index c2b33e09..1a81032c 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -236,7 +236,7 @@ The active connection type. This is automatically projected to the legacy config ### Joined devices -Open **Joined devices** to read the device records on the active CouchDB or Object Storage connection. The same dialogue is available from the command palette and version Notices. **Refresh** reads the remote again. This device appears first, including when its record is absent from the remote. Opening or refreshing this dialogue does not create a missing CouchDB database. +Choose **Review devices** under **Joined devices** to read the device records on the active CouchDB or Object Storage connection. The same **Review devices** action is available from the command palette and version Notices. **Refresh** reads the remote again. This device appears first, including when its record is absent from the remote. Opening or refreshing this dialogue does not create a missing CouchDB database. The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 44d4c2bc..054bb7ba 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -26,8 +26,8 @@ export const liveSyncProvisionalEnglishMessages = { "This remote does not currently have a record of joined devices.", "This list shows device information recorded by the remote. It does not show whether a device is online now.": "This list shows device information recorded by the remote. It does not show whether a device is online now.", - "This device is locked out. Use the existing recovery process to join again.": - "This device is locked out. Use the existing recovery process to join again.", + "This device has been locked out. Use the existing recovery process to join again.": + "This device has been locked out. Use the existing recovery process to join again.", "This device": "This device", "This remote has been cleaned. Use the existing recovery process to join devices again as needed.": "This remote has been cleaned. Use the existing recovery process to join devices again as needed.", @@ -42,10 +42,10 @@ export const liveSyncProvisionalEnglishMessages = { Refresh: "Refresh", "Plug-in version": "Plug-in version", Platform: "Platform", - "Other joined devices last reported an older plug-in version: ${versions}.": - "Other joined devices last reported an older plug-in version: ${versions}.", - "Other joined devices last reported a newer plug-in version: ${versions}.": - "Other joined devices last reported a newer plug-in version: ${versions}.", + "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.": + "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", + "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.": + "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", Open: "Open", "Not recorded": "Not recorded", "Not accepted": "Not accepted", @@ -58,6 +58,7 @@ export const liveSyncProvisionalEnglishMessages = { "Joined devices are available after database startup with an active CouchDB or Object Storage connection.": "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", "Joined devices": "Joined devices", + "Review devices": "Review devices", "Forgettingâ€Ļ": "Forgettingâ€Ļ", "Forgetting the record does not delete Vault files or revoke credentials.": "Forgetting the record does not delete Vault files or revoke credentials.", diff --git a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts index 41fd1df8..81a68312 100644 --- a/src/modules/features/SettingDialogue/PaneRemoteConfig.ts +++ b/src/modules/features/SettingDialogue/PaneRemoteConfig.ts @@ -106,7 +106,7 @@ export function paneRemoteConfig( .setName($msg("Joined devices")) .setDesc($msg("Review device records for the active CouchDB or Object Storage connection.")) .addButton((button) => - button.setButtonText($msg("Open")).onClick(async () => { + button.setButtonText($msg("Review devices")).onClick(async () => { const result = await this.services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); if (result.status === "unavailable" || result.value === "unavailable") { this.services.API.addLog( diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte index 31b30356..91c54b0f 100644 --- a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -176,7 +176,7 @@ {#if device.self && device.lockedOut}

{translate( - "This device is locked out. Use the existing recovery process to join again." + "This device has been locked out. Use the existing recovery process to join again." )}

{/if} diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts index 37d50b0f..c5fcc759 100644 --- a/src/serviceFeatures/useJoinedDevices.ts +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -62,14 +62,20 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi notices.setItem(noticeGroup, direction, { message: direction === "newer" - ? $msg("Other joined devices last reported a newer plug-in version: ${versions}.", { - versions: versions.join(", "), - }) - : $msg("Other joined devices last reported an older plug-in version: ${versions}.", { - versions: versions.join(", "), - }), + ? $msg( + "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", + { + versions: versions.join(", "), + } + ) + : $msg( + "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", + { + versions: versions.join(", "), + } + ), action: { - label: $msg("Joined devices"), + label: $msg("Review devices"), onSelect: () => { if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); @@ -190,7 +196,7 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi services.appLifecycle.onInitialise.addHandler(() => { services.API.addCommand({ id: "joined-devices", - name: $msg("Joined devices"), + name: $msg("Review devices"), icon: "monitor-smartphone", checkCallback: (checking) => { if (!available()) return false; diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index d5736139..15d496e8 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -1,3 +1,5 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; import { DEVICE_ID_PREFERRED, MILESTONE_DOCID, NODEINFO_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { evalObsidianJson } from "../runner/cli.ts"; import { @@ -27,12 +29,7 @@ import { } from "@vrtmrz/obsidian-test-session"; import { iPhoneSafeArea, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; -import { - captureObsidianDialogue, - openLiveSyncSettings, - waitForVisibleObsidianDialogue, - withObsidianPage, -} from "../runner/ui.ts"; +import { openLiveSyncSettings, waitForVisibleObsidianDialogue, withObsidianPage } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; import type { Locator, Page } from "playwright"; @@ -161,7 +158,9 @@ function createFixture(self: SelfDeviceInfo): DeviceFixture { async function openFromRemoteSettings(page: Page): Promise { const settings = await openLiveSyncSettings(page, uiTimeoutMs); const remoteConfiguration = await settings.openPage("Remote Configuration"); - await remoteConfiguration.getByRole("button", { name: "Open", exact: true }).click({ timeout: uiTimeoutMs }); + await remoteConfiguration + .getByRole("button", { name: "Review devices", exact: true }) + .click({ timeout: uiTimeoutMs }); const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await dialogue.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); return dialogue; @@ -185,7 +184,7 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa await selfCard.getByText("Locked out", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); await selfCard.getByText("Not accepted", { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); await selfCard - .getByText("This device is locked out.", { exact: false }) + .getByText("This device has been locked out.", { exact: false }) .waitFor({ state: "visible", timeout: uiTimeoutMs }); await assertDetail(selfCard, "Platform", self.platform); await assertDetail(selfCard, "Plug-in version", self.pluginVersion); @@ -339,6 +338,27 @@ async function dismissNotices(page: Page): Promise { assertEqual(await notices.count(), 0, "Notices still obscured the dialogue screenshot."); } +async function captureJoinedDevices( + port: number, + filename: string, + assertReady: (page: Page) => Promise +): Promise { + const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e"; + await mkdir(outputDirectory, { recursive: true }); + await withObsidianPage(port, async (page) => { + try { + const dialogue = await assertReady(page); + await dialogue.page().screenshot({ path: join(outputDirectory, filename), fullPage: true }); + } catch (error) { + const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", 500).catch(() => undefined); + const failurePath = join(outputDirectory, filename.replace(/\.png$/u, ".failure.png")); + await (dialogue?.page() ?? page).screenshot({ path: failurePath, fullPage: true }).catch(() => undefined); + console.error(`Joined devices UI failure screenshot: ${failurePath}`); + throw error; + } + }); +} + async function main(): Promise { const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); @@ -381,24 +401,35 @@ async function main(): Promise { assert(Boolean(preparedMilestone._rev), "Remote preparation did not create a revisioned milestone document."); await putCouchDbDocument(couchDb, dbName, { ...fixture.milestone, _rev: preparedMilestone._rev }); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { - await page.setViewportSize({ width: 1280, height: 1100 }); + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { const dialogue = await openFromRemoteSettings(page); - await assertInitialDialogue(dialogue, self, page); - await page + const dialoguePage = dialogue.page(); + await dialoguePage.setViewportSize({ width: 1280, height: 1100 }); + await assertInitialDialogue(dialogue, self, dialoguePage); + await dialoguePage .locator(".notice") - .getByText("Other joined devices last reported a newer plug-in version: 999.0.0.", { exact: true }) + .getByText( + "A newer plug-in version was detected in other device records: 999.0.0. We recommend checking whether this device needs updating.", + { exact: true } + ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); + return dialogue; }); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await dialogue.getByRole("button", { name: "Close", exact: true }).click({ timeout: uiTimeoutMs }); - await page.locator(".notice").getByRole("button", { name: "Joined devices", exact: true }).first().click(); + await dialogue + .page() + .locator(".notice") + .getByRole("button", { name: "Review devices", exact: true }) + .first() + .click(); const reopened = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await reopened.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); - await dismissNotices(page); + await dismissNotices(reopened.page()); + return reopened; }); - await captureObsidianDialogue( + await captureJoinedDevices( session.remoteDebuggingPort, "joined-devices-forget-confirmation.png", async (page) => { @@ -406,12 +437,14 @@ async function main(): Promise { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await showForgetConfirmation(dialogue); await deviceCard(dialogue, targetId).locator(".forget-confirmation").scrollIntoViewIfNeeded(); + return dialogue; } ); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-after-forget.png", async (page) => { + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-after-forget.png", async (page) => { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await confirmForgetTarget(dialogue); await dialogue.locator(".modal-content").evaluate((element) => element.scrollTo(0, 0)); + return dialogue; }); await withObsidianPage(session.remoteDebuggingPort, async (page) => { const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); @@ -423,7 +456,7 @@ async function main(): Promise { await setObsidianMobileTestMode(session.remoteDebuggingPort, true, uiTimeoutMs); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); - await captureObsidianDialogue(session.remoteDebuggingPort, "joined-devices-mobile.png", async (page) => { + await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-mobile.png", async (page) => { const opened = await page.evaluate((id) => { const app = ( window as typeof window & { @@ -435,13 +468,14 @@ async function main(): Promise { assert(opened, "The Joined devices command did not open the dialogue."); const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await dialogue.getByText(retainedId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); - await assertScrollableMobileDialogue(page, dialogue); + await assertScrollableMobileDialogue(dialogue.page(), dialogue); assertEqual( await dialogue.locator(".device-card").count(), 2, "The refreshed mobile list did not retain both remaining devices." ); - await dismissNotices(page); + await dismissNotices(dialogue.page()); + return dialogue; }); console.log(`Joined devices scenario passed against CouchDB database ${dbName}.`); diff --git a/updates.md b/updates.md index a17be86e..534eac87 100644 --- a/updates.md +++ b/updates.md @@ -27,9 +27,9 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi #### Added -- We can review participating devices through **Joined devices** in Remote Configuration, the command palette, or a version Notice. +- We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. - - Advisory Notices help us find devices which last reported newer or older plug-in versions. + - Advisory Notices highlight newer or older plug-in versions detected in device records and help us check which devices may need updating. - We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. From 4f7a6b7ae9ccabcc0b8248db6edbc7142fc04af6 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 16:32:12 +0000 Subject: [PATCH 40/68] Explain compatibility costs in device review --- docs/settings.md | 2 +- src/common/messages/LiveSyncProvisionalMessages.ts | 10 ++++++---- .../LiveSyncUI/dialogues/JoinedDevicesDialog.svelte | 5 +++++ src/serviceFeatures/useJoinedDevices.ts | 4 ++-- test/e2e-obsidian/scripts/joined-devices.ts | 9 ++++++++- updates.md | 2 +- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 1a81032c..330815e1 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -240,7 +240,7 @@ Choose **Review devices** under **Joined devices** to read the device records on The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. -Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices are advisory and do not stop synchronisation. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. +Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices recommend reviewing and aligning plug-in versions across devices; they are advisory and do not stop synchronisation. The dialogue explains that even compatible versions can reduce Chunk reuse and increase storage use when files are rewritten, while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. **Forget device** removes another device's participation records after confirmation. It preserves the remote lock state and every other device. It does not delete Vault files or revoke credentials. A device can register again on an unlocked remote; on a locked remote, the existing recovery process applies. The result is checked after saving to CouchDB; if it cannot be confirmed, refresh the list and try again. CouchDB stores these records together in a local document which does not reject simultaneous updates, so changes from other devices can be overwritten. Pausing synchronisation on the other devices avoids that overlap. Object Storage uses conditional updates; if the server cannot support them, the record is retained. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 054bb7ba..40dd3d34 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -42,10 +42,10 @@ export const liveSyncProvisionalEnglishMessages = { Refresh: "Refresh", "Plug-in version": "Plug-in version", Platform: "Platform", - "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.": - "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", - "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.": - "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", + "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": + "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", + "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": + "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", Open: "Open", "Not recorded": "Not recorded", "Not accepted": "Not accepted", @@ -59,6 +59,8 @@ export const liveSyncProvisionalEnglishMessages = { "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", "Joined devices": "Joined devices", "Review devices": "Review devices", + "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.": + "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", "Forgettingâ€Ļ": "Forgettingâ€Ļ", "Forgetting the record does not delete Vault files or revoke credentials.": "Forgetting the record does not delete Vault files or revoke credentials.", diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte index 91c54b0f..67b5a04f 100644 --- a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -2,6 +2,7 @@ import { onDestroy, onMount } from "svelte"; import type { GuestDialogProps } from "@/modules/services/LiveSyncUI/svelteDialog"; import DialogHeader from "@/modules/services/LiveSyncUI/components/DialogHeader.svelte"; + import InfoNote from "@/modules/services/LiveSyncUI/components/InfoNote.svelte"; import { $msg as translate } from "@/common/translation"; import type { JoinedDeviceRow, @@ -121,6 +122,10 @@ {translate("This list shows device information recorded by the remote. It does not show whether a device is online now.")}

+ + {#if view} {#if !view.milestonePresent}

{translate("This remote does not currently have a record of joined devices.")}

diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts index c5fcc759..424f5ce9 100644 --- a/src/serviceFeatures/useJoinedDevices.ts +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -63,13 +63,13 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi message: direction === "newer" ? $msg( - "A newer plug-in version was detected in other device records: ${versions}. We recommend checking whether this device needs updating.", + "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", { versions: versions.join(", "), } ) : $msg( - "An older plug-in version was detected in other device records: ${versions}. We recommend checking whether those devices need updating.", + "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", { versions: versions.join(", "), } diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index 15d496e8..9cca9011 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -213,6 +213,13 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa { exact: true } ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); + await dialogue + .locator(".sls-info-note") + .getByText( + "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", + { exact: true } + ) + .waitFor({ state: "visible", timeout: uiTimeoutMs }); await dialogue .getByText("This remote is locked.", { exact: false }) .waitFor({ state: "visible", timeout: uiTimeoutMs }); @@ -409,7 +416,7 @@ async function main(): Promise { await dialoguePage .locator(".notice") .getByText( - "A newer plug-in version was detected in other device records: 999.0.0. We recommend checking whether this device needs updating.", + "A newer plug-in version was detected in other device records: 999.0.0. Reviewing and aligning the plug-in versions across devices is recommended.", { exact: true } ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); diff --git a/updates.md b/updates.md index 534eac87..161bb640 100644 --- a/updates.md +++ b/updates.md @@ -29,7 +29,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. - - Advisory Notices highlight newer or older plug-in versions detected in device records and help us check which devices may need updating. + - Advisory Notices highlight newer or older plug-in versions detected in device records and recommend reviewing and aligning our versions. The dialogue explains possible storage costs and P2P compatibility limits. - We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. From b40784e66f8ebfcb46af7b671218bab4dfa9d7b9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 16:41:21 +0000 Subject: [PATCH 41/68] Shorten device version compatibility guidance --- docs/settings.md | 2 +- src/common/messages/LiveSyncProvisionalMessages.ts | 4 ++-- .../services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte | 2 +- test/e2e-obsidian/scripts/joined-devices.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 330815e1..c65b2949 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -240,7 +240,7 @@ Choose **Review devices** under **Joined devices** to read the device records on The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**. -Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices recommend reviewing and aligning plug-in versions across devices; they are advisory and do not stop synchronisation. The dialogue explains that even compatible versions can reduce Chunk reuse and increase storage use when files are rewritten, while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. +Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices recommend reviewing and aligning plug-in versions across devices; they are advisory and do not stop synchronisation. The dialogue explains that synchronisation between different versions is generally safe unless the connection is explicitly blocked, but may be less efficient or use more storage. P2P synchronisation may also be unavailable. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection. **Forget device** removes another device's participation records after confirmation. It preserves the remote lock state and every other device. It does not delete Vault files or revoke credentials. A device can register again on an unlocked remote; on a locked remote, the existing recovery process applies. The result is checked after saving to CouchDB; if it cannot be confirmed, refresh the list and try again. CouchDB stores these records together in a local document which does not reject simultaneous updates, so changes from other devices can be overwritten. Pausing synchronisation on the other devices avoids that overlap. Object Storage uses conditional updates; if the server cannot support them, the record is retained. diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index 40dd3d34..c35a9c8d 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -59,8 +59,8 @@ export const liveSyncProvisionalEnglishMessages = { "Joined devices are available after database startup with an active CouchDB or Object Storage connection.", "Joined devices": "Joined devices", "Review devices": "Review devices", - "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.": - "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", + "Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.": + "Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.", "Forgettingâ€Ļ": "Forgettingâ€Ļ", "Forgetting the record does not delete Vault files or revoke credentials.": "Forgetting the record does not delete Vault files or revoke credentials.", diff --git a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte index 67b5a04f..7ae1b3ac 100644 --- a/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte +++ b/src/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte @@ -123,7 +123,7 @@

{#if view} diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index 9cca9011..b53ef035 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -216,7 +216,7 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa await dialogue .locator(".sls-info-note") .getByText( - "Even compatible plug-in versions can create different Chunks for the same file. When files are rewritten, this can reduce Chunk reuse and increase storage use while preserving file contents. P2P synchronisation may also be unavailable between incompatible versions. Reviewing and aligning the plug-in versions across devices is recommended.", + "Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.", { exact: true } ) .waitFor({ state: "visible", timeout: uiTimeoutMs }); From 1a54d21e712497b64879957ecd6b86608a021076 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 17:37:42 +0000 Subject: [PATCH 42/68] test: cover Commonlib 0.2 consumer workflows --- package.json | 1 + .../useJoinedDevices.unit.spec.ts | 56 +++++++ test/e2e-obsidian/README.md | 8 +- test/e2e-obsidian/scripts/dialog-mounts.ts | 85 +++++++--- .../scripts/hidden-file-snippet-sync.ts | 49 ++++-- test/e2e-obsidian/scripts/joined-devices.ts | 154 +++++++++++++----- test/e2e-obsidian/scripts/local-suite.ts | 3 + test/e2e-obsidian/scripts/run-focused.ts | 1 + 8 files changed, 276 insertions(+), 81 deletions(-) diff --git a/package.json b/package.json index cc9316c3..7a758866 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", "test:e2e:obsidian:joined-devices": "tsx test/e2e-obsidian/scripts/joined-devices.ts", + "test:e2e:obsidian:joined-devices-object-storage": "tsx test/e2e-obsidian/scripts/joined-devices.ts --object-storage", "test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts", "test:e2e:obsidian:database-diagnostics-couchdb": "tsx test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", diff --git a/src/serviceFeatures/useJoinedDevices.unit.spec.ts b/src/serviceFeatures/useJoinedDevices.unit.spec.ts index 7806508a..7021f87c 100644 --- a/src/serviceFeatures/useJoinedDevices.unit.spec.ts +++ b/src/serviceFeatures/useJoinedDevices.unit.spec.ts @@ -137,6 +137,62 @@ describe("joined devices service feature", () => { data.close(); await pending; }); + it.each(["dialogue cancellation", "remote switch", "unload"] as const)( + "does not send a pending forget operation after %s", + async (invalidation) => { + const data = fixture(); + const resource = { read: data.read, forget: data.forget, dispose: data.dispose }; + let resolve!: (value: typeof resource) => void; + data.createResource.mockReturnValueOnce( + new Promise((done) => { + resolve = done; + }) + ); + const { options, pending } = await open(data); + const forgetting = options.forget("other"); + const rejected = expect(forgetting).rejects.toThrow(); + await vi.waitFor(() => expect(data.createResource).toHaveBeenCalledOnce()); + + if (invalidation === "dialogue cancellation") options.cancel(); + if (invalidation === "remote switch") data.settings.couchDB_DBNAME = "new-vault"; + if (invalidation === "unload") for (const unload of data.unload) await unload(); + resolve(resource); + + await rejected; + expect(data.forget).not.toHaveBeenCalled(); + expect(data.dispose).toHaveBeenCalledOnce(); + data.close(); + await pending; + } + ); + it("discards a completed forget result after the dialogue closes and releases its resource", async () => { + const data = fixture(); + let resolve!: (value: "forgotten") => void; + data.forget.mockReturnValueOnce( + new Promise((done) => { + resolve = done; + }) + ); + const { options, pending } = await open(data); + const forgetting = options.forget("other"); + const rejected = expect(forgetting).rejects.toThrow(); + await vi.waitFor(() => expect(data.forget).toHaveBeenCalledWith("other")); + data.close(); + await pending; + resolve("forgotten"); + await rejected; + expect(data.dispose).toHaveBeenCalledOnce(); + }); + it("propagates a failed forget operation after releasing its resource", async () => { + const data = fixture(); + const failure = new Error("conditional write failed"); + data.forget.mockRejectedValueOnce(failure); + const { options, pending } = await open(data); + await expect(options.forget("other")).rejects.toBe(failure); + expect(data.dispose).toHaveBeenCalledOnce(); + data.close(); + await pending; + }); it("deduplicates advisory Notices and fences their actions after configuration changes", async () => { const data = fixture(); const { options, pending } = await open(data); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 1438a780..aa52c0c5 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -123,9 +123,9 @@ The underlying `test:e2e:obsidian:` scripts remain available for an im `configuration-encryption` needs no remote service. It applies a custom configuration key through the settings interface, restarts the same isolated Vault, and applies a change to only the custom string. It then copies the encrypted settings into a fresh profile without that key and checks masked input, wrong-input retry, cancellation without a settings write or database initialisation, re-enabling for another attempt, access to the settings tab, and restoration after a further restart. Connection, E2EE, profile, and independent ID values must remain unchanged. These checks establish ordinary Apply and restart behaviour, without simulating process termination between the device-key and settings writes. It also checks changes from custom and ASK modes to Default through the settings interface, and recovery from a wrong saved device key, including cancellation, the validated replacement key, and another restart. -`test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. +`test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The P2P setup check delays the saved device-name read, requires the name field and both continuation controls to remain disabled, then verifies the saved name and selected connection settings reach the setup workflow. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. -`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and forgets one device through the inline confirmation. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. +`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. `test:e2e:obsidian:joined-devices-object-storage` runs the same assertions against a unique Object Storage prefix, including the conditional removal supported by the test S3 service. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and verifies that cancelling the inline confirmation leaves the remote unchanged before forgetting one device. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. `test:e2e:obsidian:settings-ui` starts with a pending compatibility review and verifies the dedicated pause summary, its detailed explanation, and the explicit resume action in a temporary real Obsidian session. It captures the desktop summary and the iPhone-sized summary and detail dialogues; the mobile checks cover viewport containment, horizontal overflow, safe-area containment, and the close control's touch target. It confirms that the acknowledged internal version advances only after the review is accepted, and checks that the Change Log contains no acknowledgement control. It then enables Advanced mode and persists one numeric Advanced setting. On Obsidian before 1.13, this covers the imperative settings fallback used by the `SettingSpec` proof. On Obsidian 1.13 or later, it verifies the task-oriented landing order, the separate Synchronisation group containing Remote Configuration and Sync Settings, and the General Settings group containing Appearance, Logging, and Extra menus. It reopens and cancels onboarding from Quick Setup, opens all 14 nested settings pages, searches globally for the Advanced control, captures the landing page, search result, Advanced page, and mobile landing page, and verifies that the value remains after the settings dialogue is closed and reopened. The mobile landing check confirms that Remote Configuration is reachable without initial scrolling. Finally, it selects the Synchronisation Settings page and verifies that the deletion panel still exposes the effective 'Keep empty folder' setting without presenting the legacy `trashInsteadDelete` control, whose value no longer changes Obsidian deletion behaviour. @@ -248,9 +248,9 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:document-history-restore` creates a normal note, records a logical deletion while retaining readable chunks, and restores the deleted content through the visible Document History dialogue. It requires the action itself to create and reflect a new non-deleted successor revision, reopens the history at that successor, and captures the file picker, readable deleted revision, restored Vault file, and new successor revision. This scenario owns the ordinary-history restoration boundary; conflict resolution remains with **Inspect conflicts and file/database differences**. -`test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. +`test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. That target-pattern case keeps a normal-file allow-list for Markdown and PDF files enabled on both devices, then requires exact hidden JSON content after the receiving device includes it. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. -`test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path. +`test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database, and also runs in the complete local suite. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path. `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy. diff --git a/test/e2e-obsidian/scripts/dialog-mounts.ts b/test/e2e-obsidian/scripts/dialog-mounts.ts index 2456d9cd..60d7cf1e 100644 --- a/test/e2e-obsidian/scripts/dialog-mounts.ts +++ b/test/e2e-obsidian/scripts/dialog-mounts.ts @@ -15,6 +15,7 @@ import { captureObsidianPage, obsidianRemoteDebuggingPort, openLiveSyncSettings, + waitForVisibleObsidianDialogue, withObsidianPage as withObsidianSessionPage, } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; @@ -43,6 +44,8 @@ type DialogueMode = "desktop" | "mobile"; type DialogueRunState = { done: boolean; deviceNameRead?: boolean; + deviceNameReadPending?: boolean; + releaseDeviceNameRead?: () => void; error?: string; expected?: unknown; kind: string; @@ -140,11 +143,20 @@ async function openP2PRemoteSelectionDialogueForInspection(): Promise { kind: "p2p-compatibility-settings", done: false, deviceNameRead: false, + deviceNameReadPending: false, }; (globalThis as unknown as Record)[stateKey] = state; + const deviceNameReadGate = new Promise((resolve) => { + state.releaseDeviceNameRead = resolve; + }); config.getValue = async (key, defaultValue) => { const value = await originalGetValue.call(config, key, defaultValue); - if (key === deviceNameKey && value === deviceName) state.deviceNameRead = true; + if (key === deviceNameKey) { + state.deviceNameReadPending = true; + await deviceNameReadGate; + state.deviceNameReadPending = false; + if (value === deviceName) state.deviceNameRead = true; + } return value; }; manager.onConfirmApplySettingsFromWizard = async (settings: unknown) => { @@ -156,16 +168,17 @@ async function openP2PRemoteSelectionDialogueForInspection(): Promise { .then( (result) => { state.result = result; - state.done = true; }, (error: unknown) => { state.error = error instanceof Error ? error.message : String(error); - state.done = true; } ) .finally(() => { + state.releaseDeviceNameRead?.(); + state.releaseDeviceNameRead = undefined; manager.onConfirmApplySettingsFromWizard = originalConfirm; config.getValue = originalGetValue; + state.done = true; }); }, { @@ -177,6 +190,15 @@ async function openP2PRemoteSelectionDialogueForInspection(): Promise { }); } +async function releasePendingP2PDeviceNameRead(): Promise { + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + await page.evaluate((stateKey) => { + const state = (globalThis as unknown as Record)[stateKey]; + state?.releaseDeviceNameRead?.(); + }, dialogRunStateKey); + }); +} + async function openSetupUriDialogue(): Promise { const opened = await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { return await page.evaluate( @@ -523,9 +545,11 @@ async function verifyCouchDBSettingsDialogue(mode: DialogueMode): Promise { await openP2PRemoteSelectionDialogueForInspection(); await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - const remoteSelection = page.locator(".modal-container").filter({ - has: page.locator(".modal-title").filter({ hasText: "Choose a synchronisation remote" }), - }); + const remoteSelection = await waitForVisibleObsidianDialogue( + page, + "Choose a synchronisation remote", + uiTimeoutMs + ); await remoteSelection .locator("label") .filter({ hasText: "Peer-to-Peer (P2P)" }) @@ -541,13 +565,36 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { obsidianRemoteDebuggingPort(), "setup-p2p-compatibility-dialogue.png", async (page) => { - const modal = page.locator(".modal-container").filter({ - has: page.locator(".modal-title").filter({ hasText: "P2P Configuration" }), - }); + const modal = await waitForVisibleObsidianDialogue(page, "P2P Configuration", uiTimeoutMs); + const modalPage = modal.page(); await modal.waitFor({ state: "visible", timeout: uiTimeoutMs }); - const peerName = modal.locator('input[name="p2p-device-peer-id"]'); + const peerName = modal.getByLabel("Device Peer ID", { exact: true }); await peerName.waitFor({ state: "visible", timeout: uiTimeoutMs }); - await page.waitForFunction( + await withObsidianPage(obsidianRemoteDebuggingPort(), async (statePage) => { + await statePage.waitForFunction( + (stateKey) => + (globalThis as unknown as Record)[stateKey] + ?.deviceNameReadPending === true, + dialogRunStateKey, + { timeout: uiTimeoutMs } + ); + }); + const testAndContinue = modal.getByRole("button", { name: "Test Settings and Continue", exact: true }); + const continueAnyway = modal.getByRole("button", { name: "Continue anyway", exact: true }); + await testAndContinue.waitFor({ state: "visible", timeout: uiTimeoutMs }); + await continueAnyway.waitFor({ state: "visible", timeout: uiTimeoutMs }); + if ( + !(await peerName.isDisabled()) || + !(await testAndContinue.isDisabled()) || + !(await continueAnyway.isDisabled()) + ) { + throw new Error( + "The P2P device name and setup actions must remain disabled while the saved device name is loading." + ); + } + + await releasePendingP2PDeviceNameRead(); + await modalPage.waitForFunction( (expectedName) => { const input = document.querySelector('input[name="p2p-device-peer-id"]'); return input instanceof HTMLInputElement && !input.disabled && input.value === expectedName; @@ -562,7 +609,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { await modal.locator('input[name="p2p-password"]').fill("e2e-passphrase"); const compatibility = modal.locator("details").filter({ - has: page.locator("summary").filter({ hasText: "Connection compatibility" }), + has: modalPage.locator("summary").filter({ hasText: "Connection compatibility" }), }); await compatibility.waitFor({ state: "visible", timeout: uiTimeoutMs }); if (!(await compatibility.evaluate((element) => (element as HTMLDetailsElement).open))) { @@ -603,7 +650,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { } const advanced = modal.locator("details").filter({ - has: page.locator("summary").filter({ hasText: "Advanced Settings" }), + has: modalPage.locator("summary").filter({ hasText: "Advanced Settings" }), }); if (!(await advanced.evaluate((element) => (element as HTMLDetailsElement).open))) { await advanced.locator("summary").click({ timeout: uiTimeoutMs }); @@ -615,13 +662,13 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { if (relayOnlyElement === null || connectionPathElement === null) { throw new Error("P2P connection-path controls were removed whilst validating TURN settings."); } - await page.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { + await modalPage.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { timeout: uiTimeoutMs, }); await connectionPath.selectOption("relay"); await turnServers.fill(""); - await page.waitForFunction( + await modalPage.waitForFunction( (element) => (element as HTMLSelectElement).value === "automatic", connectionPathElement, { timeout: uiTimeoutMs } @@ -636,7 +683,7 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { .waitFor({ state: "visible", timeout: uiTimeoutMs }); await turnServers.fill("turn:turn.example.com:3478"); - await page.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { + await modalPage.waitForFunction((element) => !(element as HTMLOptionElement).disabled, relayOnlyElement, { timeout: uiTimeoutMs, }); await connectionPath.selectOption("relay"); @@ -644,12 +691,10 @@ async function verifyP2PCompatibilitySettingsDialogue(): Promise { await advanced.locator("summary").click({ timeout: uiTimeoutMs }); await compatibility.scrollIntoViewIfNeeded(); } - ); + ).finally(releasePendingP2PDeviceNameRead); await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { - const modal = page.locator(".modal-container").filter({ - has: page.locator(".modal-title").filter({ hasText: "P2P Configuration" }), - }); + const modal = await waitForVisibleObsidianDialogue(page, "P2P Configuration", uiTimeoutMs); await modal.getByRole("button", { name: "Continue anyway", exact: true }).click({ timeout: uiTimeoutMs }); await modal.waitFor({ state: "hidden", timeout: uiTimeoutMs }); }); diff --git a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts index 5079c021..19a92322 100644 --- a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts +++ b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts @@ -69,6 +69,7 @@ type RunnerContext = { cliBinary: string; couchDb: CouchDbConfig; dbName: string; + activeSessions: Set; }; async function writeVaultFile(vaultPath: string, path: string, content: string): Promise { @@ -348,6 +349,7 @@ async function startConfiguredSession( pluginData: createE2eCouchDbPluginData(couchDbSettings, hiddenFileSettings), localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), }); + context.activeSessions.add(session); await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); await assertE2eCompatibilityMarker(context.cliBinary, session.cliEnv); await configureCouchDb(context.cliBinary, session.cliEnv, couchDbSettings, hiddenFileSettings); @@ -355,6 +357,11 @@ async function startConfiguredSession( return session; } +async function stopSession(context: RunnerContext, session: ObsidianLiveSyncSession): Promise { + await session.app.stop(); + context.activeSessions.delete(session); +} + async function uploadHiddenFile( context: RunnerContext, session: ObsidianLiveSyncSession, @@ -430,12 +437,12 @@ async function runCreateRoundTrip( throw new Error("Remote Rebuild did not declare and encrypt internal Metadata for its accepted writer."); } console.log("Remote Rebuild declared the feature and preserved encrypted Hidden File Sync Metadata."); - await session.app.stop(); + await stopSession(context, session); session = await startConfiguredSession(context, vaultB); await pullAndApplyHiddenFiles(context, session); const received = await waitForPathContent(vaultB.path, snippetPath, (content) => content === snippetContent); - await session.app.stop(); + await stopSession(context, session); assertEqual(received, snippetContent, "Hidden snippet content did not round-trip to the second vault."); console.log(`Hidden create round-trip copied ${entry.id} to the second vault.`); @@ -450,12 +457,12 @@ async function runDeleteRoundTrip( let session = await startConfiguredSession(context, vaultA); await scanHiddenStorage(context.cliBinary, session.cliEnv); await pushLocalChanges(context.cliBinary, session.cliEnv); - await session.app.stop(); + await stopSession(context, session); session = await startConfiguredSession(context, vaultB); await pullAndApplyHiddenFiles(context, session); await waitForPathDeleted(vaultB.path, snippetPath); - await session.app.stop(); + await stopSession(context, session); console.log("Hidden delete round-trip removed the snippet from the second vault."); } @@ -476,14 +483,14 @@ async function runJsonConflictRoundTrip( const mergedOnB = await waitForPathContent(vaultB.path, mergeJsonPath, (content) => hasJsonValues(content, { fromA: true, fromB: true }) ); - await session.app.stop(); + await stopSession(context, session); session = await startConfiguredSession(context, vaultA); await pullAndApplyHiddenFiles(context, session); const mergedOnA = await waitForPathContent(vaultA.path, mergeJsonPath, (content) => hasJsonValues(content, { fromA: true, fromB: true }) ); - await session.app.stop(); + await stopSession(context, session); assertEqual(mergedOnA, mergedOnB, "Merged hidden JSON content was not consistent across both vaults."); console.log("Hidden JSON conflict was automatically merged and round-tripped."); @@ -503,7 +510,7 @@ async function runJsonManualConflictResolution(context: RunnerContext, vault: Te const merged = await waitForPathContent(vault.path, manualMergeJsonPath, (content) => hasJsonValues(content, { shared: "right", fromA: true, fromB: true }) ); - await session.app.stop(); + await stopSession(context, session); const parsed = JSON.parse(merged); assertEqual(parsed.shared, "right", "Manual JSON conflict resolution did not apply the selected merged result."); @@ -517,17 +524,19 @@ async function runTargetMismatch( vaultA: TemporaryVault, vaultB: TemporaryVault ): Promise { + const selectors = { syncOnlyRegEx: String.raw`\.(md|pdf)$` }; const targetContent = JSON.stringify({ onlyA: true, targetMismatch: true }, null, 4) + "\n"; await writeVaultFile(vaultA.path, targetPath, targetContent); - let session = await startConfiguredSession(context, vaultA); + let session = await startConfiguredSession(context, vaultA, selectors); try { await uploadHiddenFile(context, session, targetPath); } finally { - await session.app.stop(); + await stopSession(context, session); } session = await startConfiguredSession(context, vaultB, { + ...selectors, syncInternalFilesTargetPatterns: "snippets", }); try { @@ -538,10 +547,11 @@ async function runTargetMismatch( "Hidden file was applied on a device where it was not a target file." ); } finally { - await session.app.stop(); + await stopSession(context, session); } session = await startConfiguredSession(context, vaultB, { + ...selectors, syncInternalFilesTargetPatterns: "", }); let received = ""; @@ -549,11 +559,17 @@ async function runTargetMismatch( await pullAndApplyHiddenFiles(context, session, { resolveConflicts: false }); received = await waitForPathContent(vaultB.path, targetPath, (content) => content === targetContent); } finally { - await session.app.stop(); + await stopSession(context, session); } - assertEqual(received, targetContent, "Hidden file was not applied after it became a target file."); - console.log("Hidden target mismatch respected per-device target patterns, then applied after enabling the target."); + assertEqual( + received, + targetContent, + "The normal file allow-list blocked a selected hidden file from the other device." + ); + console.log( + "A normal Markdown/PDF allow-list preserved hidden target exclusion and exact remote restoration after enabling the target." + ); } async function setHiddenFileNoticeFixtures(port: number, itemIds: string[], includeRestart: boolean): Promise { @@ -828,7 +844,7 @@ async function runInitialisationNoticeGrouping(context: RunnerContext, vault: Te `Hidden File Sync showed one progress Notice before settings were saved and retained it throughout initialisation. Screenshot: ${screenshotPath}` ); } finally { - await session.app.stop(); + await stopSession(context, session); } } @@ -889,7 +905,7 @@ async function runConfigurationNoticeGrouping(context: RunnerContext, vault: Tem } finally { await clearHiddenFileNoticeFixtures(port).catch(() => undefined); await setObsidianMobileTestMode(port, false, timeoutMs).catch(() => undefined); - await session.app.stop(); + await stopSession(context, session); } } @@ -904,7 +920,7 @@ async function main(): Promise { const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "hidden-roundtrip"); const vaultA = await createTemporaryVault(); const vaultB = await createTemporaryVault(); - const context: RunnerContext = { binary, cliBinary: cli.binary, couchDb, dbName }; + const context: RunnerContext = { binary, cliBinary: cli.binary, couchDb, dbName, activeSessions: new Set() }; try { await assertCouchDbReachable(couchDb); @@ -923,6 +939,7 @@ async function main(): Promise { await runInitialisationNoticeGrouping(context, vaultB); await runConfigurationNoticeGrouping(context, vaultB); } finally { + for (const session of context.activeSessions) await stopSession(context, session); await vaultA.dispose(); await vaultB.dispose(); if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index b53ef035..e3d0ccd9 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -1,3 +1,4 @@ +import { PutObjectCommand } from "@aws-sdk/client-s3"; import { mkdir } from "node:fs/promises"; import { join } from "node:path"; import { DEVICE_ID_PREFERRED, MILESTONE_DOCID, NODEINFO_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; @@ -12,12 +13,22 @@ import { putCouchDbDocument, type CouchDbDocument, } from "../runner/couchdb.ts"; +import { + createObjectStorageClient, + deleteObjectStoragePrefix, + ensureObjectStorageBucket, + loadObjectStorageConfig, + makeUniqueBucketPrefix, + readObjectStorageJson, +} from "../runner/objectStorage.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { assertEqual, assertE2eCompatibilityUnpaused, configureCouchDb, + configureObjectStorage, createE2eCouchDbPluginData, + createE2eObjectStoragePluginData, prepareRemote, waitForLiveSyncCoreReady, } from "../runner/liveSyncWorkflow.ts"; @@ -72,6 +83,73 @@ type DeviceFixture = { preferredTweaks: Record; }; +type JoinedDevicesRemote = { + label: string; + pluginData: Record; + prepare(): Promise; + configure(cliBinary: string, env: NodeJS.ProcessEnv): Promise; + read(): Promise; + write(milestone: JoinedDevicesMilestone): Promise; + cleanup(): Promise; +}; + +async function createRemoteFixture(): Promise { + const args = process.argv.slice(2); + if (args.some((argument) => argument !== "--object-storage")) { + throw new Error("Usage: joined-devices.ts [--object-storage]"); + } + if (args.includes("--object-storage")) { + const config = await loadObjectStorageConfig(); + const bucketPrefix = makeUniqueBucketPrefix("joined-devices"); + const settings = { ...config, bucketPrefix }; + const key = `${bucketPrefix}_00000000-milestone.json`; + return { + label: "Object Storage", + pluginData: createE2eObjectStoragePluginData(settings), + prepare: () => ensureObjectStorageBucket(config), + configure: (cliBinary, env) => configureObjectStorage(cliBinary, env, settings), + read: () => readObjectStorageJson(config, key), + write: async (milestone) => { + const client = createObjectStorageClient(config); + try { + await client.send( + new PutObjectCommand({ + Bucket: config.bucket, + Key: key, + Body: JSON.stringify(milestone), + ContentType: "application/json", + }) + ); + } finally { + client.destroy(); + } + }, + cleanup: () => deleteObjectStoragePrefix(config, bucketPrefix), + }; + } + const config = await loadCouchDbConfig(); + const dbName = makeUniqueDatabaseName(config.dbPrefix, "joined-devices"); + const settings = { uri: config.uri, username: config.username, password: config.password, dbName }; + return { + label: "CouchDB", + pluginData: createE2eCouchDbPluginData(settings), + prepare: async () => { + await assertCouchDbReachable(config); + await createCouchDbDatabase(config, dbName); + }, + configure: (cliBinary, env) => configureCouchDb(cliBinary, env, settings), + read: async () => { + const milestone = await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID); + assert(Boolean(milestone._rev), "The remote milestone did not have a revision."); + return milestone as JoinedDevicesMilestone; + }, + write: async (milestone) => { + await putCouchDbDocument(config, dbName, milestone); + }, + cleanup: () => deleteCouchDbDatabase(config, dbName), + }; +} + function assert(condition: boolean, message: string): void { if (!condition) throw new Error(message); } @@ -228,13 +306,9 @@ async function assertInitialDialogue(dialogue: Locator, self: SelfDeviceInfo, pa .waitFor({ state: "visible", timeout: uiTimeoutMs }); } -async function refreshRemotePlatform( - page: Page, - config: Awaited>, - dbName: string -): Promise { - const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; - await putCouchDbDocument(config, dbName, { +async function refreshRemotePlatform(page: Page, remote: JoinedDevicesRemote): Promise { + const milestone = await remote.read(); + await remote.write({ ...milestone, node_info: { ...milestone.node_info, @@ -258,6 +332,15 @@ async function showForgetConfirmation(dialogue: Locator): Promise { }); } +async function assertCancelledForget(dialogue: Locator, remote: JoinedDevicesRemote): Promise { + const before = await remote.read(); + await showForgetConfirmation(dialogue); + const confirmation = deviceCard(dialogue, targetId).locator(".forget-confirmation"); + await confirmation.getByRole("button", { name: "Cancel", exact: true }).click({ timeout: uiTimeoutMs }); + await confirmation.waitFor({ state: "detached", timeout: uiTimeoutMs }); + assertEqual(JSON.stringify(await remote.read()), JSON.stringify(before), "Cancelling Forget changed the remote."); +} + async function confirmForgetTarget(dialogue: Locator): Promise { const targetCard = deviceCard(dialogue, targetId); await targetCard.getByRole("button", { name: "Forget device", exact: true }).click({ timeout: uiTimeoutMs }); @@ -268,12 +351,8 @@ async function confirmForgetTarget(dialogue: Locator): Promise { await targetCard.waitFor({ state: "detached", timeout: uiTimeoutMs }); } -async function assertTargetForgotten( - config: Awaited>, - dbName: string, - fixture: DeviceFixture -): Promise { - const milestone = (await fetchCouchDbDocument(config, dbName, MILESTONE_DOCID)) as JoinedDevicesMilestone; +async function assertTargetForgotten(remote: JoinedDevicesRemote, fixture: DeviceFixture): Promise { + const milestone = await remote.read(); assertEqual( JSON.stringify(milestone.accepted_nodes), JSON.stringify([retainedId]), @@ -371,42 +450,32 @@ async function main(): Promise { const cli = discoverObsidianCli(); if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); - const couchDb = await loadCouchDbConfig(); - const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "joined-devices"); + const remote = await createRemoteFixture(); const vault = await createTemporaryVault(); let session: ObsidianLiveSyncSession | undefined; try { - await assertCouchDbReachable(couchDb); - await createCouchDbDatabase(couchDb, dbName); + await remote.prepare(); session = await startObsidianLiveSyncSession({ binary, cliBinary: cli.binary, vault, startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), - pluginData: createE2eCouchDbPluginData({ - uri: couchDb.uri, - username: couchDb.username, - password: couchDb.password, - dbName, - }), + pluginData: remote.pluginData, }); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); await assertE2eCompatibilityUnpaused(cli.binary, session.cliEnv, session.remoteDebuggingPort); - await configureCouchDb(cli.binary, session.cliEnv, { - uri: couchDb.uri, - username: couchDb.username, - password: couchDb.password, - dbName, - }); + await remote.configure(cli.binary, session.cliEnv); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); await prepareRemote(cli.binary, session.cliEnv); const self = await readSelfDeviceInfo(cli.binary, session.cliEnv); const fixture = createFixture(self); - const preparedMilestone = await fetchCouchDbDocument(couchDb, dbName, MILESTONE_DOCID); - assert(Boolean(preparedMilestone._rev), "Remote preparation did not create a revisioned milestone document."); - await putCouchDbDocument(couchDb, dbName, { ...fixture.milestone, _rev: preparedMilestone._rev }); + const preparedMilestone = await remote.read(); + await remote.write({ + ...fixture.milestone, + ...(preparedMilestone._rev ? { _rev: preparedMilestone._rev } : {}), + }); await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-notice.png", async (page) => { const dialogue = await openFromRemoteSettings(page); @@ -440,8 +509,9 @@ async function main(): Promise { session.remoteDebuggingPort, "joined-devices-forget-confirmation.png", async (page) => { - await refreshRemotePlatform(page, couchDb, dbName); + await refreshRemotePlatform(page, remote); const dialogue = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); + await assertCancelledForget(dialogue, remote); await showForgetConfirmation(dialogue); await deviceCard(dialogue, targetId).locator(".forget-confirmation").scrollIntoViewIfNeeded(); return dialogue; @@ -459,7 +529,7 @@ async function main(): Promise { const settings = await openLiveSyncSettings(page, uiTimeoutMs); await settings.close(); }); - await assertTargetForgotten(couchDb, dbName, fixture); + await assertTargetForgotten(remote, fixture); await setObsidianMobileTestMode(session.remoteDebuggingPort, true, uiTimeoutMs); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); @@ -485,14 +555,16 @@ async function main(): Promise { return dialogue; }); - console.log(`Joined devices scenario passed against CouchDB database ${dbName}.`); + console.log(`Joined devices scenario passed against ${remote.label}.`); } finally { - if (session) await session.app.stop(); - await vault.dispose(); - if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { - await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => { - console.warn(error instanceof Error ? error.message : error); - }); + try { + if (session) await session.app.stop(); + } finally { + try { + await vault.dispose(); + } finally { + await remote.cleanup(); + } } } } diff --git a/test/e2e-obsidian/scripts/local-suite.ts b/test/e2e-obsidian/scripts/local-suite.ts index 8453c584..ee953160 100644 --- a/test/e2e-obsidian/scripts/local-suite.ts +++ b/test/e2e-obsidian/scripts/local-suite.ts @@ -26,6 +26,7 @@ const testSteps: Step[] = [ { name: "Review Harness", args: ["run", "test:e2e:obsidian:review-harness"] }, { name: "P2P status pane", args: ["run", "test:e2e:obsidian:p2p-pane"] }, { name: "vault reflection", args: ["run", "test:e2e:obsidian:vault-reflection"] }, + { name: "joined devices with CouchDB", args: ["run", "test:e2e:obsidian:joined-devices"] }, { name: "CouchDB upload", args: ["run", "test:e2e:obsidian:couchdb-upload"] }, { name: "manual CouchDB setup workflow", @@ -35,6 +36,7 @@ const testSteps: Step[] = [ name: "CLI to real Obsidian synchronisation", args: ["run", "test:e2e:obsidian:cli-to-obsidian-sync"], }, + { name: "joined devices with Object Storage", args: ["run", "test:e2e:obsidian:joined-devices-object-storage"] }, { name: "Object Storage upload", args: ["run", "test:e2e:obsidian:minio-upload"] }, { name: "Object Storage Setup URI workflow", @@ -56,6 +58,7 @@ const testSteps: Step[] = [ { name: "startup scan", args: ["run", "test:e2e:obsidian:startup-scan"] }, { name: "provisioned Setup URI workflow", args: ["run", "test:e2e:obsidian:setup-uri-workflow"] }, { name: "two-vault synchronisation", args: ["run", "test:e2e:obsidian:two-vault-sync"] }, + { name: "hidden file selectors", args: ["run", "test:e2e:obsidian:hidden-file-selectors"] }, { name: "hidden file snippet synchronisation", args: ["run", "test:e2e:obsidian:hidden-file-snippet-sync"] }, { name: "Customisation Sync", args: ["run", "test:e2e:obsidian:customisation-sync"] }, { name: "internal Metadata Doctor", args: ["run", "test:e2e:obsidian:internal-metadata-doctor"] }, diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 78d39fb8..31a8f339 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -9,6 +9,7 @@ const focusedScenarios = new Set([ "onboarding-invitation", "dialog-mounts", "joined-devices", + "joined-devices-object-storage", "database-diagnostics", "database-diagnostics-couchdb", "conflict-dialog-policy", From 916784c5c709c08cbf58e33668894eea441de329 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 23 Oct 2026 20:18:29 +0000 Subject: [PATCH 43/68] fix: clarify device version notices --- .../messages/LiveSyncProvisionalMessages.ts | 10 +- src/serviceFeatures/useJoinedDevices.ts | 43 ++++---- .../useJoinedDevices.unit.spec.ts | 98 ++++++++++++++++++- test/e2e-obsidian/README.md | 2 +- test/e2e-obsidian/scripts/joined-devices.ts | 27 +++-- updates.md | 2 +- 6 files changed, 144 insertions(+), 38 deletions(-) diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index c35a9c8d..0747ac4f 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -42,10 +42,12 @@ export const liveSyncProvisionalEnglishMessages = { Refresh: "Refresh", "Plug-in version": "Plug-in version", Platform: "Platform", - "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": - "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", - "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.": - "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", + "Older Self-hosted LiveSync versions detected: ${versions}.": + "Older Self-hosted LiveSync versions detected: ${versions}.", + "Newer Self-hosted LiveSync versions detected: ${versions}.": + "Newer Self-hosted LiveSync versions detected: ${versions}.", + "Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.": + "Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.", Open: "Open", "Not recorded": "Not recorded", "Not accepted": "Not accepted", diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts index 424f5ce9..53b7dbc9 100644 --- a/src/serviceFeatures/useJoinedDevices.ts +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -53,6 +53,7 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi const differences = differentDeviceVersions(snapshot, ownId, version); const seen = notified.get(identity) ?? new Set(); notified.set(identity, seen); + let updated = false; for (const direction of ["newer", "older"] as const) { const versions = differences[direction]; if (versions.length === 0) continue; @@ -62,29 +63,29 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi notices.setItem(noticeGroup, direction, { message: direction === "newer" - ? $msg( - "A newer plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", - { - versions: versions.join(", "), - } - ) - : $msg( - "An older plug-in version was detected in other device records: ${versions}. Reviewing and aligning the plug-in versions across devices is recommended.", - { - versions: versions.join(", "), - } - ), - action: { - label: $msg("Review devices"), - onSelect: () => { - if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { - void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); - } - }, - }, + ? $msg("Newer Self-hosted LiveSync versions detected: ${versions}.", { + versions: versions.join(", "), + }) + : $msg("Older Self-hosted LiveSync versions detected: ${versions}.", { + versions: versions.join(", "), + }), }); - notices.finish(noticeGroup, { durationMs: 15000 }); + updated = true; } + if (!updated) return; + notices.removeItem(noticeGroup, "review"); + notices.setItem(noticeGroup, "review", { + message: $msg("Reviewing and aligning Self-hosted LiveSync versions across devices is recommended."), + action: { + label: $msg("Review devices"), + onSelect: () => { + if (!disposed && identity === remoteIdentity(services.setting.currentSettings())) { + void services.UI.commands.invoke(OPEN_JOINED_DEVICES, undefined); + } + }, + }, + }); + notices.finish(noticeGroup, { durationMs: 15000 }); }; const createOperations = (controller: AbortController): JoinedDevicesDialogOptions => { diff --git a/src/serviceFeatures/useJoinedDevices.unit.spec.ts b/src/serviceFeatures/useJoinedDevices.unit.spec.ts index 7021f87c..188afe5a 100644 --- a/src/serviceFeatures/useJoinedDevices.unit.spec.ts +++ b/src/serviceFeatures/useJoinedDevices.unit.spec.ts @@ -5,7 +5,10 @@ import { REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; import type { JoinedDevicesDialogOptions } from "./joinedDevicesDialogTypes"; vi.mock("@/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte", () => ({ default: {} })); -vi.mock("@/common/translation", () => ({ $msg: (message: string) => message })); +vi.mock("@/common/translation", () => ({ + $msg: (message: string, values: Record = {}) => + message.replace(/\$\{([^}]+)\}/g, (placeholder, key: string) => values[key] ?? placeholder), +})); import { useJoinedDevicesFeature } from "./useJoinedDevices"; import { OPEN_JOINED_DEVICES } from "./joinedDevicesCommand"; @@ -37,7 +40,7 @@ function fixture() { close = resolve; }) ); - const notices = { setItem: vi.fn(), finish: vi.fn(), hide: vi.fn() }; + const notices = { setItem: vi.fn(), removeItem: vi.fn(), finish: vi.fn(), hide: vi.fn() }; const palette: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = []; const database = { localDatabase: { get: vi.fn(async () => ({ nodeid: "self" })) } }; const host = { @@ -193,13 +196,100 @@ describe("joined devices service feature", () => { data.close(); await pending; }); + it("shares one review action across newer and older device versions", async () => { + const data = fixture(); + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [ + { id: "newer-a", accepted: true, pluginVersion: "2.0.0" }, + { id: "newer-b", accepted: true, pluginVersion: "3.0.0" }, + { id: "older-a", accepted: true, pluginVersion: "0.9.0" }, + { id: "older-b", accepted: true, pluginVersion: "0.8.0" }, + { id: "older-duplicate", accepted: true, pluginVersion: "0.8.0" }, + ], + }); + const { options, pending } = await open(data); + await options.refresh(); + const items = data.notices.setItem.mock.calls.map(([, , item]) => item); + expect(items.filter((item) => item.action)).toHaveLength(1); + const messages = items.map((item) => item.message).join("\n"); + expect(messages).toContain("Self-hosted LiveSync"); + expect(messages).toMatch(/Newer.*2\.0\.0, 3\.0\.0/); + expect(messages).toMatch(/Older.*0\.8\.0, 0\.9\.0/); + expect(messages.match(/0\.8\.0/g)).toHaveLength(1); + expect(messages.match(/Reviewing and aligning/g)).toHaveLength(1); + expect(items[items.length - 1]?.action?.label).toBe("Review devices"); + expect(data.notices.finish).toHaveBeenCalledExactlyOnceWith("joined-device-versions", { + durationMs: 15000, + }); + data.close(); + await pending; + }); + it("keeps the shared action last when another version direction arrives", async () => { + const data = fixture(); + const visible = new Map(); + data.notices.setItem.mockImplementation((_group, key, item) => visible.set(key, item)); + data.notices.removeItem.mockImplementation((_group, key) => visible.delete(key)); + const { options, pending } = await open(data); + await options.refresh(); + expect([...visible.keys()]).toEqual(["newer", "review"]); + + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [ + { id: "other", accepted: true, pluginVersion: "2.0.0" }, + { id: "older", accepted: true, pluginVersion: "0.9.0" }, + ], + }); + await options.refresh(); + expect([...visible.keys()]).toEqual(["newer", "older", "review"]); + expect([...visible.values()].filter((item) => item.action)).toHaveLength(1); + + visible.clear(); + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [ + { id: "other", accepted: true, pluginVersion: "2.0.0" }, + { id: "older", accepted: true, pluginVersion: "0.8.0" }, + ], + }); + await options.refresh(); + expect([...visible.keys()]).toEqual(["older", "review"]); + expect([...visible.values()].filter((item) => item.action)).toHaveLength(1); + data.close(); + await pending; + }); + it("does not show a review action when no different version is reported", async () => { + const data = fixture(); + data.read.mockResolvedValue({ + milestonePresent: true, + locked: false, + cleaned: false, + devices: [{ id: "other", accepted: true, pluginVersion: "1.0.0" }], + }); + const { options, pending } = await open(data); + await options.refresh(); + expect(data.notices.setItem).not.toHaveBeenCalled(); + expect(data.notices.removeItem).not.toHaveBeenCalled(); + expect(data.notices.finish).not.toHaveBeenCalled(); + data.close(); + await pending; + }); it("deduplicates advisory Notices and fences their actions after configuration changes", async () => { const data = fixture(); const { options, pending } = await open(data); await options.refresh(); await options.refresh(); - expect(data.notices.setItem).toHaveBeenCalledTimes(1); - const action = data.notices.setItem.mock.calls[0][2].action; + expect(data.notices.setItem).toHaveBeenCalledTimes(2); + const actions = data.notices.setItem.mock.calls.map(([, , item]) => item.action).filter(Boolean); + expect(actions).toHaveLength(1); + const action = actions[0]; data.close(); await pending; data.settings.couchDB_DBNAME = "new-vault"; diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index aa52c0c5..5f04c8c8 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -125,7 +125,7 @@ The underlying `test:e2e:obsidian:` scripts remain available for an im `test:e2e:obsidian:dialog-mounts` starts a temporary real Obsidian session and exercises remote selection and CouchDB settings through `SetupManager`, plus Setup URI entry through the registered command. It verifies the compatibility pause and remote-size review, the distinction between a central data-storage server and P2P signalling, the explicit tested and untested CouchDB save actions, the internal-API warning, the Setup URI controls, automatic adjustment when differences are limited to compatible chunk settings, and both manual configuration-mismatch routes. The P2P setup check delays the saved device-name read, requires the name field and both continuation controls to remain disabled, then verifies the saved name and selected connection settings reach the setup workflow. The same session opens the live log and generated full report, reaches the `Hatch` recovery controls, writes and removes its own persistent log, and runs the missing-chunk recreation and file-verification actions against the empty disposable Vault. It captures representative desktop and mobile dialogues, checks the mobile layout and vertically stacked actions, closes each route through its normal controls, and verifies that each mounted operation settles without an error. It does not apply a remote configuration, contact a remote service, or claim to repair a deliberately damaged database. -`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. `test:e2e:obsidian:joined-devices-object-storage` runs the same assertions against a unique Object Storage prefix, including the conditional removal supported by the test S3 service. It checks that the current device appears first with current metadata, reviews locked-out and other-device records, confirms a manual refresh reads an updated remote record, and verifies that cancelling the inline confirmation leaves the remote unchanged before forgetting one device. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. +`test:e2e:obsidian:joined-devices` uses a temporary Vault and isolated CouchDB database to open Joined devices from Remote Configuration, a version Notice, and its registered command. `test:e2e:obsidian:joined-devices-object-storage` runs the same assertions against a unique Object Storage prefix, including the conditional removal supported by the test S3 service. It checks that newer and older Self-hosted LiveSync versions share one Notice with one recommendation and review action, that the current device appears first with current metadata, and that locked-out and other-device records are shown. It confirms that a manual refresh reads an updated remote record and that cancelling the inline confirmation leaves the remote unchanged before forgetting one device. It verifies the raw milestone retains the other device, remote state, and preferred-ID tweaks, then checks mobile scrolling, safe-area bounds, and touch targets. It captures the version Notice, desktop list, forget confirmation, completed removal, and mobile dialogue; unrelated Notices are dismissed before the list captures. `test:e2e:obsidian:settings-ui` starts with a pending compatibility review and verifies the dedicated pause summary, its detailed explanation, and the explicit resume action in a temporary real Obsidian session. It captures the desktop summary and the iPhone-sized summary and detail dialogues; the mobile checks cover viewport containment, horizontal overflow, safe-area containment, and the close control's touch target. It confirms that the acknowledged internal version advances only after the review is accepted, and checks that the Change Log contains no acknowledgement control. It then enables Advanced mode and persists one numeric Advanced setting. On Obsidian before 1.13, this covers the imperative settings fallback used by the `SettingSpec` proof. On Obsidian 1.13 or later, it verifies the task-oriented landing order, the separate Synchronisation group containing Remote Configuration and Sync Settings, and the General Settings group containing Appearance, Logging, and Extra menus. It reopens and cancels onboarding from Quick Setup, opens all 14 nested settings pages, searches globally for the Advanced control, captures the landing page, search result, Advanced page, and mobile landing page, and verifies that the value remains after the settings dialogue is closed and reopened. The mobile landing check confirms that Remote Configuration is reachable without initial scrolling. Finally, it selects the Synchronisation Settings page and verifies that the deletion panel still exposes the effective 'Keep empty folder' setting without presenting the legacy `trashInsteadDelete` control, whose value no longer changes Obsidian deletion behaviour. diff --git a/test/e2e-obsidian/scripts/joined-devices.ts b/test/e2e-obsidian/scripts/joined-devices.ts index e3d0ccd9..ee9ca83c 100644 --- a/test/e2e-obsidian/scripts/joined-devices.ts +++ b/test/e2e-obsidian/scripts/joined-devices.ts @@ -482,13 +482,27 @@ async function main(): Promise { const dialoguePage = dialogue.page(); await dialoguePage.setViewportSize({ width: 1280, height: 1100 }); await assertInitialDialogue(dialogue, self, dialoguePage); - await dialoguePage - .locator(".notice") - .getByText( - "A newer plug-in version was detected in other device records: 999.0.0. Reviewing and aligning the plug-in versions across devices is recommended.", - { exact: true } - ) + const notice = dialoguePage.locator(".notice").filter({ + hasText: "Newer Self-hosted LiveSync versions detected: 999.0.0.", + }); + await notice.waitFor({ state: "visible", timeout: uiTimeoutMs }); + await notice + .getByText("Older Self-hosted LiveSync versions detected: 0.1.0.", { exact: true }) .waitFor({ state: "visible", timeout: uiTimeoutMs }); + assertEqual( + await notice + .getByText("Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.", { + exact: true, + }) + .count(), + 1, + "The version Notice did not contain exactly one recommendation." + ); + assertEqual( + await notice.getByRole("button", { name: "Review devices", exact: true }).count(), + 1, + "The version Notice did not contain exactly one shared review action." + ); return dialogue; }); await captureJoinedDevices(session.remoteDebuggingPort, "joined-devices-desktop.png", async (page) => { @@ -498,7 +512,6 @@ async function main(): Promise { .page() .locator(".notice") .getByRole("button", { name: "Review devices", exact: true }) - .first() .click(); const reopened = await waitForVisibleObsidianDialogue(page, "Joined devices", uiTimeoutMs); await reopened.getByText(targetId, { exact: true }).waitFor({ state: "visible", timeout: uiTimeoutMs }); diff --git a/updates.md b/updates.md index 161bb640..ce3527fd 100644 --- a/updates.md +++ b/updates.md @@ -29,7 +29,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. - - Advisory Notices highlight newer or older plug-in versions detected in device records and recommend reviewing and aligning our versions. The dialogue explains possible storage costs and P2P compatibility limits. + - An advisory Notice identifies Self-hosted LiveSync and lists newer and older versions separately, with one shared recommendation and **Review devices** action. The dialogue explains possible storage costs and P2P compatibility limits. - We can forget records belonging to lost or retired devices while preserving the remote lock state and other devices. - Forgetting a record does not delete Vault files or revoke credentials. A returning device can register again on an unlocked remote. From 73a14d4e4b0b414a1acfe170d01500abb3dca2e9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 24 Oct 2026 02:41:12 +0000 Subject: [PATCH 44/68] Releasing 1.0.40 --- docs/releases/1.0.md | 24 +++++++++++++++++ manifest.json | 2 +- package-lock.json | 10 +++---- package.json | 2 +- src/apps/cli/package.json | 2 +- src/apps/webapp/package.json | 2 +- src/apps/webpeer/package.json | 2 +- updates.md | 49 ++++++++++++----------------------- versions.json | 3 ++- 9 files changed, 52 insertions(+), 44 deletions(-) diff --git a/docs/releases/1.0.md b/docs/releases/1.0.md index 96ce8a6b..88bd667f 100644 --- a/docs/releases/1.0.md +++ b/docs/releases/1.0.md @@ -2,6 +2,30 @@ This document contains earlier published releases from the 1.0 line of the [current Self-hosted LiveSync release history](../../updates.md). Beta and release-candidate builds published before 1.0.0 are recorded in the [1.0 preview history](1.0-previews.md). Earlier release lines continue in the [0.25 history](0.25.md) and the [legacy history](legacy.md). +## 1.0.35 + +5th October, 2026 + +### Synchronisation + +#### Fixed + +- We can continue synchronising after rebuilding an encrypted CouchDB remote, including when Hidden File Sync and Customisation Sync are enabled. (#1234) + - This plug-in reads the rebuilt data without reusing HTTP responses encrypted with the previous Security Seed. + +### Customisation Sync + +#### Fixed + +- Customisation Sync now waits until this plug-in is ready before handling configuration file events, and applying settings while synchronisation is suspended no longer scans configuration files. (#1220) + - Resuming synchronisation scans once and restarts the periodic scan when enabled. + +### Acknowledgements + +Thank you for your contributions! + +- @kimjansheden (#1220) + ## 1.0.34 2nd October, 2026 diff --git a/manifest.json b/manifest.json index f12a57e2..267d35b5 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.39", + "version": "1.0.40", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index 4cf6a3e2..211e8c3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.39", + "version": "1.0.40", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.39", + "version": "1.0.40", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -11408,7 +11408,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.39-cli", + "version": "1.0.40-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11433,7 +11433,7 @@ }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.39-webapp", + "version": "1.0.40-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11445,7 +11445,7 @@ } }, "src/apps/webpeer": { - "version": "1.0.39-webpeer", + "version": "1.0.40-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, diff --git a/package.json b/package.json index 7a758866..cd383dde 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.39", + "version": "1.0.40", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 367e9fce..e8d7ca9c 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.39-cli", + "version": "1.0.40-cli", "main": "dist/index.cjs", "type": "module", "scripts": { diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index 43800d07..487af8bf 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.39-webapp", + "version": "1.0.40-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index dbe5c64e..22594247 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.39-webpeer", + "version": "1.0.40-webpeer", "type": "module", "scripts": { "dev": "vite", diff --git a/updates.md b/updates.md index ce3527fd..cefb2bce 100644 --- a/updates.md +++ b/updates.md @@ -12,20 +12,15 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased -### CLI +## 1.0.40 -- Unreadable or malformed device-local state now stops initialisation instead of being treated as empty. Existing valid state files keep their format. +24th October, 2026 -### Hidden File Sync - -#### Fixed - -- We can synchronise selected hidden files while the normal-file allow-list is set. (#1272) - - Normal-file exclusion patterns still apply to hidden files, and the Selector descriptions now explain their scope. +This has turned into a slightly larger set of changes overall, but I have finally been able to implement the features I felt we would need. ### Synchronisation -#### Added +#### New Feature - We can choose **Review devices** in Remote Configuration, the command palette, or a version Notice to review participating devices. - Our device appears first, with its lockout state. Other records show the platform, plug-in version, app version, and last recorded connection time when available. @@ -45,8 +40,20 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi #### Changed - New setup uses the same Chunk-size recommendation of 0 across remote types, and Setting Doctor no longer recommends increasing every self-hosted V3 configuration to 60. + - Setting Doctor no longer flags the saved **Enhance chunk size** value. When rebuilding with V3 Rabin–Karp, we recommend setting it to 0. This is now an efficient choice across all supported remote types. - Existing explicit and agreed values are retained. Smaller Chunks favour reuse during editing, but increase document counts and can make first uploads and large rebuilds slower. Mostly immutable binary Vaults can still benefit from a larger explicit value. +### Hidden File Sync + +#### Fixed + +- We can synchronise selected hidden files while the normal-file allow-list is set. (#1272) + - Normal-file exclusion patterns still apply to hidden files, and the Selector descriptions now explain their scope. + +### CLI + +- Unreadable or malformed device-local state now stops initialisation instead of being treated as empty. Existing valid state files keep their format. + ## 1.0.39 20th October, 2026 @@ -161,27 +168,3 @@ Thank you for your contributions! - @zeedif (#1213, #1214) - @lingting0 (#1242) - -## 1.0.35 - -5th October, 2026 - -### Synchronisation - -#### Fixed - -- We can continue synchronising after rebuilding an encrypted CouchDB remote, including when Hidden File Sync and Customisation Sync are enabled. (#1234) - - This plug-in reads the rebuilt data without reusing HTTP responses encrypted with the previous Security Seed. - -### Customisation Sync - -#### Fixed - -- Customisation Sync now waits until this plug-in is ready before handling configuration file events, and applying settings while synchronisation is suspended no longer scans configuration files. (#1220) - - Resuming synchronisation scans once and restarts the periodic scan when enabled. - -### Acknowledgements - -Thank you for your contributions! - -- @kimjansheden (#1220) diff --git a/versions.json b/versions.json index 1ae45d74..19b183c0 100644 --- a/versions.json +++ b/versions.json @@ -50,5 +50,6 @@ "1.0.36": "1.7.2", "1.0.37": "1.7.2", "1.0.38": "1.7.2", - "1.0.39": "1.7.2" + "1.0.39": "1.7.2", + "1.0.40": "1.7.2" } From fd2eebdf38b0eb24196f07c2237dd74ff9ece494 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 24 Oct 2026 11:41:56 +0000 Subject: [PATCH 45/68] fix(startup): recheck sizes after reading storage --- .../startupLifecycle/incompleteDocuments.ts | 4 +++ .../incompleteDocuments.unit.spec.ts | 33 +++++++++++++++++++ updates.md | 4 +++ 3 files changed, 41 insertions(+) diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts index 0b4ee3de..42d4ee47 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.ts @@ -117,6 +117,10 @@ export async function checkIncompleteDocuments( continue; } sizeOnStorage = storageFileContent.byteLength; + // The file may have changed since stat, so recheck using the bytes read. + if (recordedSize === actualSize && sizeOnStorage === actualSize && !isConflicted) { + continue; + } const contentMatched = await isDocContentSame(doc.data, storageFileContent); errorFiles.push({ path, diff --git a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts index 2f30f12b..c67eddbc 100644 --- a/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts +++ b/src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts @@ -182,6 +182,39 @@ describe("checkIncompleteDocuments", () => { expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); }); + it("does not offer repair when all sizes agree after reading storage", async () => { + const healthy = documentFixture("settled.md", { storageContent: "abc" }); + const fixture = createDependencies(documentsFrom([healthy]), [healthy]); + vi.mocked(fixture.dependencies.storageAccess.statHidden).mockResolvedValueOnce({ ...healthy.stub.stat, size: 4 }); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("settled.md"); + expect(fixture.askSelectStringDialogue).not.toHaveBeenCalled(); + expect(fixture.storeFileToDB).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).toHaveBeenCalledWith("checkIncompleteDocs", true); + }); + + it("uses the bytes read from storage when a size mismatch remains", async () => { + const mismatched = documentFixture("changing.md", { recordedSize: 3, storageContent: "abcde" }); + const fixture = createDependencies(documentsFrom([mismatched]), [mismatched]); + vi.mocked(fixture.dependencies.storageAccess.statHidden).mockResolvedValueOnce({ + ...mismatched.stub.stat, + size: 4, + }); + fixture.askSelectStringDialogue.mockImplementation(selectButton(0)); + + await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true); + + expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("changing.md"); + expect(fixture.askSelectStringDialogue).toHaveBeenCalledOnce(); + expect(fixture.askSelectStringDialogue.mock.calls[0]?.[0]).toEqual( + expect.stringContaining("changing.md (M: 3, A: 3, S: 5)") + ); + expect(fixture.storeFileToDB).not.toHaveBeenCalled(); + expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled(); + }); + it("still reads conflicted storage for the existing repair dialogue", async () => { const conflicted = documentFixture("conflicted.md", { storageContent: "abc", conflicts: ["2-other"] }); const fixture = createDependencies(documentsFrom([conflicted]), [conflicted]); diff --git a/updates.md b/updates.md index ca2e3497..7e31a95e 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,10 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +- We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) + ## 1.0.39 20th October, 2026 From 68fa04af3c9aa8f6d971280ceaab138e917ffbd2 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 02:47:55 +0000 Subject: [PATCH 46/68] Expose explicit LiveSync batching and honour save failures Show the batching choice for the current synchronisation mode while preserving both mode preferences. Observe local save failures before file-open work and report failed asynchronous unload preparation. Record the accepted recovery and batching contract, update the primary settings documentation, and add two-device runtime coverage for batching, deletions, saved settings, and interrupted local processing. Requires Commonlib commit 72bdc66af5d7338d8f2f69ebe83c3866def93360. Update the exact registry dependency when its package version is published. Validation: full repository check; latest packed dependency type checks, production build, 1,373 unit tests (one skipped), and six focused runtime recovery cases. The earlier implementation passed the full 23-case runtime comparison and five additional interruption conditions. --- ...local_file_save_completion_and_batching.md | 356 +++++++ docs/settings.md | 17 +- src/main.ts | 9 +- .../essentialObsidian/ModuleObsidianEvents.ts | 13 +- .../ModuleObsidianEvents.unit.spec.ts | 42 + .../SettingDialogue/PaneSyncSettings.ts | 16 +- .../PaneSyncSettings.unit.spec.ts | 98 ++ .../scripts/issue1207-livesync-batch.ts | 961 ++++++++++++++++++ updates.md | 15 + 9 files changed, 1512 insertions(+), 15 deletions(-) create mode 100644 docs/adr/2026_10_local_file_save_completion_and_batching.md create mode 100644 src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/issue1207-livesync-batch.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md new file mode 100644 index 00000000..8abef973 --- /dev/null +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -0,0 +1,356 @@ +--- +date: 2026-10-24 +commonlib-version: "0.2.0" +self-hosted-livesync-version: "1.0.40" +status: accepted-unreleased +--- + +# Architectural Decision Record: Local File Save Completion and Batching + +## Status + +Accepted on 2026-10-25. The package versions above identify the investigated +baseline, rather than a released implementation of this decision. + +## Context + +[Issue #1207](https://github.com/vrtmrz/obsidian-livesync/issues/1207) includes +conflicts around automated note creation, editing, and renaming. Batching local +file updates can reduce the intermediate revisions which another device receives. +It also extends the period during which the Vault contains an edit which the +local database has not saved. Recovery and incoming-file protection must remain +correct throughout that period. + +Two-device Obsidian E2E tests using Self-hosted LiveSync 1.0.40 and Commonlib +0.2.0 found that a test-only removal of the LiveSync mode restriction reduced +six consecutive edits from six additional revisions to one, in both directions. +Incoming identical content produced no extra conflict, while genuinely different +edits or a deletion preserved the necessary branches. A synthetic +create–edit–rename sequence still delivered intermediate content. These tests +used Vault operations, without Templater or Modal Forms themselves. + +Restart tests exposed a separate recovery gap: + +1. The event manager removes an event from its waiting map when its timer is + released. It also removes a buffered event before dispatching it. Its saved + snapshot covers buffered and waiting events, but omits work in progress. +2. `waitForIdle()` waits for the waiting gates to open, rather than for the file + handlers to finish. `ServiceFileAccessBase` reports that pending events have + been committed after this gate-only wait. Some handler failures also settle + without propagating a failure to that caller. +3. If the process stops before the database write, the file can retain the new + content while the database retains the old content and the saved event + snapshot is empty. +4. At start-up, the Offline Scanner's `EVEN` branch tries to record missing + provenance, then reports completion. It ignores that helper's Boolean result. + Different content can consequently remain unsaved when both modification + times fall in the same two-second comparison bucket. + +The last condition was reproduced by pausing the target operation immediately +before database processing, then restarting the same isolated profile. A +different timestamp bucket recovered the edit; the same bucket did not. The +changed-condition success demonstrates the boundary, rather than a fix. This +does not establish the cause of the reporter's historical conflicts. + +The runtime evidence covers Linux aarch64, Obsidian 1.12.7, and CouchDB 3.5.0. +Windows, mobile lifecycles, and the reporter's full automation remain outside +that evidence. The inspected Commonlib source is commit +`aa54282937c20d3db7f43778d240feeb1586cced`. + +## Decision + +Make local file processing recoverable until its result is established, use +content and provenance to reconcile ambiguous timestamps, and enable batching +in LiveSync mode only through an explicit choice after those contracts pass +verification. Apply the recovery contract with batching enabled or disabled. + +The content-reconciliation change targets selected ordinary files in the +Offline Scanner's `NEWER_WINS` / `sync-newer` action. Keep the existing action +matrix, explicit `DB_APPLY` policy, selection and size limits, conflict guards, +and recovery suspensions. Hidden and internal files retain their owning +feature's recovery contract; an ordinary file scan cannot recover those paths +on its behalf. + +### 1. Distinguish local persistence from replication + +The following boundaries have different meanings: + +| Boundary | Meaning | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Batch wait released | The file operation may start. Its content can still be absent from the local database. | +| Local processing completed | The intended change is represented durably by the local database and its required content, or has been proved already represented or safely superseded. | +| Replication completed | A finite replication reports its own outcome. A local save alone does not prove remote receipt or reflection into another Vault. | + +Ordinary file saving by Obsidian remains independent of database saving. Pending +and processing indicators must include work in progress and unresolved failures, +so an empty timer map cannot imply that local processing is complete. + +### 2. Keep a recoverable record through processing + +The event manager owns each operation from queue admission through completion. +Its existing persistent snapshot must represent buffered, waiting, and +in-progress work in an order which preserves deletion and rename dependencies. +Moving work between those states must not create a persistent gap. + +An event can leave the recoverable set only after its effect is durable, current +state proves that the effect is already represented, or a durably recorded +successor safely subsumes it. Finishing an older operation must not remove a +newer operation for the same path. Cancellation of a batch timer alone is not +evidence that its content is safe to forget. + +Snapshot writes must preserve mutation order. An older asynchronous write must +not replace a newer snapshot, and a failed snapshot write must remain visible +to the lifecycle caller. Reuse the existing device-local snapshot store; a +second journal and a transaction spanning the file system and database are not +part of this proposal. + +The first implementation should retain the existing event snapshot format, +including event keys and rename intent, and add pending ownership within the +event manager. The persistence adapter remains responsible for storing the +ordered snapshots. Handler outcomes must distinguish a failed attempt from a +policy-approved skip before the manager can acknowledge either safely. + +A crash after the database write but before snapshot removal can replay an +operation. Replay must check current storage and revision provenance, retain +the existing protection against stale restored events, and avoid duplicate +revisions or deletion of a newer same-path file. It must be safe to interrupt +replay again. Existing saved snapshots remain readable. + +Boolean failure, an exception, and an unavailable comparison are not successful +completion. Keep unresolved work recoverable and report the failure through the +existing service and diagnostic boundaries. Explicit exclusions or a verified +superseding change may finish work without a new revision; a generic handler +failure may not. Recovery may continue with independent paths, while retaining +the failed path and its dependencies. This proposal adds no unbounded retry loop. + +The recoverable snapshot is an operation record, not a backup of every transient +editor buffer. File content already saved by the host remains available for +start-up reconciliation if the process stops before the first snapshot write. +Crash guarantees cannot include bytes or events which the host never persisted +or delivered, or repair arbitrary storage failure. + +### 3. Give flush a finite, truthful completion boundary + +An explicit commit of pending file events captures the work accepted before +that request, releases its batch delays, and waits for the actual processing +results, including buffered and already-running work. Later edits cannot extend +that boundary indefinitely. A newer coalesced operation may satisfy earlier work +only when its successful result covers that work. + +The internal ordering barrier before a delete or rename waits only for its +predecessors. It must not wait for itself or later events in the same dispatch +loop. Replacing every gate wait with a wait for global idleness would risk a +dependency cycle or starvation. + +Manual synchronisation and replication readiness use the completed local +boundary. Settings application must observe its result before reporting success +or resuming work which depends on it. Persisting a settings file is a separate +operation: `saveSettingData()` returning is not a file-processing completion +signal, and making every settings-saved notification synchronous is unnecessary. + +When a host permits orderly shutdown, stop admitting watcher work before +draining accepted operations or preserving their pending records. Keep their +storage available until that work settles. Suspension, forced termination, and +Obsidian's synchronous unload callback cannot guarantee that asynchronous work +finishes; correctness must also follow from the persistent record and start-up +reconciliation. Remote availability must not determine whether local pending +work survives shutdown. + +### 4. Reconcile content when timestamps are inconclusive + +Retain the existing timestamp tolerance for platforms with coarse modification +times. Timestamp equality, including exactly equal raw timestamps, does not +prove content equality. The Offline Scanner must establish one of the following +outcomes through the ordinary non-forced file handler and its provenance rules: + +| Evidence | Required outcome | +| ----------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Vault content already matches an eligible current revision | Record or retain valid provenance as appropriate; create no duplicate revision. | +| Vault content matches its recorded revision or saved content evidence, while the current database revision has advanced | Apply the database state through the existing protected reflection path. Do not resurrect the unchanged older content as an edit. | +| Vault content differs from its recorded revision | Preserve the edit on that recorded branch. Keep independent concurrent branches available for conflict resolution. | +| Vault content has no provable origin and matches no current live leaf | Preserve it according to the existing independent-branch rules. Do not attach it to an arbitrary database winner. | +| Required content or provenance cannot be read, or state changes during comparison | Retain unresolved work and report or re-evaluate it. Do not mark it reconciled merely because the timestamp comparison is `EVEN`. | + +Revalidate the relevant file and database state before acting on a comparison. +Reuse the existing content and provenance checks; avoid a separate conflict +algorithm in the scanner. Metadata continues to reference Chunks containing the +file content. + +`tryRecordUntrackedFileRevision()` answers whether it could safely record +provenance. Its `false` also covers an already-populated record, conflicting +history, unavailable content, and a concurrent change. It is not an 'edited +file' predicate. A fallback which treats every `false` as permission for a +forced save would violate the required outcomes. + +Propagate the file handler's actual outcome into the pair result, including for +storage-to-database writes. Preserve the scanner's `completed`, `skipped`, and +`failed` distinctions and its existing `continueOnFileFailure` policy. Ordinary +Obsidian start-up can report `completed-with-file-failures` and allow unaffected +files to proceed; an unresolved pair must not become `completed` to achieve +that result. Keep the stricter callers' existing completion requirements. + +This deliberately extends the current equal-time scanner contract. Existing +provenance tests expect no database write during that scan, including for an +edited file. Preserve their unchanged-file and later incoming-protection +scenarios; change the edited-file expectation to require preservation during +the scan itself. Test both identical timestamps and distinct raw timestamps +within one comparison bucket. + +### 5. Make batch timing and activation explicit + +Batching is a bounded delay before local database processing. The quiet-period +timer follows the latest eligible content update; the maximum intentional delay +starts with the first uncommitted update and is not reset by later edits or time +spent awaiting a processing slot. Once due, work must become eligible to run. +The maximum delay is not a guarantee of database completion, network delivery, +or another device's reflection time. Waiting timers should not consume the +capacity needed to process unrelated ready files. + +For ordinary local content updates, queue the changed path and operation intent +without reading the file body merely to wait. Acquire the current content when +the operation becomes due, after coalescing eligible events. Reuse that content +snapshot within the operation for comparison, provenance evidence, and Chunk +creation. This retains the current ordinary-save boundary; it does not require +reading and hashing the file for every keystroke or queued notification. + +Content reuse within one operation is distinct from keeping a body cache across +edits. A cached body would need a proven invalidation and freshness contract; +mtime and size alone cannot establish that it is current. A saved provenance +hash records evidence about earlier content, rather than replacing acquisition +of the current bytes. Restart recovery, an incoming update, or a comparison +which needs revalidation can still require additional reads. Preserve those +checks rather than making a global promise of one read per path. + +Batching therefore aims to reduce repeated content acquisition, comparison, +Chunk processing, and local database updates, as well as intermediate revisions +and subsequent replication and reflection work. Event filtering, bookkeeping, +and pending-operation persistence still occur while waiting. The revision +reduction measured by the existing E2E does not quantify file reads, CPU cost, +physical disk I/O, or network traffic; measure those costs separately. Host or +operating-system caching also means that a read API call is not a physical disk +read count. + +Manual flush and required deletion or rename ordering can end a batch early. +Ordinary incoming replication continues, using the existing protection for +unsaved local content. It must not require all unrelated local batches to flush +for every incoming document. + +Add an independent `batchSaveInLiveSync` Boolean, defaulting to `false` +for both existing settings and new Vaults. In LiveSync mode, this value selects +batching; other modes continue to use `batchSave`. Both choices use the existing +minimum and maximum delay settings. The interface presents the choice for the +current mode. An ordinary mode switch preserves each mode's choice. + +This adds one persisted setting to distinguish a new explicit choice from old +`liveSync: true, batchSave: true` configurations, where batching was ignored. +Neither upgrade nor import of that legacy pair may activate batching. An older +client continues to apply its existing LiveSync restriction. Include the new +property in Commonlib's setting policy and import/export tests; do not infer +consent from the presence of the old `batchSave` property. + +The setting and scheduling changes follow the recovery work. Recovery in the +currently supported modes must pass verification before the new mode is enabled. + +### 6. Keep batching separate from automation completion + +The intended benefit is fewer short-lived intermediate revisions. Batching +cannot discover when a third-party template has finished, particularly while a +dialogue is open or another device also runs automation. A maximum delay or a +rename boundary can legitimately publish intermediate content. + +Coalescing `CREATE` followed by `CHANGED` for the same still-existing file is a +useful follow-up, but it needs its own ordering and interruption tests. Preserve +the existing cross-path rename rule: store the destination before deleting the +proven source branch. Do not retarget or discard pending source operations +without proving the effect is already covered. + +A guarantee that a multi-step template appears remotely only after completion +would need an explicit producer completion contract and its failure behaviour. +That is a separate design. Enabling batching alone must not be described as +eliminating genuine concurrent-edit conflicts or as a confirmed fix for #1207. + +## Ownership and implementation order + +1. **Commonlib recovery:** event ownership, ordered snapshot persistence, + successful completion and failure propagation, finite flush boundaries, and + same-time content reconciliation. Add focused failing regressions before + changing production code. Reuse the existing event manager, file handler, + scanner, and provenance store. +2. **Host integration:** keep the Obsidian persistence adapter and lifecycle + wiring aligned with those contracts. Verify callers of + `commitPendingFileEvents`, including settings application and database + preparation, and preserve their failure outcomes. Validate the packed + Commonlib artefact and update the consumer's exact dependency. +3. **LiveSync mode batching:** add the explicit setting and its primary-language + presentation only after the recovery regressions pass. Establish the stated + timing semantics and repeat the two-device comparisons with the actual + package, including start-up with batching already enabled. +4. **Further coalescing:** assess same-path creation/editing and any proposed + rename optimisation separately, against the original automation problem. + +Keep these changes independently reviewable. Shared behaviour belongs in +Commonlib; the plug-in must not acquire a source mirror or patch installed +package files. Snapshot format changes, a second persistent store, or a new +automation transaction API require a further design decision. + +## Verification required + +Use controlled promises at the manager and persistence boundaries for focused +regressions, and real PouchDB tests for content and ancestry. The acceptance +matrix is: + +| Scenario | Required observation | +| -------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Stop after dequeue, timer release, or during a database write | Pending work remains recoverable; restart eventually represents the saved Vault edit. | +| Database write succeeds before snapshot cleanup; restart twice | Replay creates no duplicate revision and preserves later edits. | +| Old and new events share a path; snapshot writes overlap | Completing old work cannot remove new work or regress the durable snapshot. | +| Handler returns false, throws, or snapshot persistence fails | Flush reports failure; pending work is retained; unrelated recoverable work can continue. | +| Buffered or in-progress work exists with no waiting timer | Flush still waits for actual completion. | +| A flush overlaps new typing or a rename/delete barrier | The captured work completes without waiting for later work or deadlocking. | +| Equal timestamps: unchanged, edited, untracked, pruned provenance, unavailable Chunks, or concurrent recheck changes | Preserve correct ancestry and bytes, avoid duplicate revisions, and distinguish unresolved comparisons from successful reconciliation. | +| Settings save followed by stop; ordinary unload; interruption during replay | No operation is forgotten because a settings callback or shutdown released its timer. | +| Sustained edits and more paths than available processing slots | Observe the intentional-delay deadline and progress of unrelated ready files. | +| Old settings, mode changes, explicit opt-in, and settings import | Existing LiveSync mode users retain immediate saving until they choose batching. | +| Incoming identical content, an edit, a deletion, or delete/recreate while waiting | Preserve genuine branches without extra conflicts or loss of local work. | + +Keep the existing E2E comparison between batching disabled, an ignored legacy +setting, and explicitly enabled batching. Repeat both directions, manual +synchronisation, saved-snapshot restart, and controlled in-progress restart with +the fix. Record the original failure separately from repaired results; a pass +with a different timestamp bucket or a test-only bundle is not verification of +the production fix. Run platform-specific lifecycle checks before extending the +evidence to Windows or mobile. + +## Alternatives and consequences + +- **Remove only the LiveSync guard:** leaves recovery gaps and can activate a + previously ignored setting. The explicit choice follows correctness work. +- **Wait only at unload:** cannot cover abrupt termination and hosts which do + not await asynchronous unload. Continuous recoverability is still required. +- **Repair only the scanner:** improves surviving-file recovery, but does not + establish event ordering, deletion intent, or truthful flush completion. +- **Retain only in-progress events:** closes a persistence gap, but still needs + success/failure handling, replay checks, and recovery before the first snapshot. +- **Remove timestamp rounding or force the Vault copy into the current winner:** + fails for exactly equal timestamps and can destroy branch intent. Content and + provenance determine the safe action. + +Keeping work recoverable adds snapshot bookkeeping and can make local flush +wait longer because it now waits for real processing. Same-time reconciliation +can require additional content reads. Batching trades prompt remote visibility +for fewer database updates and may increase the window for genuine concurrent +edits. Measure these costs without weakening the content-preservation contract. + +## References + +- [Conflict resolution and revision provenance](../specs_conflict_resolution.md) +- [Batch database update settings](../settings.md#batch-database-update) +- [Common library package boundary](2026_07_common_library_package_boundary.md) +- [Fast Fetch persistence and completion](2026_08_fast_fetch_persistence_and_completion.md) +- [Two-device Batch save reproduction](../../test/e2e-obsidian/scripts/issue1207-livesync-batch.ts) +- [Commonlib event manager at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/managers/StorageEventManager.ts) +- [Commonlib Offline Scanner at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceFeatures/offlineScanner.ts) +- [Commonlib file handler at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceModules/ServiceFileHandlerBase.ts) +- [Commonlib scanner policy and stored event interpretation](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/docs/offline-scanner.md) +- [Commonlib event admission and reflection contract](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/docs/storage-events-and-reflection.md) +- [Existing equal-time provenance regression](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceFeatures/offlineScanner.provenance.unit.spec.ts) diff --git a/docs/settings.md b/docs/settings.md index c65b2949..b7592846 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -669,17 +669,28 @@ Setup URIs preserve both sleep preferences. Older Setup URIs which do not contai #### Batch database update Setting key: batchSave -Reducing the frequency with which on-disk changes are reflected into the DB + +Combine repeated local file updates before saving them to the local database in modes other than LiveSync mode. This reduces intermediate database revisions and delays their availability to other devices. + +#### Batch database updates in LiveSync mode + +Setting key: batchSaveInLiveSync + +Apply the same bounded delay in LiveSync mode. This separate choice is disabled by default for both existing settings and new Vaults. An older configuration containing `liveSync: true` and `batchSave: true` does not enable it. The settings interface shows the choice for the current mode, and switching modes preserves both choices. Older clients continue to ignore batching in LiveSync mode. + +Both choices share the delays below. Manual synchronisation, deletion, or renaming can end a batch early. Batching reduces intermediate updates; it does not identify when a template has finished or prevent genuine concurrent edits on different devices. #### Minimum delay for batch database updating Setting key: batchSaveMinimumDelay -Seconds. Saving to the local database will be delayed until this value after we stop typing or saving. + +Seconds of quiet after the latest eligible edit before saving to the local database, subject to the maximum delay below. #### Maximum delay for batch database updating Setting key: batchSaveMaximumDelay -Saving will be performed forcefully after this number of seconds. + +Maximum intentional delay in seconds from the first pending update to a file. Later edits do not restart this limit, and waiting for a processing slot does not add another batch delay. This limit makes the work eligible to run; it does not guarantee database completion or receipt by another device within that time. ### 4. Deletion Propagation (Advanced) diff --git a/src/main.ts b/src/main.ts index 8aebf8ad..d4e7d346 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,4 +1,5 @@ import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation"; +import { LOG_LEVEL_URGENT } from "octagonal-wheels/common/logger"; import { getLanguage, Notice, Plugin, type App, type PluginManifest } from "./deps"; import { setGetLanguage } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; setGetLanguage(getLanguage); @@ -233,6 +234,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin { void this._startUp(); } override onunload(): void { - return void this.core.services.control.onUnload(); + void this.core.services.control.onUnload().catch((error) => { + this.core.services.API.addLog( + "Failed to unload Self-hosted LiveSync: " + String(error), + LOG_LEVEL_URGENT, + "shutdown" + ); + }); } } diff --git a/src/modules/essentialObsidian/ModuleObsidianEvents.ts b/src/modules/essentialObsidian/ModuleObsidianEvents.ts index 5f689bc6..10001ca2 100644 --- a/src/modules/essentialObsidian/ModuleObsidianEvents.ts +++ b/src/modules/essentialObsidian/ModuleObsidianEvents.ts @@ -230,7 +230,13 @@ export class ModuleObsidianEvents extends AbstractObsidianModule { } this.isLastHidden = isHidden; - await this.services.fileProcessing.commitPendingFileEvents(); + const pendingFileEventsCommitted = await this.services.fileProcessing.commitPendingFileEvents(); + if (!pendingFileEventsCommitted) { + this._log( + "Pending file events remain because their commit did not complete before the window visibility changed.", + LOG_LEVEL_NOTICE + ); + } // Desktop opt-in (LiveSync/Periodic only): keep the background channel running while the // window is hidden, instead of suspending on hide. On hide we skip the suspend for both @@ -287,7 +293,10 @@ export class ModuleObsidianEvents extends AbstractObsidianModule { if (this.settings.suspendFileWatching) return; if (!this.settings.isConfigured) return; if (!this.services.appLifecycle.isReady()) return; - await this.services.fileProcessing.commitPendingFileEvents(); + const pendingFileEventsCommitted = await this.services.fileProcessing.commitPendingFileEvents(); + if (!pendingFileEventsCommitted) { + return; + } if (file == null) { return; } diff --git a/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts b/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts index 64b92b2b..5b61464b 100644 --- a/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts +++ b/src/modules/essentialObsidian/ModuleObsidianEvents.unit.spec.ts @@ -3,6 +3,7 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { ModuleObsidianEvents } from "./ModuleObsidianEvents"; import { DEFAULT_SETTINGS, REMOTE_COUCHDB } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { reactiveSource } from "octagonal-wheels/dataobject/reactive"; +import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; type SetupOptions = { settings?: Partial; @@ -23,6 +24,8 @@ function setup(opts: SetupOptions) { onResumed: vi.fn(async () => true), }; const fileProcessing = { commitPendingFileEvents: vi.fn(async () => true) }; + const replication = { replicateUnattendedByEvent: vi.fn(async () => undefined) }; + const conflict = { queueCheckForIfOpen: vi.fn(async () => undefined) }; const boundedRemoteActivityCount = reactiveSource(0); const boundedLocalApplicationActivityCount = reactiveSource(0); @@ -39,6 +42,8 @@ function setup(opts: SetupOptions) { setting: { saveSettingData: vi.fn(async () => undefined) }, appLifecycle, fileProcessing, + replication, + conflict, replicator: { boundedRemoteActivityCount, boundedLocalApplicationActivityCount }, }, settings: { @@ -61,6 +66,9 @@ function setup(opts: SetupOptions) { module, appLifecycle, fileProcessing, + replication, + conflict, + api: core._services.API, boundedRemoteActivityCount, boundedLocalApplicationActivityCount, }; @@ -303,6 +311,23 @@ describe("watchWindowVisibilityAsync — keepReplicationActiveInBackground", () expect(appLifecycle.onResumed).toHaveBeenCalledTimes(1); }); + it("logs a failed pending-file commit and still suspends on hide", async () => { + const { module, appLifecycle, fileProcessing, api } = setup({ + settings: { keepReplicationActiveInBackground: false, liveSync: true }, + hidden: true, + }); + fileProcessing.commitPendingFileEvents.mockResolvedValue(false); + + await module.watchWindowVisibilityAsync(); + + expect(appLifecycle.onSuspending).toHaveBeenCalledTimes(1); + expect(api.addLog).toHaveBeenCalledWith( + expect.stringContaining("Pending file events remain"), + LOG_LEVEL_NOTICE, + expect.any(String) + ); + }); + it("does not apply on mobile even if the flag is set", async () => { const { module, appLifecycle } = setup({ settings: { keepReplicationActiveInBackground: true, liveSync: true }, @@ -313,3 +338,20 @@ describe("watchWindowVisibilityAsync — keepReplicationActiveInBackground", () expect(appLifecycle.onSuspending).toHaveBeenCalledTimes(1); }); }); + +describe("watchWorkspaceOpenAsync — pending file event commit", () => { + it("skips replication and conflict checks when pending file events fail to commit", async () => { + const { module, fileProcessing, replication, conflict } = setup({ + settings: { syncOnFileOpen: true }, + hidden: false, + }); + fileProcessing.commitPendingFileEvents.mockResolvedValue(false); + + await module.watchWorkspaceOpenAsync({ path: "note.md" } as Parameters< + typeof module.watchWorkspaceOpenAsync + >[0]); + + expect(replication.replicateUnattendedByEvent).not.toHaveBeenCalled(); + expect(conflict.queueCheckForIfOpen).not.toHaveBeenCalled(); + }); +}); diff --git a/src/modules/features/SettingDialogue/PaneSyncSettings.ts b/src/modules/features/SettingDialogue/PaneSyncSettings.ts index 70a4b566..76c5b1a9 100644 --- a/src/modules/features/SettingDialogue/PaneSyncSettings.ts +++ b/src/modules/features/SettingDialogue/PaneSyncSettings.ts @@ -171,20 +171,18 @@ export function paneSyncSettings( } }); - void addPanel( - paneEl, - $msg("obsidianLiveSyncSettingTab.titleUpdateThinning"), - undefined, - visibleOnly(() => !this.isConfiguredAs("syncMode", "LIVESYNC")) - ).then((paneEl) => { - new Setting(paneEl).autoWireToggle("batchSave"); + void addPanel(paneEl, $msg("obsidianLiveSyncSettingTab.titleUpdateThinning")).then((paneEl) => { + const inLiveSyncMode = () => this.isConfiguredAs("syncMode", "LIVESYNC"); + const batchEnabled = () => this.isConfiguredAs(inLiveSyncMode() ? "batchSaveInLiveSync" : "batchSave", true); + new Setting(paneEl).autoWireToggle("batchSave", { onUpdate: visibleOnly(() => !inLiveSyncMode()) }); + new Setting(paneEl).autoWireToggle("batchSaveInLiveSync", { onUpdate: visibleOnly(inLiveSyncMode) }); new Setting(paneEl).autoWireNumeric("batchSaveMinimumDelay", { acceptZero: true, - onUpdate: visibleOnly(() => this.isConfiguredAs("batchSave", true)), + onUpdate: visibleOnly(batchEnabled), }); new Setting(paneEl).autoWireNumeric("batchSaveMaximumDelay", { acceptZero: true, - onUpdate: visibleOnly(() => this.isConfiguredAs("batchSave", true)), + onUpdate: visibleOnly(batchEnabled), }); }); diff --git a/src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts b/src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts new file mode 100644 index 00000000..ca88eaee --- /dev/null +++ b/src/modules/features/SettingDialogue/PaneSyncSettings.unit.spec.ts @@ -0,0 +1,98 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab"; +import type { OnUpdateFunc, PageFunctions } from "./SettingPane"; + +const controls = vi.hoisted(() => new Map { visibility?: boolean } }>()); +vi.mock("@/common/translation", () => ({ $msg: (key: string) => key })); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/logger", () => ({ Logger: vi.fn() })); +vi.mock("./LiveSyncSetting.ts", () => ({ + LiveSyncSetting: class { + autoWireDropDown(key: string, options = {}) { + controls.set(key, options); + return this; + } + autoWireToggle(key: string, options = {}) { + controls.set(key, options); + return this; + } + autoWireNumeric(key: string, options = {}) { + controls.set(key, options); + return this; + } + }, +})); + +import { paneSyncSettings } from "./PaneSyncSettings"; + +function setup(liveSync: boolean, batchSave: boolean, batchSaveInLiveSync: boolean) { + const callbacks = new Map Promise>(); + const panels = new Map(); + const tab = { + editingSettings: { ...DEFAULT_SETTINGS, liveSync, batchSave, batchSaveInLiveSync }, + services: { API: { isMobile: () => false }, control: { applySettings: vi.fn(async () => {}) } }, + addOnSaved: (key: string, callback: (value: string) => Promise) => callbacks.set(key, callback), + isConfiguredAs(key: string, value: unknown) { + if (key === "syncMode") { + return ( + (this.editingSettings.liveSync + ? "LIVESYNC" + : this.editingSettings.periodicReplication + ? "PERIODIC" + : "ONEVENTS") === value + ); + } + return this.editingSettings[key as keyof typeof this.editingSettings] === value; + }, + saveSettings: vi.fn(async () => {}), + requestCatalogueRefresh: vi.fn(), + }; + const functions = { + addPanel: (_element: HTMLElement, title: string, _callback: unknown, onUpdate?: OnUpdateFunc) => ({ + then(callback: (element: HTMLElement) => void) { + if (!title.endsWith("titleSynchronizationMethod") && !title.endsWith("titleUpdateThinning")) return; + panels.set(title, onUpdate); + callback({} as HTMLElement); + }, + }), + } as unknown as PageFunctions; + paneSyncSettings.call(tab as unknown as ObsidianLiveSyncSettingTab, {} as HTMLElement, functions); + return { tab, callbacks, panels }; +} + +function visible(key: string): boolean { + const control = controls.get(key); + if (!control) throw new Error(`Missing setting control: ${key}`); + return control.onUpdate?.().visibility !== false; +} + +beforeEach(() => controls.clear()); + +describe("mode-specific database batching settings", () => { + it.each([ + { liveSync: true, batchSave: true, batchSaveInLiveSync: false, delays: false }, + { liveSync: true, batchSave: false, batchSaveInLiveSync: true, delays: true }, + { liveSync: false, batchSave: true, batchSaveInLiveSync: false, delays: true }, + { liveSync: false, batchSave: false, batchSaveInLiveSync: true, delays: false }, + ])("shows the current mode choice and its shared delays: %j", ({ delays, ...settings }) => { + const { panels } = setup(settings.liveSync, settings.batchSave, settings.batchSaveInLiveSync); + expect(panels.get("obsidianLiveSyncSettingTab.titleUpdateThinning")?.().visibility).not.toBe(false); + expect(visible("batchSave")).toBe(!settings.liveSync); + expect(visible("batchSaveInLiveSync")).toBe(settings.liveSync); + expect(visible("batchSaveMinimumDelay")).toBe(delays); + expect(visible("batchSaveMaximumDelay")).toBe(delays); + }); + + it.each([ + [true, false], + [false, true], + ])("retains both batching choices across mode changes: %j", async (batchSave, batchSaveInLiveSync) => { + const { tab, callbacks } = setup(false, batchSave, batchSaveInLiveSync); + const switchMode = callbacks.get("syncMode")!; + for (const mode of ["LIVESYNC", "PERIODIC", "ONEVENTS", "LIVESYNC"]) { + await switchMode(mode); + expect(tab.editingSettings).toMatchObject({ batchSave, batchSaveInLiveSync }); + } + expect(tab.services.control.applySettings).toHaveBeenCalledTimes(4); + }); +}); diff --git a/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts new file mode 100644 index 00000000..10f6f6a6 --- /dev/null +++ b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts @@ -0,0 +1,961 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:net"; +import { join } from "node:path"; +import type { Page } from "playwright"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + loadCouchDbConfig, + makeUniqueDatabaseName, +} from "../runner/couchdb.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + assertE2eCompatibilityUnpaused, + configureCouchDb, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; + +type Mode = "normal" | "ignored-batch-setting" | "enabled-batch"; +type Slot = { name: string; vault: TemporaryVault; session?: ObsidianLiveSyncSession }; +type Leaf = { rev: string; ancestry: string[]; logicalDeleted: boolean; pouchDeleted: boolean; body: string | null }; +type Snapshot = { + path: string; + rev: string | null; + content: string | null; + fileMtime: number | null; + databaseMtime: number | null; + exists: boolean; + leaves: Leaf[]; + waiting: Array<{ path: string; type: string; since: number }>; + queued: Array<{ path: string | null; type: string }>; + shouldBatchSave: boolean; +}; +type Pair = { a: Page; b: Page }; + +const diagnostics = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/issue1207-livesync-batch"; +const artifactRoot = process.env.ISSUE1207_ARTIFACT_ROOT ?? process.cwd(); +const evidence: Record = { cases: [], observations: [] }; +const cases = evidence.cases as Array>; +const observations = evidence.observations as unknown[]; +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const prefix = "issue1207-batch-"; + +function check(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function hashes() { + return Object.fromEntries( + ["main.js", "manifest.json", "styles.css"].map((name) => [ + name, + createHash("sha256") + .update(new Uint8Array(readFileSync(join(artifactRoot, name)))) + .digest("hex"), + ]) + ); +} + +async function availablePort(): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + check(address && typeof address !== "string", "Missing debug port"); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + return address.port; +} + +async function evaluate(page: Page, code: string): Promise { + return await page.evaluate(`(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + const manager=core.serviceModules.storageAccess.vaultManager; + ${code} + })()`); +} + +async function installProbe(page: Page): Promise { + await evaluate( + page, + ` + if(globalThis.__issue1207Batch) return; + const probe={events:[],changes:[]}; + globalThis.__issue1207Batch=probe; + for(const type of ['create','modify','rename','delete']) { + app.vault.on(type,(file,oldPath)=>{ + const path=file.path; + if(!path.startsWith(${JSON.stringify(prefix)})) return; + const item={type,path,oldPath:oldPath??null,at:Date.now(),content:null}; + probe.events.push(item); + if(type!=='delete' && file.stat) { + void app.vault.read(file).then(body=>item.content=body).catch(error=>item.error=String(error)); + } + }); + } + probe.feed=core.localDatabase.localDatabase.changes({live:true,since:'now',include_docs:true}) + .on('change',change=>{ + if(typeof change.doc?.path==='string' && change.doc.path.startsWith(${JSON.stringify(prefix)})) { + probe.changes.push({path:change.doc.path,rev:change.doc._rev, + logicalDeleted:!!change.doc.deleted,pouchDeleted:!!change.doc._deleted,at:Date.now()}); + } + }).on('error',error=>probe.changes.push({error:String(error),at:Date.now()})); + ` + ); +} + +async function options(page: Page, mode: Mode, minimum = 2, maximum = 6): Promise { + await evaluate( + page, + ` + const settings=core.services.setting.currentSettings(); + Object.assign(settings,{liveSync:true,periodicReplication:false, + batchSave:${mode === "ignored-batch-setting"},batchSaveInLiveSync:${mode === "enabled-batch"}, + batchSaveMinimumDelay:${minimum},batchSaveMaximumDelay:${maximum},deleteMetadataOfDeletedFiles:false}); + return {liveSync:settings.liveSync,batchSave:settings.batchSave, + batchSaveInLiveSync:settings.batchSaveInLiveSync,shouldBatchSave:manager.shouldBatchSave}; + ` + ); +} + +async function snapshot(page: Page, path: string): Promise { + return await evaluate( + page, + ` + const path=${JSON.stringify(path)}; + const id=await core.services.path.path2id(path); + const row=(await core.localDatabase.allDocsRaw({keys:[id],include_docs:true})).rows[0]; + let leaves=[]; + if(row?.value?.rev) { + const rawLeaves=await core.localDatabase.getRaw(id,{open_revs:'all',revs:true}); + for(const item of rawLeaves) { + if(!item.ok) continue; + const raw=item.ok; + let body=null; + if(!raw._deleted) { + const entry=await core.localDatabase.getDBEntryFromMeta(raw,false,false,true); + if(entry!==false) body=Array.isArray(entry.data)?entry.data.join(''):entry.data; + } + leaves.push({rev:raw._rev,logicalDeleted:!!raw.deleted,pouchDeleted:!!raw._deleted,body, + ancestry:raw._revisions?.ids.map((hash,i)=>(raw._revisions.start-i)+'-'+hash)??[]}); + } + } + const file=app.vault.getAbstractFileByPath(path); + return {path,rev:row?.value?.rev??null,exists:!!file?.stat, + fileMtime:file?.stat?.mtime??null,databaseMtime:row?.doc?.mtime??null, + content:file?.stat?await app.vault.read(file):null,leaves, + waiting:[...manager._waitingMap].map(([key,value])=>({path:key,type:value.type,since:value.since})), + queued:manager.bufferedQueuedItems.map(item=>({path:item.args?.file?.path??null,type:item.type})), + shouldBatchSave:!!manager.shouldBatchSave}; + ` + ); +} + +function liveLeaves(state: Snapshot) { + return state.leaves.filter((leaf) => !leaf.logicalDeleted && !leaf.pouchDeleted); +} + +async function waitFor( + page: Page, + path: string, + predicate: (state: Snapshot) => boolean, + timeout = 20000 +): Promise { + const deadline = Date.now() + timeout; + let state: Snapshot; + do { + state = await snapshot(page, path); + if (predicate(state)) return state; + await pause(100); + } while (Date.now() < deadline); + throw new Error(`State did not settle: ${JSON.stringify(state)}`); +} + +async function settled(pair: Pair, path: string, body: string): Promise<[Snapshot, Snapshot]> { + const ready = (state: Snapshot) => + state.content === body && + liveLeaves(state).length === 1 && + liveLeaves(state)[0].body === body && + state.leaves.length === 1 && + !state.waiting.some((item) => item.path === path); + const deadline = Date.now() + 20000; + let states: [Snapshot, Snapshot]; + do { + states = await Promise.all([snapshot(pair.a, path), snapshot(pair.b, path)]); + // The row, leaves, and storage are read asynchronously. Require each + // sample to agree internally, then compare both current devices. + if ( + states.every((state) => ready(state) && state.rev === liveLeaves(state)[0].rev) && + states[0].rev === states[1].rev + ) + return states; + await pause(100); + } while (Date.now() < deadline); + throw new Error(`Devices did not converge: ${JSON.stringify(states)}`); +} + +async function create(page: Page, path: string, body: string): Promise { + await evaluate(page, `await app.vault.create(${JSON.stringify(path)},${JSON.stringify(body)});`); +} + +async function modify(page: Page, path: string, body: string): Promise { + await evaluate( + page, + `await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),${JSON.stringify(body)});` + ); +} + +async function trace(page: Page, path: string) { + return await evaluate( + page, + `return { + events:globalThis.__issue1207Batch.events.filter(item=>item.path===${JSON.stringify(path)}||item.oldPath===${JSON.stringify(path)}), + changes:globalThis.__issue1207Batch.changes.filter(item=>item.path===${JSON.stringify(path)}||item.error)};` + ); +} + +async function main(): Promise { + await mkdir(diagnostics, { recursive: true }); + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + check(cli.binary, "Missing Obsidian CLI"); + const cliBinary = cli.binary; + const config = await loadCouchDbConfig(process.env.ISSUE1207_COUCHDB_ENV); + check(["127.0.0.1", "localhost"].includes(new URL(config.uri).hostname), "Use a disposable local CouchDB"); + await assertCouchDbReachable(config); + const dbName = makeUniqueDatabaseName(config.dbPrefix, "live-batch"); + const couch = { ...config, dbName }; + const overrides = { + doctorProcessedVersion: "1.0.0", + notifyThresholdOfRemoteStorageSize: -1, + liveSync: false, + periodicReplication: false, + batchSave: false, + batchSaveInLiveSync: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + checkConflictOnlyOnOpen: true, + showMergeDialogOnlyOnActive: true, + disableMarkdownAutoMerge: true, + resolveConflictsByNewerFile: false, + writeDocumentsIfConflicted: false, + writeLogToTheFile: false, + }; + const before = hashes(); + evidence.artefactsBefore = before; + evidence.manifest = JSON.parse(await readFile(join(artifactRoot, "manifest.json"), "utf8")); + evidence.commonlibVersion = JSON.parse( + await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8") + ).version; + evidence.sourceBaseRevision = "73a14d4e4b0b414a1acfe170d01500abb3dca2e9"; + evidence.unreleasedImplementation = true; + if (process.env.ISSUE1207_COMMONLIB_ARTEFACT) { + evidence.commonlibArtefactSha256 = createHash("sha256") + .update(new Uint8Array(readFileSync(process.env.ISSUE1207_COMMONLIB_ARTEFACT))) + .digest("hex"); + } + evidence.timing = { minimumSeconds: 2, maximumSeconds: 6, incomingAndManualSeconds: { minimum: 5, maximum: 60 } }; + const slots: Slot[] = []; + let ownedDatabase = false; + + function redact(value: unknown): string { + return JSON.stringify(value, null, 2) + .split(config.password) + .join("") + .split(Buffer.from(`${config.username}:${config.password}`).toString("base64")) + .join(""); + } + + async function start(slot: Slot, restart = false) { + console.log(JSON.stringify({ phase: "starting", device: slot.name, restart })); + slot.session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + artifactRoot, + vault: slot.vault, + startupGraceMs: 1000, + pluginData: restart ? undefined : createE2eCouchDbPluginData(couch, overrides), + localStorageEntries: createE2eObsidianDeviceLocalState(slot.vault.name), + env: { E2E_OBSIDIAN_REMOTE_DEBUGGING_PORT: String(await availablePort()) }, + lifecycle: { + beforeLaunch: async () => { + for (const registry of [ + join(slot.vault.homePath, ".config", "obsidian", "obsidian.json"), + join(slot.vault.homePath, "Library", "Application Support", "obsidian", "obsidian.json"), + join(slot.vault.xdgConfigPath, "obsidian", "obsidian.json"), + join(slot.vault.userDataPath, "obsidian.json"), + ]) { + const state = JSON.parse(await readFile(registry, "utf8")); + state.updateDisabled = true; + await writeFile(registry, JSON.stringify(state)); + } + }, + }, + }); + console.log( + JSON.stringify({ + phase: "device-started", + device: slot.name, + port: slot.session.remoteDebuggingPort, + vault: slot.vault.path, + }) + ); + await waitForLiveSyncCoreReady(cliBinary, slot.session.cliEnv); + await assertE2eCompatibilityUnpaused(cliBinary, slot.session.cliEnv, slot.session.remoteDebuggingPort); + if (!restart) { + await configureCouchDb(cliBinary, slot.session.cliEnv, couch, overrides); + await waitForLiveSyncCoreReady(cliBinary, slot.session.cliEnv); + await prepareRemote(cliBinary, slot.session.cliEnv); + } + } + + async function stop(slot: Slot) { + if (!slot.session) return; + await slot.session.app.stop(); + await writeFile( + join(diagnostics, `process-${slot.name}-${slot.session.remoteDebuggingPort}.json`), + redact(slot.session.app.output()) + ); + slot.session = undefined; + } + + async function withPair(callback: (pair: Pair) => Promise): Promise { + check(slots[0].session && slots[1].session, "Both devices must be running"); + return await withObsidianPage( + slots[0].session.remoteDebuggingPort, + async (a) => + await withObsidianPage(slots[1].session!.remoteDebuggingPort, async (b) => { + await installProbe(a); + await installProbe(b); + return await callback({ a, b }); + }) + ); + } + + async function run(name: string, body: () => Promise) { + if (process.env.ISSUE1207_CASE_FILTER && !new RegExp(process.env.ISSUE1207_CASE_FILTER).test(name)) return; + console.log(JSON.stringify({ phase: "case-start", name })); + try { + const result = await body(); + cases.push({ name, passed: true, result }); + console.log(JSON.stringify({ phase: "case-passed", name })); + } catch (error) { + const failure = error instanceof Error ? error.stack : String(error); + cases.push({ name, passed: false, error: failure }); + console.log(JSON.stringify({ phase: "case-failed", name, error: failure })); + await withPair(async (pair) => { + let flushFailure: unknown; + for (const [device, page] of Object.entries(pair)) { + observations.push({ + name, + device, + probe: await evaluate( + page, + `return { + events:globalThis.__issue1207Batch.events,changes:globalThis.__issue1207Batch.changes, + pending:(await manager.adapter.persistence.loadSnapshot()).map(item=>({ + type:item.type,path:item.args?.file?.path,oldPath:item.args?.oldPath}))};` + ), + }); + await page.screenshot({ path: join(diagnostics, `${name}-${device}-failure.png`) }); + try { + await evaluate(page, "await manager.waitForIdle();"); + } catch (error) { + observations.push({ name, device, flushFailure: String(error) }); + flushFailure ??= error; + } + } + if (flushFailure) throw flushFailure; + }); + await pause(500); + } + await writeFile(join(diagnostics, "evidence.json"), redact(evidence)); + } + + try { + await createCouchDbDatabase(config, dbName); + ownedDatabase = true; + for (const name of ["a", "b"]) { + const slot = { name, vault: await createTemporaryVault(`issue1207-live-batch-${name}-`) }; + slots.push(slot); + await start(slot); + } + evidence.runtime = await withPair(async (pair) => { + const versions = []; + for (const page of [pair.a, pair.b]) { + versions.push( + await evaluate( + page, + `let appVersion=null; + try {appVersion=JSON.parse(require('fs').readFileSync(require('path').join(process.resourcesPath,'app.asar','package.json'),'utf8')).version;} catch {} + return {appVersion,electronVersion:process.versions.electron, + pluginVersion:app.plugins.plugins['obsidian-livesync'].manifest.version};` + ) + ); + await options(page, "normal"); + const outcome = await evaluate( + page, + `return await core.services.replication.startContinuous({ + trigger:'daemon',interaction:{kind:'forbidden'}});` + ); + check( + (outcome as { status?: string })?.status === "completed", + `Continuous start failed: ${JSON.stringify(outcome)}` + ); + } + return versions; + }); + + evidence.connectivity = await withPair(async (pair) => { + const path = `${prefix}connectivity.md`; + await create(pair.a, path, "Connectivity probe\n"); + const first = await settled(pair, path, "Connectivity probe\n"); + await modify(pair.b, path, "Bidirectional connectivity probe\n"); + const second = await settled(pair, path, "Bidirectional connectivity probe\n"); + return { first, second }; + }); + + for (const mode of ["normal", "ignored-batch-setting", "enabled-batch"] as const) { + for (let repeat = 0; repeat < 2; repeat++) { + await run(`${mode}-coalesce-${repeat}`, () => + withPair(async (pair) => { + for (const page of [pair.a, pair.b]) await options(page, mode); + const source = repeat === 0 ? pair.a : pair.b; + const receiver = repeat === 0 ? pair.b : pair.a; + const path = `${prefix}${mode}-edits-${repeat}.md`; + await create(source, path, "Initial body\n"); + const initial = (await settled(pair, path, "Initial body\n"))[0]; + for (let edit = 0; edit < 6; edit++) { + await modify(source, path, `Edit ${edit}\n`); + await pause(150); + } + const during = await snapshot(source, path); + const final = await settled(pair, path, "Edit 5\n"); + const finalSource = repeat === 0 ? final[0] : final[1]; + const delta = Number(finalSource.rev?.split("-")[0]) - Number(initial.rev?.split("-")[0]); + if (mode === "enabled-batch") { + check( + during.waiting.some((item) => item.path === path), + "Batch did not retain the pending edit" + ); + check(delta === 1, `Repeated edits created ${delta} revisions`); + } else { + check(!during.shouldBatchSave, "LiveSync batching was enabled without an explicit choice"); + check(delta >= 3, `The control did not expose enough edits: ${delta}`); + } + return { + path, + initial, + during, + final, + revisionDelta: delta, + source: await trace(source, path), + receiver: await trace(receiver, path), + }; + }) + ); + } + + await run(`${mode}-template-lifecycle`, () => + withPair(async (pair) => { + for (const page of [pair.a, pair.b]) await options(page, mode); + const oldPath = `${prefix}${mode}-untitled.md`; + const path = `${prefix}${mode}-completed.md`; + await create(pair.a, oldPath, ""); + await pause(300); + await modify(pair.a, oldPath, "Template in progress\n"); + await pause(300); + await evaluate( + pair.a, + `await app.fileManager.renameFile(app.vault.getFileByPath(${JSON.stringify(oldPath)}),${JSON.stringify(path)});` + ); + await pause(300); + await modify(pair.a, path, "Completed template\n"); + const during = await snapshot(pair.a, path); + const final = await settled(pair, path, "Completed template\n"); + await waitFor(pair.b, oldPath, (state) => !state.exists && liveLeaves(state).length === 0); + return { + oldPath, + path, + during, + final, + oldSource: await snapshot(pair.a, oldPath), + oldReceiver: await snapshot(pair.b, oldPath), + source: await trace(pair.a, path), + receiver: await trace(pair.b, path), + oldSourceTrace: await trace(pair.a, oldPath), + oldReceiverTrace: await trace(pair.b, oldPath), + }; + }) + ); + + for (const physical of [false, true]) { + await run(`${mode}-recreate-${physical}`, () => + withPair(async (pair) => { + for (const page of [pair.a, pair.b]) { + await options(page, mode); + await evaluate( + page, + `core.services.setting.currentSettings().deleteMetadataOfDeletedFiles=${physical};` + ); + } + const path = `${prefix}${mode}-recreate-${physical}.md`; + await create(pair.a, path, "Transient body\n"); + await pause(400); + await evaluate( + pair.a, + `await app.vault.delete(app.vault.getFileByPath(${JSON.stringify(path)}),true);` + ); + await pause(400); + await create(pair.a, path, "Recreated body\n"); + const final = await settled(pair, path, "Recreated body\n"); + return { + physical, + final, + source: await trace(pair.a, path), + receiver: await trace(pair.b, path), + }; + }) + ); + } + } + + for (const incoming of ["identical", "different", "delete"] as const) { + await run(`enabled-pending-incoming-${incoming}`, () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal", 5, 60); + const path = `${prefix}pending-${incoming}.md`; + await create(pair.b, path, "Shared base\n"); + const initial = await settled(pair, path, "Shared base\n"); + const local = "Local edit\n"; + await modify(pair.a, path, local); + const pending = await waitFor(pair.a, path, (state) => + state.waiting.some((item) => item.path === path) + ); + check(pending.rev === initial[0].rev, "Local edit reached the DB before incoming delivery"); + const sentAt = Date.now(); + if (incoming === "delete") { + await evaluate( + pair.b, + `await app.vault.delete(app.vault.getFileByPath(${JSON.stringify(path)}),true);` + ); + } else await modify(pair.b, path, incoming === "identical" ? local : "Remote edit\n"); + const arrived = await waitFor( + pair.a, + path, + (state) => + incoming === "identical" + ? state.rev !== initial[0].rev && liveLeaves(state).some((leaf) => leaf.body === local) + : state.leaves.some((leaf) => leaf.body === local) && + state.leaves.some((leaf) => + incoming === "delete" ? leaf.logicalDeleted : leaf.body === "Remote edit\n" + ), + 4000 + ); + check( + arrived.waiting.some((item) => item.path === path), + "The incoming assertion did not occur during Batch waiting" + ); + const leavesBefore = arrived.leaves.map((leaf) => leaf.rev).sort(); + if (incoming !== "identical") { + check(arrived.content === local, "Incoming reflection overwrote the pending local edit"); + check(arrived.leaves.length === 2, "Expected exactly two genuine concurrent branches"); + check( + arrived.leaves.every((leaf) => leaf.ancestry[1] === initial[0].rev), + "Concurrent branches lost their shared base" + ); + } + await pause(5600); + const after = await snapshot(pair.a, path); + const peer = await snapshot(pair.b, path); + check(!after.waiting.some((item) => item.path === path), "Batch timer did not complete"); + check( + JSON.stringify(after.leaves.map((leaf) => leaf.rev).sort()) === JSON.stringify(leavesBefore), + "The delayed event created another revision" + ); + check( + JSON.stringify(peer.leaves.map((leaf) => leaf.rev).sort()) === JSON.stringify(leavesBefore), + "Both branches did not converge to the peer" + ); + if (incoming === "identical") + check(after.leaves.length === 1, "Identical content created a branch"); + return { initial, pending, arrived, after, peer, incomingLatencyMs: Date.now() - sentAt - 5600 }; + }) + ); + } + + await run("enabled-maximum-delay", () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 2, 6); + await options(pair.b, "normal"); + const path = `${prefix}maximum.md`; + await create(pair.b, path, "Initial maximum body\n"); + const initial = await settled(pair, path, "Initial maximum body\n"); + const started = Date.now(); + let finalBody = ""; + for (let edit = 0; edit < 26; edit++) { + finalBody = `Continuous edit ${edit}\n`; + await modify(pair.a, path, finalBody); + await pause(350); + } + const stopped = Date.now(); + const final = await settled(pair, path, finalBody); + const changes = await evaluate>( + pair.a, + `return globalThis.__issue1207Batch.changes.filter(item=>item.path===${JSON.stringify(path)}&&item.rev!==${JSON.stringify(initial[0].rev)});` + ); + check(changes.length >= 2, "Continuous edits did not produce bounded intermediate and final saves"); + const first = changes[0].at - started; + check(first >= 5000 && first <= 8000, `Unexpected first maximum-delay flush: ${first}ms`); + check(changes[0].at < stopped - 1000, "The maximum flush only occurred after typing stopped"); + return { started, stopped, firstSaveMs: first, changes, final }; + }) + ); + + await run("enabled-many-paths", () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 2, 6); + await options(pair.b, "normal"); + const started = Date.now(); + for (let index = 0; index < 12; index++) + await create(pair.a, `${prefix}many-${index}.md`, `File ${index}\n`); + const final = []; + for (let index = 0; index < 12; index++) + final.push(await settled(pair, `${prefix}many-${index}.md`, `File ${index}\n`)); + return { elapsedMs: Date.now() - started, final }; + }) + ); + + await run("enabled-manual-sync", () => + withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal"); + const path = `${prefix}manual.md`; + await create(pair.b, path, "Before manual sync\n"); + await settled(pair, path, "Before manual sync\n"); + await modify(pair.a, path, "After manual sync\n"); + const pending = await waitFor(pair.a, path, (state) => + state.waiting.some((item) => item.path === path) + ); + const started = Date.now(); + const outcome = await evaluate(pair.a, "return await core.services.replication.replicate(true);"); + const atReturn = await snapshot(pair.a, path); + const final = await settled(pair, path, "After manual sync\n"); + const elapsedMs = Date.now() - started; + check(elapsedMs < 4000, `Manual sync waited for the Batch timer: ${elapsedMs}ms`); + return { pending, outcome, atReturn, final, elapsedMs }; + }) + ); + + const saveAfterEdit = process.env.ISSUE1207_RESTART_SAVE_AFTER_EDIT === "true"; + const restartBucket = process.env.ISSUE1207_RESTART_BUCKET; + check( + !restartBucket || ["same", "exact", "different"].includes(restartBucket), + "Unknown restart timestamp condition" + ); + const restartName = restartBucket + ? `enabled-inflight-restart-${restartBucket}` + : saveAfterEdit + ? "enabled-settings-save-restart" + : "enabled-restart"; + await run(restartName, async () => { + const path = `${prefix}restart.md`; + const beforeRestart = await withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal"); + if (!saveAfterEdit) + await evaluate( + pair.a, + "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" + ); + await create(pair.b, path, "Before restart\n"); + const initial = await settled(pair, path, "Before restart\n"); + if (restartBucket) { + check(initial[0].databaseMtime !== null, "Missing initial database mtime"); + const mtime = + restartBucket === "exact" + ? initial[0].databaseMtime + : Math.floor(initial[0].databaseMtime / 2000) * 2000 + + (restartBucket === "same" ? 1000 : 3000); + await evaluate( + pair.a, + `const original=manager.requestProcessQueue.bind(manager); + manager.requestProcessQueue=async item=>{ + if(item.args?.file?.path===${JSON.stringify(path)}) { + globalThis.__issue1207Batch.inflightPaused=true; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + return await original(item); + }; + await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),"Retained after restart\\n",{mtime:${mtime}});` + ); + } else await modify(pair.a, path, "Retained after restart\n"); + const pending = await waitFor(pair.a, path, (state) => + state.waiting.some((item) => item.path === path) + ); + if (saveAfterEdit) await evaluate(pair.a, "await core.services.setting.saveSettingData();"); + if (restartBucket) + await evaluate( + pair.a, + `const deadline=Date.now()+3000; + while(!globalThis.__issue1207Batch.inflightPaused&&Date.now()setTimeout(resolve,10)); + if(!globalThis.__issue1207Batch.inflightPaused) throw new Error('The in-flight test gate did not start');` + ); + await evaluate(pair.a, "await manager._takeSnapshot();"); + check(pending.rev === initial[0].rev, "Restart test did not retain a pending edit"); + if (restartBucket) { + const inFlight = await snapshot(pair.a, path); + const saved = await evaluate>( + pair.a, + `return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));` + ); + check( + inFlight.rev === initial[0].rev && !inFlight.waiting.some((item) => item.path === path), + "The in-flight fixture did not stop between gate release and database storage" + ); + check( + saved.some((item) => item.path === path), + "The in-flight event was lost from the recoverable snapshot" + ); + check(inFlight.fileMtime !== null && inFlight.databaseMtime !== null, "Missing in-flight mtimes"); + const sameBucket = + Math.floor(inFlight.fileMtime / 2000) === Math.floor(inFlight.databaseMtime / 2000); + check(sameBucket === (restartBucket !== "different"), "The timestamp condition was not applied"); + if (restartBucket === "exact") + check(inFlight.fileMtime === inFlight.databaseMtime, "Raw mtimes differ"); + observations.push({ + phase: "inflight-before-stop", + restartBucket, + pauseBeforeDatabaseMs: 30000, + inFlight, + saved, + sameBucket, + }); + } + if (!saveAfterEdit && !restartBucket) { + const strictState = await snapshot(pair.a, path); + const saved = await evaluate>( + pair.a, + `return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));` + ); + check( + strictState.rev === initial[0].rev && strictState.waiting.some((item) => item.path === path), + "Restart fixture flushed before shutdown" + ); + check( + saved.some((item) => item.path === path), + "Restart fixture did not persist the pending event" + ); + observations.push({ + phase: "restart-verified-pending-snapshot", + strictState, + saved, + initialized: await evaluate(pair.a, "return await core.kvDB.get('initialized');"), + }); + } + if (!saveAfterEdit) + observations.push({ + phase: "restart-before-stop", + initial, + pending, + persisted: await evaluate( + pair.a, + `const events=await manager.adapter.persistence.loadSnapshot(); + const settings=core.services.setting.currentSettings(); + return {initialized:await core.kvDB.get('initialized'),restorationRequested:manager.snapShotRestored!==null, + settings:{isConfigured:settings.isConfigured,liveSync:settings.liveSync,batchSave:settings.batchSave,batchSaveInLiveSync:settings.batchSaveInLiveSync, + suspendFileWatching:settings.suspendFileWatching}, + events:events?.map(item=>({type:item.type,path:item.args?.file?.path, + stat:item.args?.file?.stat,skipBatchWait:item.skipBatchWait}))};` + ), + }); + return { initial, pending }; + }); + await stop(slots[0]); + const persistedSettings = JSON.parse( + await readFile(join(slots[0].vault.path, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8") + ); + observations.push({ + phase: "restart-on-disk-settings", + settings: Object.fromEntries( + [ + "isConfigured", + "liveSync", + "batchSave", + "batchSaveInLiveSync", + "suspendFileWatching", + "settingVersion", + ].map((key) => [key, persistedSettings[key]]) + ), + }); + await start(slots[0], true); + const afterRestart = await withPair(async (pair) => { + observations.push({ + phase: "restart-after-boot", + state: await snapshot(pair.a, path), + persisted: await evaluate( + pair.a, + `const events=await manager.adapter.persistence.loadSnapshot(); + const settings=core.services.setting.currentSettings(); + return {initialized:await core.kvDB.get('initialized'),restorationRequested:manager.snapShotRestored!==null, + settings:{isConfigured:settings.isConfigured,liveSync:settings.liveSync,batchSave:settings.batchSave,batchSaveInLiveSync:settings.batchSaveInLiveSync, + suspendFileWatching:settings.suspendFileWatching}, + events:events?.map(item=>({type:item.type,path:item.args?.file?.path,stat:item.args?.file?.stat}))};` + ), + }); + const result = await settled(pair, path, "Retained after restart\n"); + check(result[0].shouldBatchSave, "The saved LiveSync batching choice was not restored"); + return { result, restoredGuard: result[0].shouldBatchSave }; + }); + return { beforeRestart, afterRestart }; + }); + + await run("enabled-after-write-restart", async () => { + const path = `${prefix}after-write.md`; + const saved = await withPair(async (pair) => { + await options(pair.a, "enabled-batch", 1, 3); + await options(pair.b, "normal"); + await evaluate( + pair.a, + "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" + ); + await create(pair.b, path, "Before the durable write\n"); + const initial = await settled(pair, path, "Before the durable write\n"); + await evaluate( + pair.a, + ` + const original=manager.requestProcessQueue.bind(manager); + manager.requestProcessQueue=async item=>{ + await original(item); + if(item.args?.file?.path===${JSON.stringify(path)}) { + globalThis.__issue1207Batch.afterWritePaused=true; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + }; + await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),"Durable before restart\\n"); + const deadline=Date.now()+6000; + while(!globalThis.__issue1207Batch.afterWritePaused&&Date.now()setTimeout(resolve,20)); + if(!globalThis.__issue1207Batch.afterWritePaused) throw new Error('The after-write gate did not start'); + await manager._takeSnapshot(); + ` + ); + const durable = await snapshot(pair.a, path); + const pending = await evaluate>( + pair.a, + "return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));" + ); + check(durable.rev !== initial[0].rev, "The database write did not complete before interruption"); + check( + pending.some((item) => item.path === path), + "The after-write operation was not recoverable" + ); + return { initial, durable, pending }; + }); + const restarts = []; + for (let attempt = 0; attempt < 2; attempt++) { + await stop(slots[0]); + await start(slots[0], true); + const final = await withPair((pair) => settled(pair, path, "Durable before restart\n")); + check(final[0].rev === saved.durable.rev, "Replay created a duplicate database revision"); + restarts.push(final); + } + const later = await withPair(async (pair) => { + await modify(pair.a, path, "A later edit survives\n"); + return await settled(pair, path, "A later edit survives\n"); + }); + return { saved, restarts, later }; + }); + + await run("enabled-before-snapshot-restart-exact", async () => { + const path = `${prefix}before-snapshot.md`; + const beforeStop = await withPair(async (pair) => { + await options(pair.a, "enabled-batch", 5, 60); + await options(pair.b, "normal"); + await evaluate( + pair.a, + "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" + ); + await create(pair.b, path, "Before the first snapshot\n"); + const initial = await settled(pair, path, "Before the first snapshot\n"); + check(initial[0].databaseMtime !== null, "Missing initial database mtime"); + await evaluate( + pair.a, + ` + await manager.waitForIdle(); + const persistence=manager.adapter.persistence; + const original=persistence.saveSnapshot.bind(persistence); + persistence.saveSnapshot=async events=>{ + if(events.some(item=>item.type==='CHANGED'&&item.args?.file?.path===${JSON.stringify(path)})) { + globalThis.__issue1207Batch.beforeSnapshotPaused=true; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + return await original(events); + }; + await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}), + "Recovered without an event snapshot\\n",{mtime:${initial[0].databaseMtime}}); + const deadline=Date.now()+3000; + while(!globalThis.__issue1207Batch.beforeSnapshotPaused&&Date.now()setTimeout(resolve,10)); + if(!globalThis.__issue1207Batch.beforeSnapshotPaused) + throw new Error('The snapshot persistence gate did not start'); + ` + ); + const state = await snapshot(pair.a, path); + const persisted = await evaluate>( + pair.a, + "return (await manager.adapter.persistence.loadSnapshot()??[]).map(item=>({path:item.args?.file?.path}));" + ); + check(state.rev === initial[0].rev, "The unsaved edit reached the database before interruption"); + check(state.fileMtime === state.databaseMtime, "Raw mtimes differ before interruption"); + check(!persisted.some((item) => item.path === path), "The interrupted edit already has a saved event"); + return { initial, state, persisted }; + }); + await stop(slots[0]); + await start(slots[0], true); + const recovered = await withPair((pair) => settled(pair, path, "Recovered without an event snapshot\n")); + check(recovered[0].shouldBatchSave, "The saved LiveSync batching choice was not restored"); + return { beforeStop, recovered }; + }); + + evidence.artefactsAfter = hashes(); + check( + JSON.stringify(before) === JSON.stringify(evidence.artefactsAfter), + "Plug-in artefacts changed during testing" + ); + evidence.passed = cases.every((item) => item.passed); + await writeFile(join(diagnostics, "evidence.json"), redact(evidence)); + console.log( + JSON.stringify({ + phase: "completed", + passed: evidence.passed, + cases: cases.length, + failed: cases.filter((item) => !item.passed).map((item) => item.name), + }) + ); + if (!evidence.passed) process.exitCode = 1; + } finally { + for (const slot of slots) await stop(slot); + for (const slot of slots) await slot.vault.dispose(); + if (ownedDatabase) await deleteCouchDbDatabase(config, dbName); + evidence.cleanup = { devicesStopped: slots.every((slot) => !slot.session), databaseRemoved: ownedDatabase }; + await writeFile(join(diagnostics, "evidence.json"), redact(evidence)); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/updates.md b/updates.md index cefb2bce..e3e425e5 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,21 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### New Feature + +- We can enable **Batch database updates in LiveSync mode** to combine repeated local edits before sending them to other devices. + - The choice starts disabled, including when older settings contain the previously ignored batch option. Each mode retains its own choice and uses the same minimum and maximum delays. + - Batching delays remote visibility and reduces intermediate updates. It does not establish when a template has finished or prevent genuine concurrent edits. + +#### Fixed + +- Local changes remain available for recovery while their database save is running, including when the application stops after a batch delay ends. + - Manual synchronisation and settings application now wait for the accepted local saves to finish and report failures. Pending events retain the existing snapshot format. +- Start-up reconciliation checks content when modification times are equal, so an edit within the same timestamp interval is not mistaken for an unchanged file. + - Existing revision provenance preserves the correct edit branch. Unreadable or continually changing content remains unresolved rather than being reported as successfully reconciled. + ## 1.0.40 24th October, 2026 From 68dc40ff05f429c8b47448dbbfa9c27ee937179e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 05:40:10 +0000 Subject: [PATCH 47/68] Show CouchDB comparison progress in customisable status items --- README.md | 18 + package.json | 1 + src/modules/features/ModuleLog.ts | 129 ++- .../features/ReplicationStatusLabel.ts | 209 +++++ .../ReplicationStatusLabel.unit.spec.ts | 241 ++++++ src/modules/features/StatusBarDisplay.ts | 51 +- .../features/StatusBarDisplay.unit.spec.ts | 18 + src/modules/features/StatusBarRenderer.ts | 151 ++++ .../features/StatusBarRenderer.unit.spec.ts | 300 +++++++ styles.css | 15 + test/e2e-obsidian/README.md | 3 + .../scripts/replication-status.ts | 770 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + updates.md | 2 + 14 files changed, 1837 insertions(+), 72 deletions(-) create mode 100644 src/modules/features/ReplicationStatusLabel.ts create mode 100644 src/modules/features/ReplicationStatusLabel.unit.spec.ts create mode 100644 src/modules/features/StatusBarRenderer.ts create mode 100644 src/modules/features/StatusBarRenderer.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/replication-status.ts diff --git a/README.md b/README.md index f84b0267..38c78d2c 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Synchronisation status is shown in the status bar with the following icons. - 🌐N Approximate remote requests currently in progress - Status - ⏚ī¸ Stopped + - ⏊ Comparing CouchDB change history, including entries already held locally - 💤 LiveSync enabled. Waiting for changes - ⚡ī¸ Synchronisation in progress - ⚠ An error occurred @@ -104,6 +105,23 @@ Synchronisation status is shown in the status bar with the following icons. To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files. +When CouchDB change history is being compared, `↓ 0 (⏊ 1200)` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It disappears after three seconds without a change and reappears when the count changes. The ⏊ status icon follows comparison activity independently of that timer. + +### Customising status items with CSS snippets + +The status bar and editor overlay use individual spans with the common `livesync-stat-` class prefix. CSS snippets can hide or style the items we need. For example, this hides the comparison count and P2P summary: + +```css +.livesync-stat-ff, +.livesync-stat-p2p { + display: none; +} +``` + +The item classes include `livesync-stat-network`, `livesync-stat-sync`, `livesync-stat-sent`, `livesync-stat-push-remaining`, `livesync-stat-arrived`, `livesync-stat-pull-remaining`, `livesync-stat-ff`, and `livesync-stat-p2p`. File-processing items use `livesync-stat-file-processing`, `livesync-stat-file-pending`, and `livesync-stat-file-batched`; queue items use `livesync-stat-queue-` followed by `replication`, `database`, `storage`, `chunk-initial`, `chunk-retrying`, `plugin-scan`, `hidden-files`, or `conflicts`. The skipped-file indicator uses `livesync-stat-file-skip`. + +All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count until it is hidden, reducing movement as counts decrease. + ## Tips and Troubleshooting - If you want a faster and simpler initial replication when setting up subsequent devices, see the [Fast Setup Guide](docs/tips/fast-setup.md). - Configure [Hidden File Sync](docs/tips/hidden-file-sync.md) only after ordinary note synchronisation works. diff --git a/package.json b/package.json index cd383dde..90d9e730 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "test:e2e:obsidian:configuration-encryption-sync": "tsx test/e2e-obsidian/scripts/configuration-encryption-sync.ts", "test:e2e:obsidian:review-harness": "tsx test/e2e-obsidian/scripts/review-harness.ts", "test:e2e:obsidian:p2p-pane": "tsx test/e2e-obsidian/scripts/p2p-pane.ts", + "test:e2e:obsidian:replication-status": "tsx test/e2e-obsidian/scripts/replication-status.ts", "test:e2e:obsidian:vault-reflection": "tsx test/e2e-obsidian/scripts/vault-reflection.ts", "test:e2e:obsidian:couchdb-upload": "tsx test/e2e-obsidian/scripts/couchdb-upload.ts", "test:e2e:obsidian:couchdb-http-cache": "tsx test/e2e-obsidian/scripts/couchdb-http-cache.ts", diff --git a/src/modules/features/ModuleLog.ts b/src/modules/features/ModuleLog.ts index 99170ec1..0755506b 100644 --- a/src/modules/features/ModuleLog.ts +++ b/src/modules/features/ModuleLog.ts @@ -4,7 +4,6 @@ import { LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE, PREFIXMD_LOGFILE, - type DatabaseConnectingStatus, type LOG_LEVEL, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { scheduleTask } from "octagonal-wheels/concurrency/task"; @@ -31,16 +30,14 @@ import { LogPaneView, VIEW_TYPE_LOG } from "./Log/LogPaneView.ts"; import { serialized } from "octagonal-wheels/concurrency/lock"; import { $msg } from "@/common/translation"; import { P2PLogCollector } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/P2PLogCollector"; -import { - REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS, - formatRemoteActivityStatusLabel, - getTrackedRequestCount, -} from "./RemoteActivityStatus.ts"; +import { REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS, getTrackedRequestCount } from "./RemoteActivityStatus.ts"; import { createChunkFetchCounterLabel, createMinimumVisibleActivityCount, createPaddedCounterLabel, } from "./StatusBarDisplay.ts"; +import { createFastForwardCheckedChangesDisplay, projectReplicationStatusLabel } from "./ReplicationStatusLabel.ts"; +import { createStatusBarRenderer, type StatusBarPart, type StatusBarRenderer } from "./StatusBarRenderer.ts"; import type { LiveSyncCore } from "@/main.ts"; import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError"; import { isValidPath } from "@/common/utils.ts"; @@ -124,7 +121,14 @@ export class ModuleLog extends AbstractObsidianModule { logHistory?: HTMLDivElement; messageArea?: HTMLDivElement; - statusBarLabels!: ReactiveValue<{ message: string; status: string }>; + statusBarRenderer?: StatusBarRenderer; + statusLineRenderer?: StatusBarRenderer; + statusBarLabels!: ReactiveValue<{ + message: string; + status: string; + parts: readonly StatusBarPart[]; + statusBarParts: readonly StatusBarPart[]; + }>; statusLog = reactiveSource(""); activeFileStatus = reactiveSource(""); notifies: { [key: string]: { count: number } } = {}; @@ -164,79 +168,54 @@ export class ModuleLog extends AbstractObsidianModule { ); const requestingStatLabel = computed(() => { - return formatRemoteActivityStatusLabel({ + return { remoteOperationCount: Math.max(0, this.services.replicator.boundedRemoteActivityCount.value), trackedRequestCount: displayedTrackedRequestCount.value, - }); + }; }); - const replicationStatLabel = computed(() => { - const e = this.services.replicator.replicationStatics.value; - const sent = e.sent; - const arrived = e.arrived; - const maxPullSeq = e.maxPullSeq; - const maxPushSeq = e.maxPushSeq; - const lastSyncPullSeq = e.lastSyncPullSeq; - const lastSyncPushSeq = e.lastSyncPushSeq; - let pushLast = ""; - let pullLast = ""; - let w = ""; - const labels: Partial> = { - CONNECTED: "⚡", - JOURNAL_SEND: "đŸ“Ļ↑", - JOURNAL_RECEIVE: "đŸ“Ļ↓", - }; - switch (e.syncStatus) { - case "CLOSED": - case "COMPLETED": - case "NOT_CONNECTED": - w = "⏚"; - break; - case "STARTED": - w = "🌀"; - break; - case "PAUSED": - w = "💤"; - break; - case "CONNECTED": - case "JOURNAL_SEND": - case "JOURNAL_RECEIVE": - w = labels[e.syncStatus] || "⚡"; - pushLast = - lastSyncPushSeq == 0 - ? "" - : lastSyncPushSeq >= maxPushSeq - ? " (LIVE)" - : ` (${maxPushSeq - lastSyncPushSeq})`; - pullLast = - lastSyncPullSeq == 0 - ? "" - : lastSyncPullSeq >= maxPullSeq - ? " (LIVE)" - : ` (${maxPullSeq - lastSyncPullSeq})`; - break; - case "ERRORED": - w = "⚠"; - break; - default: - w = "?"; - } - return { w, sent, pushLast, arrived, pullLast }; - }); + const fastForwardCount = registerDisplay( + createFastForwardCheckedChangesDisplay(this.services.replicator.replicationStatics) + ); + const replicationStatLabel = computed(() => + projectReplicationStatusLabel( + this.services.replicator.replicationStatics.value, + requestingStatLabel(), + fastForwardCount.value + ) + ); const labelProc = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.processing, `âŗ`)); const labelPend = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.totalQueued, `đŸ›Ģ`)); const labelInBatchDelay = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.batched, `đŸ“Ŧ`)); const waitingLabel = computed(() => { return `${labelProc.value}${labelPend.value}${labelInBatchDelay.value}`; }); + const counterPart = (key: string, mark: string, value: number | undefined): StatusBarPart[] => + value === undefined ? [] : [{ key, text: " " + mark, value }]; const statusLineLabel = computed(() => { - const { w, sent, pushLast, arrived, pullLast } = replicationStatLabel(); + const replication = replicationStatLabel(); const queued = queueCountLabel(); const waiting = waitingLabel(); - const networkActivity = requestingStatLabel(); const p2p = this.p2pLogCollector.p2pReplicationLine.value; + const firstLineParts = [ + ...replication.parts, + ...counterPart("file-processing", "âŗ", labelProc.count.value), + ...counterPart("file-pending", "đŸ›Ģ", labelPend.count.value), + ...counterPart("file-batched", "đŸ“Ŧ", labelInBatchDelay.count.value), + ...counterPart("queue-replication", "đŸ“Ĩ", labelReplication.count.value), + ...counterPart("queue-database", "📄", labelDBCount.count.value), + ...counterPart("queue-storage", "💾", labelStorageCount.count.value), + ...counterPart("queue-chunk-initial", "🛄", labelChunkCount.initialCount.value), + ...counterPart("queue-chunk-retrying", "🔁", labelChunkCount.retryingCount.value), + ...counterPart("queue-plugin-scan", "🔌", labelPluginScanCount.count.value), + ...counterPart("queue-hidden-files", "⚙ī¸", labelHiddenFilesCount.count.value), + ...counterPart("queue-conflicts", "🔩", labelConflictProcessCount.count.value), + ]; return { - message: `${networkActivity}Sync: ${w} ↑ ${sent}${pushLast} ↓ ${arrived}${pullLast}${waiting}${queued}${p2p == "" ? "" : "\n" + p2p}`, + message: replication.message + waiting + queued + (p2p === "" ? "" : "\n" + p2p), + parts: p2p === "" ? firstLineParts : [...firstLineParts, { key: "p2p", text: "\n" + p2p }], + statusBarParts: firstLineParts, + hasP2P: p2p !== "", }; }); @@ -244,13 +223,16 @@ export class ModuleLog extends AbstractObsidianModule { const scheduleMessage = this.services.appLifecycle.isReloadingScheduled() ? `WARNING! RESTARTING OBSIDIAN IS SCHEDULED!\n` : ""; - const { message } = statusLineLabel(); + const { message, parts, statusBarParts, hasP2P } = statusLineLabel(); const fileStatus = this.activeFileStatus.value; const status = scheduleMessage + this.statusLog.value; const fileStatusIcon = `${fileStatus && this.settings.hideFileWarningNotice ? " ⛔ SKIP" : ""}`; + const skipPart = fileStatusIcon === "" ? [] : [{ key: "file-skip", text: fileStatusIcon }]; return { message: `${message}${fileStatusIcon}`, status, + parts: [...parts, ...skipPart], + statusBarParts: hasP2P ? statusBarParts : [...statusBarParts, ...skipPart], }; }); this.statusBarLabels = statusBarLabels; @@ -391,7 +373,8 @@ export class ModuleLog extends AbstractObsidianModule { newLog = newLog.substring(moduleTagEnd + MARK_LOG_SEPARATOR.length + 1); } - this.statusBar?.setText(newMsg.split("\n")[0]); + const { parts, statusBarParts } = this.statusBarLabels.value; + this.statusBarRenderer?.render(statusBarParts); if (this.statusDiv) { this.statusDiv.setCssStyles({ display: this.settings?.showStatusOnEditor ? "" : "none" }); } @@ -409,7 +392,7 @@ export class ModuleLog extends AbstractObsidianModule { const recentLogs = recent.reverse().join("\n"); if (isDirty("recentLogs", recentLogs)) this.logHistory!.innerText = recentLogs; } - if (isDirty("newMsg", newMsg)) this.statusLine!.innerText = newMsg; + if (isDirty("newMsg", newMsg)) this.statusLineRenderer?.render(parts); if (isDirty("newLog", newLog)) this.logMessage!.innerText = newLog; } else { // const root = activeDocument.documentElement; @@ -505,6 +488,16 @@ ${stringifyYaml(info)} this.statusBar = this.services.API.addStatusBarItem(); this.statusBar?.addClass("syncstatusbar"); } + if (this.statusLine) { + const renderer = createStatusBarRenderer(this.statusLine); + this.statusLineRenderer = renderer; + this.plugin.register(() => renderer.dispose()); + } + if (this.statusBar) { + const renderer = createStatusBarRenderer(this.statusBar); + this.statusBarRenderer = renderer; + this.plugin.register(() => renderer.dispose()); + } this.adjustStatusDivPosition(); this._log("Log module loaded", LOG_LEVEL_INFO); this._log("Verbose log", LOG_LEVEL_VERBOSE); diff --git a/src/modules/features/ReplicationStatusLabel.ts b/src/modules/features/ReplicationStatusLabel.ts new file mode 100644 index 00000000..b36d4c95 --- /dev/null +++ b/src/modules/features/ReplicationStatusLabel.ts @@ -0,0 +1,209 @@ +import { reactiveSource, type ReactiveValue } from "octagonal-wheels/dataobject/reactive"; +import type { ReplicationStatics } from "@vrtmrz/livesync-commonlib/replication"; +import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; + +import type { DisposableReactiveValue } from "./StatusBarDisplay.ts"; +import type { StatusBarPart } from "./StatusBarRenderer.ts"; +import { + formatRemoteActivityStatusLabel, + REMOTE_OPERATION_ACTIVITY_ICON, + REMOTE_REQUEST_ACTIVITY_ICON, + type RemoteActivityStatus, +} from "./RemoteActivityStatus.ts"; + +export const FAST_FORWARD_COUNT_LINGER_MS = 3_000; + +function readPositiveCheckedChanges(source: ReactiveValue): number | undefined { + const count = source.value.couchDB?.pull?.checkedChanges; + return typeof count === "number" && Number.isFinite(count) && count > 0 ? count : undefined; +} + +/** + * Briefly displays changing CouchDB pull-feed comparison counts. + * + * Unrelated replication-stat changes do not extend the linger or redisplay an + * expired count. A positive initial snapshot is displayed to late subscribers. + */ +export function createFastForwardCheckedChangesDisplay( + source: ReactiveValue, + lingerMs = FAST_FORWARD_COUNT_LINGER_MS +): DisposableReactiveValue { + const displayed = reactiveSource(undefined); + const linger = Math.max(0, lingerMs); + let lastObserved = readPositiveCheckedChanges(source); + let clearTimer: number | undefined; + let disposed = false; + + const cancelClear = () => { + if (clearTimer === undefined) return; + compatGlobal.clearTimeout(clearTimer); + clearTimer = undefined; + }; + const scheduleClear = () => { + clearTimer = compatGlobal.setTimeout(() => { + clearTimer = undefined; + if (disposed) return; + if (readPositiveCheckedChanges(source) !== lastObserved) { + update(); + return; + } + displayed.value = undefined; + }, linger); + }; + const update = () => { + if (disposed) return; + const nextCount = readPositiveCheckedChanges(source); + if (nextCount === lastObserved) return; + lastObserved = nextCount; + cancelClear(); + if (nextCount === undefined) { + displayed.value = undefined; + return; + } + displayed.value = nextCount; + scheduleClear(); + }; + + if (lastObserved !== undefined) { + displayed.value = lastObserved; + scheduleClear(); + } + source.onChanged(update); + + return { + get value() { + return displayed.value; + }, + onChanged(handler) { + displayed.onChanged(handler); + }, + offChanged(handler) { + displayed.offChanged(handler); + }, + dispose() { + if (disposed) return; + disposed = true; + cancelClear(); + source.offChanged(update); + }, + }; +} + +export type ReplicationStatusProjection = { + icon: string; + message: string; + parts: StatusBarPart[]; +}; + +function remainingSequenceLabel( + key: string, + lastSyncSeq: number, + maxSeq: number +): { text: string; part?: StatusBarPart } { + if (lastSyncSeq === 0) return { text: "" }; + if (lastSyncSeq >= maxSeq) return { text: " (LIVE)", part: { key, text: " (LIVE)" } }; + const remaining = maxSeq - lastSyncSeq; + return { + text: " (" + remaining + ")", + part: { key, text: " (", value: remaining, suffix: ")" }, + }; +} + +/** Projects the existing replication labels into reusable numeric status items. */ +export function projectReplicationStatusLabel( + stats: ReplicationStatics, + networkStatus: RemoteActivityStatus, + fastForwardCount?: number +): ReplicationStatusProjection { + const { sent, arrived, maxPullSeq, maxPushSeq, lastSyncPullSeq, lastSyncPushSeq } = stats; + const networkActivity = formatRemoteActivityStatusLabel(networkStatus); + let pushRemaining: { text: string; part?: StatusBarPart } = { text: "" }; + let pullRemaining: { text: string; part?: StatusBarPart } = { text: "" }; + let icon: string; + switch (stats.syncStatus) { + case "CLOSED": + case "COMPLETED": + case "NOT_CONNECTED": + icon = "⏚"; + break; + case "STARTED": + icon = "🌀"; + break; + case "PAUSED": + icon = "💤"; + break; + case "CONNECTED": + case "JOURNAL_SEND": + case "JOURNAL_RECEIVE": + icon = stats.syncStatus === "JOURNAL_SEND" ? "đŸ“Ļ↑" : stats.syncStatus === "JOURNAL_RECEIVE" ? "đŸ“Ļ↓" : "⚡"; + pushRemaining = remainingSequenceLabel("push-remaining", lastSyncPushSeq, maxPushSeq); + pullRemaining = remainingSequenceLabel("pull-remaining", lastSyncPullSeq, maxPullSeq); + break; + case "ERRORED": + icon = "⚠"; + break; + default: + icon = "?"; + } + + const couchDB = stats.couchDB; + const isFastForwardChecking = + stats.syncStatus !== "ERRORED" && + couchDB !== undefined && + couchDB.phase !== "idle" && + (couchDB.pull?.checking === true || couchDB.push?.checking === true); + if (isFastForwardChecking) icon = "⏊"; + + const visibleFastForwardCount = + couchDB !== undefined && + typeof fastForwardCount === "number" && + Number.isFinite(fastForwardCount) && + fastForwardCount > 0 + ? fastForwardCount + : undefined; + const fastForwardSuffix = visibleFastForwardCount === undefined ? "" : " (⏊ " + visibleFastForwardCount + ")"; + + const parts: StatusBarPart[] = []; + if (networkStatus.trackedRequestCount > 0) { + parts.push({ + key: "network", + text: + (networkStatus.remoteOperationCount > 0 ? `${REMOTE_OPERATION_ACTIVITY_ICON} ` : "") + + REMOTE_REQUEST_ACTIVITY_ICON, + value: networkStatus.trackedRequestCount, + suffix: " ", + }); + } else if (networkActivity !== "") { + parts.push({ key: "network", text: networkActivity }); + } + parts.push({ key: "sync", text: "Sync: " + icon }); + parts.push({ key: "sent", text: " ↑ ", value: sent }); + if (pushRemaining.part !== undefined) parts.push(pushRemaining.part); + parts.push({ key: "arrived", text: " ↓ ", value: arrived }); + if (pullRemaining.part !== undefined) parts.push(pullRemaining.part); + if (visibleFastForwardCount !== undefined) { + parts.push({ + key: "ff", + text: " (⏊ ", + value: visibleFastForwardCount, + suffix: ")", + title: "Compared CouchDB feed entries", + }); + } + + return { + icon, + message: + networkActivity + + "Sync: " + + icon + + " ↑ " + + sent + + pushRemaining.text + + " ↓ " + + arrived + + pullRemaining.text + + fastForwardSuffix, + parts, + }; +} diff --git a/src/modules/features/ReplicationStatusLabel.unit.spec.ts b/src/modules/features/ReplicationStatusLabel.unit.spec.ts new file mode 100644 index 00000000..6f322e8e --- /dev/null +++ b/src/modules/features/ReplicationStatusLabel.unit.spec.ts @@ -0,0 +1,241 @@ +import { reactiveSource } from "octagonal-wheels/dataobject/reactive"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ReplicationStatics } from "@vrtmrz/livesync-commonlib/replication"; + +import { + FAST_FORWARD_COUNT_LINGER_MS, + createFastForwardCheckedChangesDisplay, + projectReplicationStatusLabel, +} from "./ReplicationStatusLabel.ts"; + +function couchDBProgress( + phase: "idle" | "one-shot" | "catch-up" | "continuous", + checkedChanges = 0, + checking = false +): NonNullable { + return { + phase, + pull: { checkedChanges, completedBatches: 0, checking }, + push: { checkedChanges: 0, completedBatches: 0, checking: false }, + }; +} + +const noNetworkActivity = { remoteOperationCount: 0, trackedRequestCount: 0 }; + +function stats(overrides: Partial = {}): ReplicationStatics { + return { + sent: 0, + arrived: 0, + maxPullSeq: 0, + maxPushSeq: 0, + lastSyncPullSeq: 0, + lastSyncPushSeq: 0, + syncStatus: "CONNECTED", + ...overrides, + }; +} + +describe("projectReplicationStatusLabel", () => { + it("exposes the remote-request count as a numeric status item", () => { + const projection = projectReplicationStatusLabel(stats(), { remoteOperationCount: 1, trackedRequestCount: 12 }); + expect(projection.parts.find((part) => part.key === "network")).toMatchObject({ + text: "📲 🌐", + value: 12, + suffix: " ", + }); + }); + + it("preserves the connected label, directions, and remaining-sequence text", () => { + const projection = projectReplicationStatusLabel( + stats({ + sent: 4, + arrived: 2, + maxPullSeq: 20, + maxPushSeq: 30, + lastSyncPullSeq: 17, + lastSyncPushSeq: 22, + }), + { remoteOperationCount: 0, trackedRequestCount: 1 } + ); + + expect(projection.message).toBe("🌐1 Sync: ⚡ ↑ 4 (8) ↓ 2 (3)"); + expect(projection.parts.map((part) => part.key)).toEqual([ + "network", + "sync", + "sent", + "push-remaining", + "arrived", + "pull-remaining", + ]); + expect(projection.parts[2]).toMatchObject({ key: "sent", text: " ↑ ", value: 4 }); + expect(projection.parts[3]).toMatchObject({ key: "push-remaining", text: " (", value: 8, suffix: ")" }); + expect(projection.parts[4]).toMatchObject({ key: "arrived", text: " ↓ ", value: 2 }); + expect(projection.parts[5]).toMatchObject({ key: "pull-remaining", text: " (", value: 3, suffix: ")" }); + }); + + it("keeps the existing status icons when CouchDB is idle or not present", () => { + const statuses = [ + ["CLOSED", "⏚"], + ["COMPLETED", "⏚"], + ["NOT_CONNECTED", "⏚"], + ["STARTED", "🌀"], + ["PAUSED", "💤"], + ["CONNECTED", "⚡"], + ["JOURNAL_SEND", "đŸ“Ļ↑"], + ["JOURNAL_RECEIVE", "đŸ“Ļ↓"], + ["ERRORED", "⚠"], + ] as const; + + for (const [syncStatus, icon] of statuses) { + const idleWithCount = projectReplicationStatusLabel( + stats({ syncStatus, couchDB: couchDBProgress("idle", 19, true) }), + noNetworkActivity, + 19 + ); + expect(idleWithCount.icon).toBe(icon); + expect(idleWithCount.parts.some((part) => part.key === "ff")).toBe(true); + + const withoutCouchDB = projectReplicationStatusLabel(stats({ syncStatus }), noNetworkActivity, 19); + expect(withoutCouchDB.icon).toBe(icon); + expect(withoutCouchDB.parts.some((part) => part.key === "ff")).toBe(false); + } + + const noFastForward = projectReplicationStatusLabel(stats(), noNetworkActivity); + expect(noFastForward.parts.some((part) => part.key === "ff")).toBe(false); + expect(noFastForward.message).toBe("Sync: ⚡ ↑ 0 ↓ 0"); + }); + + it("shows fast-forward checking with zero arrivals, transferred data, and the final idle count", () => { + const checking = projectReplicationStatusLabel( + stats({ + syncStatus: "PAUSED", + arrived: 0, + couchDB: couchDBProgress("catch-up", 0, true), + }), + noNetworkActivity + ); + expect(checking.icon).toBe("⏊"); + expect(checking.parts[2]).toMatchObject({ key: "arrived", value: 0 }); + + const transferred = projectReplicationStatusLabel( + stats({ + arrived: 7, + couchDB: couchDBProgress("continuous", 1200, false), + }), + noNetworkActivity, + 1200 + ); + expect(transferred.icon).toBe("⚡"); + expect(transferred.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏊ 1200)"); + expect(transferred.parts[transferred.parts.length - 1]).toMatchObject({ + key: "ff", + text: " (⏊ ", + value: 1200, + suffix: ")", + }); + + const idle = projectReplicationStatusLabel( + stats({ + syncStatus: "CONNECTED", + arrived: 7, + couchDB: couchDBProgress("idle", 1200, false), + }), + noNetworkActivity, + 1200 + ); + expect(idle.icon).toBe("⚡"); + expect(idle.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏊ 1200)"); + }); + + it("keeps the error icon ahead of an active CouchDB checking indicator", () => { + const projection = projectReplicationStatusLabel( + stats({ + syncStatus: "ERRORED", + couchDB: { + ...couchDBProgress("one-shot", 2, true), + push: { checkedChanges: 0, completedBatches: 0, checking: true }, + }, + }), + noNetworkActivity + ); + + expect(projection.icon).toBe("⚠"); + }); +}); + +describe("createFastForwardCheckedChangesDisplay", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("uses the initial snapshot, expires unchanged counts, and only reappears for new progress", () => { + const source = reactiveSource( + stats({ + couchDB: couchDBProgress("catch-up", 1200, true), + }) + ); + const display = createFastForwardCheckedChangesDisplay(source); + + expect(display.value).toBe(1200); + vi.advanceTimersByTime(1000); + source.value = stats({ + sent: 3, + couchDB: couchDBProgress("idle", 1200, false), + }); + expect(display.value).toBe(1200); + vi.advanceTimersByTime(FAST_FORWARD_COUNT_LINGER_MS - 1001); + expect(display.value).toBe(1200); + + source.value = stats({ + sent: 4, + couchDB: couchDBProgress("idle", 1200, false), + }); + vi.advanceTimersByTime(1); + expect(display.value).toBeUndefined(); + + source.value = stats({ + syncStatus: "PAUSED", + couchDB: couchDBProgress("continuous", 1200, true), + }); + expect(display.value).toBeUndefined(); + + source.value = stats({ + couchDB: couchDBProgress("continuous", 1201, true), + }); + expect(display.value).toBe(1201); + + source.value = stats({ + couchDB: couchDBProgress("one-shot", 0, true), + }); + expect(display.value).toBeUndefined(); + + source.value = stats(); + expect(display.value).toBeUndefined(); + display.dispose(); + }); + + it("clears its timer and subscription when disposed", () => { + const source = reactiveSource(stats({ couchDB: couchDBProgress("catch-up", 5, true) })); + const display = createFastForwardCheckedChangesDisplay(source); + + display.dispose(); + vi.advanceTimersByTime(FAST_FORWARD_COUNT_LINGER_MS); + source.value = stats({ couchDB: couchDBProgress("continuous", 6, true) }); + + expect(display.value).toBe(5); + }); + + it("does not display progress for providers without CouchDB replication state", () => { + const source = reactiveSource(stats()); + const display = createFastForwardCheckedChangesDisplay(source); + + expect(display.value).toBeUndefined(); + source.value = stats({ couchDB: { phase: "continuous" } }); + expect(display.value).toBeUndefined(); + display.dispose(); + }); +}); diff --git a/src/modules/features/StatusBarDisplay.ts b/src/modules/features/StatusBarDisplay.ts index 66fd10ed..9c24a972 100644 --- a/src/modules/features/StatusBarDisplay.ts +++ b/src/modules/features/StatusBarDisplay.ts @@ -10,6 +10,15 @@ export type DisposableReactiveValue = ReactiveValue & { dispose(): void; }; +export type PaddedCounterLabel = DisposableReactiveValue & { + readonly count: ReactiveValue; +}; + +export type ChunkFetchCounterLabel = DisposableReactiveValue & { + readonly initialCount: ReactiveValue; + readonly retryingCount: ReactiveValue; +}; + function asDisposableReactiveValue(value: ReactiveSource, dispose: () => void): DisposableReactiveValue { return { get value() { @@ -92,9 +101,10 @@ export function createPaddedCounterLabel( source: ReactiveValue, mark: string, inactiveLingerMs = STATUS_COUNTER_INACTIVE_LINGER_MS -): DisposableReactiveValue { +): PaddedCounterLabel { const linger = Math.max(0, inactiveLingerMs); const formatted = reactiveSource(""); + const displayedCount = reactiveSource(undefined); let maximumLength = 1; let clearTimer: number | undefined; let disposed = false; @@ -114,22 +124,39 @@ export function createPaddedCounterLabel( cancelClear(); const count = source.value; formatted.value = format(count); + displayedCount.value = count; if (count !== 0) return; clearTimer = compatGlobal.setTimeout(() => { clearTimer = undefined; if (disposed) return; formatted.value = ""; + displayedCount.value = undefined; maximumLength = 1; }, linger); }; source.onChanged(update); - return asDisposableReactiveValue(formatted, () => { + const display = asDisposableReactiveValue(formatted, () => { if (disposed) return; disposed = true; cancelClear(); source.offChanged(update); }); + return { + get value() { + return display.value; + }, + onChanged(handler) { + display.onChanged(handler); + }, + offChanged(handler) { + display.offChanged(handler); + }, + count: displayedCount, + dispose() { + display.dispose(); + }, + }; } /** @@ -138,7 +165,7 @@ export function createPaddedCounterLabel( */ export function createChunkFetchCounterLabel( source: ReactiveValue<{ initial: number; retrying: number }> -): DisposableReactiveValue { +): ChunkFetchCounterLabel { const initialCount = reactiveSource(0); const retryingCount = reactiveSource(0); const initialLabel = createPaddedCounterLabel(initialCount, "🛄"); @@ -168,7 +195,7 @@ export function createChunkFetchCounterLabel( source.onChanged(updateCounts); updateCounts(); - return asDisposableReactiveValue(formatted, () => { + const display = asDisposableReactiveValue(formatted, () => { if (disposed) return; disposed = true; source.offChanged(updateCounts); @@ -177,4 +204,20 @@ export function createChunkFetchCounterLabel( initialLabel.dispose(); retryingLabel.dispose(); }); + return { + get value() { + return display.value; + }, + onChanged(handler) { + display.onChanged(handler); + }, + offChanged(handler) { + display.offChanged(handler); + }, + initialCount: initialLabel.count, + retryingCount: retryingLabel.count, + dispose() { + display.dispose(); + }, + }; } diff --git a/src/modules/features/StatusBarDisplay.unit.spec.ts b/src/modules/features/StatusBarDisplay.unit.spec.ts index 777bf9cd..5ce8f69e 100644 --- a/src/modules/features/StatusBarDisplay.unit.spec.ts +++ b/src/modules/features/StatusBarDisplay.unit.spec.ts @@ -111,6 +111,24 @@ describe("createPaddedCounterLabel", () => { display.dispose(); }); + it("exposes the numeric value while retaining zero through the existing linger", () => { + const source = reactiveSource(0); + const display = createPaddedCounterLabel(source, "đŸ“Ĩ"); + + expect(display.count.value).toBeUndefined(); + source.value = 9; + expect(display.count.value).toBe(9); + source.value = 0; + expect(display.count.value).toBe(0); + + vi.advanceTimersByTime(STATUS_COUNTER_INACTIVE_LINGER_MS - 1); + expect(display.count.value).toBe(0); + vi.advanceTimersByTime(1); + expect(display.count.value).toBeUndefined(); + + display.dispose(); + }); + it("cancels the pending clear when counter activity resumes", () => { const source = reactiveSource(0); const display = createPaddedCounterLabel(source, "📄"); diff --git a/src/modules/features/StatusBarRenderer.ts b/src/modules/features/StatusBarRenderer.ts new file mode 100644 index 00000000..990cc9f5 --- /dev/null +++ b/src/modules/features/StatusBarRenderer.ts @@ -0,0 +1,151 @@ +export type StatusBarPart = { + key: string; + text: string; + value?: number; + suffix?: string; + title?: string; +}; + +export type StatusBarRenderer = { + render(parts: readonly StatusBarPart[]): void; + dispose(): void; +}; + +type RenderedItem = { + element: HTMLSpanElement; + prefix: Text; + value: HTMLSpanElement; + valueText: Text; + suffix: Text; + widestValueLength: number; +}; + +export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer { + const items = new Map(); + const managedElements = new Set(); + let disposed = false; + + const setText = (node: Text, text: string) => { + if (node.data !== text) node.data = text; + }; + + const setHidden = (element: HTMLElement, hidden: boolean) => { + if (element.hidden !== hidden) element.hidden = hidden; + }; + + const setWidth = (element: HTMLElement, width: string) => { + if (element.style.getPropertyValue("--livesync-stat-width") !== width) { + element.setCssProps({ "--livesync-stat-width": width }); + } + }; + + const createItem = (key: string): RenderedItem => { + const document = parent.ownerDocument; + const element = parent.createSpan(); + element.className = "livesync-stat-item livesync-stat-" + key; + + const prefix = document.createTextNode(""); + const value = element.createSpan(); + value.className = "livesync-stat-value"; + const valueText = document.createTextNode(""); + value.appendChild(valueText); + value.hidden = true; + const suffix = document.createTextNode(""); + + element.appendChild(prefix); + element.appendChild(value); + element.appendChild(suffix); + + const item = { element, prefix, value, valueText, suffix, widestValueLength: 0 }; + items.set(key, item); + managedElements.add(element); + return item; + }; + + const updateItem = (item: RenderedItem, part: StatusBarPart) => { + setText(item.prefix, part.text); + const title = part.title ?? ""; + if (item.element.title !== title) item.element.title = title; + + if (part.value === undefined) { + setText(item.valueText, ""); + setText(item.suffix, ""); + setHidden(item.value, true); + setWidth(item.value, ""); + item.widestValueLength = 0; + return; + } + + const formattedValue = String(part.value); + setText(item.valueText, formattedValue); + setText(item.suffix, part.suffix ?? ""); + setHidden(item.value, false); + + if (formattedValue.length > item.widestValueLength) { + item.widestValueLength = formattedValue.length; + const width = item.widestValueLength + "ch"; + setWidth(item.value, width); + } + }; + + const reorderIfNeeded = (visibleItems: readonly RenderedItem[]) => { + const currentOrder = Array.from(parent.childNodes).filter((node): node is HTMLSpanElement => + managedElements.has(node as HTMLSpanElement) + ); + const visibleElements = new Set(visibleItems.map((item) => item.element)); + const desiredOrder = [ + ...visibleItems.map((item) => item.element), + ...currentOrder.filter((element) => !visibleElements.has(element)), + ]; + if ( + currentOrder.length === desiredOrder.length && + currentOrder.every((node, index) => node === desiredOrder[index]) + ) { + return; + } + + let reference: ChildNode | null = currentOrder[0] ?? null; + for (const element of desiredOrder) { + if (element !== reference) parent.insertBefore(element, reference); + const childNodes = Array.from(parent.childNodes); + const elementIndex = childNodes.indexOf(element); + reference = + childNodes.slice(elementIndex + 1).find((node) => managedElements.has(node as HTMLSpanElement)) ?? null; + } + }; + + return { + render(parts) { + if (disposed) return; + + const visibleItems: RenderedItem[] = []; + const visibleKeys = new Set(); + for (const part of parts) { + let item = items.get(part.key); + if (!item) item = createItem(part.key); + updateItem(item, part); + setHidden(item.element, false); + visibleItems.push(item); + visibleKeys.add(part.key); + } + + for (const [key, item] of items) { + if (visibleKeys.has(key)) continue; + setHidden(item.element, true); + setWidth(item.value, ""); + item.widestValueLength = 0; + } + + reorderIfNeeded(visibleItems); + }, + dispose() { + if (disposed) return; + disposed = true; + for (const item of items.values()) { + if (item.element.parentNode === parent) parent.removeChild(item.element); + } + items.clear(); + managedElements.clear(); + }, + }; +} diff --git a/src/modules/features/StatusBarRenderer.unit.spec.ts b/src/modules/features/StatusBarRenderer.unit.spec.ts new file mode 100644 index 00000000..1f1be039 --- /dev/null +++ b/src/modules/features/StatusBarRenderer.unit.spec.ts @@ -0,0 +1,300 @@ +import { describe, expect, it } from "vitest"; + +import { createStatusBarRenderer, type StatusBarPart } from "./StatusBarRenderer.ts"; + +type TestNode = TestElement | TestTextNode; + +class TestDocument { + mutations = 0; + + createElement(tagName: string): TestElement { + return new TestElement(this, tagName); + } + + createTextNode(data: string): TestTextNode { + return new TestTextNode(this, data); + } +} + +class TestTextNode { + readonly nodeType = 3; + parentNode: TestElement | undefined; + private currentData: string; + + constructor( + private readonly document: TestDocument, + data: string + ) { + this.currentData = data; + } + + get data(): string { + return this.currentData; + } + + set data(value: string) { + this.document.mutations++; + this.currentData = value; + } + + get textContent(): string { + return this.currentData; + } +} + +class TestStyle { + private properties = new Map(); + + constructor(private readonly document: TestDocument) {} + + getPropertyValue(name: string): string { + return this.properties.get(name) ?? ""; + } + + setProperty(name: string, value: string) { + this.document.mutations++; + this.properties.set(name, value); + } +} + +class TestElement { + readonly nodeType = 1; + readonly ownerDocument: Document; + readonly childNodes: TestNode[] = []; + parentNode: TestElement | undefined; + readonly style: TestStyle; + private currentClassName = ""; + private currentHidden = false; + private currentTitle = ""; + + constructor( + private readonly document: TestDocument, + readonly tagName: string + ) { + this.ownerDocument = document as unknown as Document; + this.style = new TestStyle(document); + } + + get className(): string { + return this.currentClassName; + } + + set className(value: string) { + this.document.mutations++; + this.currentClassName = value; + } + + get hidden(): boolean { + return this.currentHidden; + } + + set hidden(value: boolean) { + this.document.mutations++; + this.currentHidden = value; + } + + get title(): string { + return this.currentTitle; + } + + set title(value: string) { + this.document.mutations++; + this.currentTitle = value; + } + + get children(): TestElement[] { + return this.childNodes.filter((node): node is TestElement => node instanceof TestElement); + } + + get textContent(): string { + return this.childNodes.map((node) => node.textContent ?? "").join(""); + } + + createSpan(): TestElement { + return this.appendChild(this.document.createElement("span")); + } + + setCssProps(properties: Record) { + for (const [name, value] of Object.entries(properties)) this.style.setProperty(name, value); + } + + appendChild(node: T): T { + if (node.parentNode) { + node.parentNode.removeChild(node); + } + this.childNodes.push(node); + node.parentNode = this; + this.document.mutations++; + return node; + } + + insertBefore(node: T, reference: TestNode | null): T { + if (node === reference) return node; + if (node.parentNode) { + node.parentNode.removeChild(node); + } + const index = reference ? this.childNodes.indexOf(reference) : -1; + if (index < 0) { + this.childNodes.push(node); + } else { + this.childNodes.splice(index, 0, node); + } + node.parentNode = this; + this.document.mutations++; + return node; + } + + removeChild(node: T): T { + const index = this.childNodes.indexOf(node); + if (index < 0) throw new Error("Node is not a child"); + this.childNodes.splice(index, 1); + node.parentNode = undefined; + this.document.mutations++; + return node; + } +} + +function createFixture() { + const document = new TestDocument(); + const root = new TestElement(document, "div"); + return { document, root, parent: root as unknown as HTMLElement }; +} + +function findItem(root: TestElement, key: string): TestElement { + const item = root.children.find((child) => child.className.split(" ").includes("livesync-stat-" + key)); + if (!item) throw new Error("Missing item: " + key); + return item; +} + +function findValue(item: TestElement): TestElement { + const value = item.children.find((child) => child.className.split(" ").includes("livesync-stat-value")); + if (!value) throw new Error("Missing numeric value"); + return value; +} + +function visibleItems(root: TestElement): TestElement[] { + return root.children.filter((child) => !child.hidden && child.className.split(" ").includes("livesync-stat-item")); +} + +describe("createStatusBarRenderer", () => { + it("renders complete text safely and gives each item an independent class hook", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + + renderer.render([ + { key: "sync", text: "" }, + { key: "queue", text: " (queued: ", value: 2, suffix: ")" }, + ]); + + const sync = findItem(root, "sync"); + const queue = findItem(root, "queue"); + expect(sync.className.split(" ")).toEqual(["livesync-stat-item", "livesync-stat-sync"]); + expect(sync.textContent).toBe(""); + expect(queue.className.split(" ")).toContain("livesync-stat-queue"); + expect(queue.textContent).toBe(" (queued: 2)"); + expect(findValue(queue).className).toBe("livesync-stat-value"); + + renderer.render([{ key: "queue", text: " (queued: ", value: 3, suffix: ")" }]); + expect(sync.hidden).toBe(true); + expect(queue.hidden).toBe(false); + expect(queue.textContent).toBe(" (queued: 3)"); + renderer.render([{ key: "queue", text: "Queue idle" }]); + expect(queue.textContent).toBe("Queue idle"); + expect(findValue(queue).hidden).toBe(true); + renderer.dispose(); + }); + + it("does not mutate the DOM when the same parts are rendered again", () => { + const { document, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + const parts: StatusBarPart[] = [ + { key: "pending", text: " pending ", value: 7, suffix: " files", title: "Pending" }, + ]; + + renderer.render(parts); + const mutations = document.mutations; + renderer.render(parts); + + expect(document.mutations).toBe(mutations); + renderer.dispose(); + }); + + it("updates values while retaining the widest visible numeric width", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + + renderer.render([{ key: "count", text: "(", value: 9, suffix: ")" }]); + const item = findItem(root, "count"); + const value = findValue(item); + expect(item.textContent).toBe("(9)"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("1ch"); + + renderer.render([{ key: "count", text: "(", value: 123, suffix: ")" }]); + expect(value.textContent).toBe("123"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("3ch"); + + renderer.render([{ key: "count", text: "(", value: 4, suffix: ")" }]); + expect(value.textContent).toBe("4"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("3ch"); + renderer.dispose(); + }); + + it("resets numeric width while omitted and reuses the same item when it returns", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + + renderer.render([{ key: "count", text: " ", value: 1234 }]); + const item = findItem(root, "count"); + const value = findValue(item); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); + + renderer.render([]); + expect(item.hidden).toBe(true); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe(""); + + renderer.render([{ key: "count", text: " ", value: 5 }]); + expect(findItem(root, "count")).toBe(item); + expect(item.hidden).toBe(false); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("1ch"); + renderer.dispose(); + }); + + it("reorders existing nodes without replacing them", () => { + const { root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + renderer.render([ + { key: "first", text: "first " }, + { key: "second", text: "second " }, + ]); + const first = findItem(root, "first"); + const second = findItem(root, "second"); + + renderer.render([ + { key: "second", text: "second " }, + { key: "first", text: "first " }, + ]); + + expect(visibleItems(root)).toEqual([second, first]); + expect(findItem(root, "first")).toBe(first); + expect(findItem(root, "second")).toBe(second); + renderer.dispose(); + }); + + it("removes its nodes on dispose and ignores later renders", () => { + const { root, parent } = createFixture(); + const externalNode = new TestElement(new TestDocument(), "strong"); + root.appendChild(externalNode); + const renderer = createStatusBarRenderer(parent); + renderer.render([{ key: "sync", text: "sync" }]); + const item = findItem(root, "sync"); + + renderer.dispose(); + const childCountAfterDispose = root.childNodes.length; + renderer.dispose(); + renderer.render([{ key: "later", text: "later" }]); + + expect(item.parentNode).toBeUndefined(); + expect(root.childNodes).toEqual([externalNode]); + expect(root.childNodes).toHaveLength(childCountAfterDispose); + }); +}); diff --git a/styles.css b/styles.css index 462eab07..4a4b0d61 100644 --- a/styles.css +++ b/styles.css @@ -839,3 +839,18 @@ body.is-mobile .livesync-compatibility-review-notice { outline-offset: 1px; border-radius: 2px; } + +.livesync-stat-item { + font-variant-numeric: tabular-nums; +} + +.livesync-stat-value { + display: inline-block; + width: var(--livesync-stat-width, auto); + text-align: right; + font-variant-numeric: tabular-nums; +} + +.livesync-stat-value[hidden] { + display: none; +} diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 5f04c8c8..a13018fb 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -83,10 +83,13 @@ npm run test:e2e:obsidian:focused -- stale-file-restart npm run test:e2e:obsidian:focused -- folder-batch npm run test:e2e:obsidian:focused -- security-seed-reconnect npm run test:e2e:obsidian:focused -- couchdb-http-cache +npm run test:e2e:obsidian:focused -- replication-status ``` The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. +`replication-status` uses an isolated Vault without a remote service. It injects synthetic replication statistics into the real plug-in to check the ⏊ icon, the comparison count and its three-second expiry, independently styled numeric items, stable rendered digit widths, and unchanged DOM nodes for repeated values. It switches between split panes, opens a popout editor, checks log updates there, and returns to the main window. Where Electron exposes native window control, it also hides the main window and verifies that popout logs still update before restoring it. This checks display behaviour; Commonlib owns the tests which establish that PouchDB produces the underlying statistics. + `database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, the default-on integrity-only remote option, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout at 800×600 and 375×667, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. The same scenario checks the five actions in Reporting Tools, the separate Checks and Recovery panel, Metadata progress with its document total and gauge, and collapsible Markdown. It also observes Usage revision progress with a count and discovery note during a gated read. Local lookup checks cover multiple document IDs for one path, selection of an exact ID, and a follow-up report using the original salt. Closing the follow-up dialogue during a pending Metadata read must abort the report after that read settles, without querying Chunk availability. The compact Markdown must retain every JSON observation reference and pseudonym, and its copy and save payloads must match the full output. Results and export controls are checked at 800×600 on desktop and 375×667 in mobile emulation, including after copy and save feedback appears. diff --git a/test/e2e-obsidian/scripts/replication-status.ts b/test/e2e-obsidian/scripts/replication-status.ts new file mode 100644 index 00000000..1ef380a1 --- /dev/null +++ b/test/e2e-obsidian/scripts/replication-status.ts @@ -0,0 +1,770 @@ +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; +import type { Page } from "playwright"; + +const fixturePath = "replication-status-e2e.md"; +const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_UI_TIMEOUT_MS ?? 10000); +const fastForwardLingerMs = 3000; +const numericKeys = ["network", "sent", "push-remaining", "arrived", "pull-remaining", "ff"] as const; + +type TestReplicationStatics = { + sent: number; + arrived: number; + maxPullSeq: number; + maxPushSeq: number; + lastSyncPullSeq: number; + lastSyncPushSeq: number; + syncStatus: "CONNECTED"; + couchDB: { + phase: "continuous"; + pull: { checkedChanges: number; completedBatches: number; checking: boolean }; + push: { checkedChanges: number; completedBatches: number; checking: boolean }; + }; +}; + +type WritableSource = { value: T }; +type TestLeaf = { + containerEl?: HTMLElement; + view?: { containerEl?: HTMLElement }; + openFile?: (file: unknown) => Promise; + detach?: () => void; +}; +type TestWorkspace = { + activeLeaf?: TestLeaf | null; + getLeaf?: (newLeaf?: boolean | "split" | "tab" | "window", direction?: "horizontal" | "vertical") => TestLeaf; + openPopoutLeaf?: () => TestLeaf; + setActiveLeaf?: (leaf: TestLeaf, params?: { focus?: boolean }) => void; + iterateAllLeaves?: (callback: (leaf: TestLeaf) => void) => void; + getActiveFile?: () => { path?: string } | null; +}; +type TestLogModule = { + statusLog?: WritableSource; + statusDiv?: HTMLElement; + statusLine?: HTMLElement; + statusLineRenderer?: { render?: (parts: readonly unknown[]) => void }; + statusBarLabels?: WritableSource<{ parts: readonly unknown[] }>; + __addLog?: (message: unknown, level?: number, key?: string) => void; +}; +type TestCore = { + modules?: unknown[]; + services?: { + setting?: { currentSettings?: () => Record }; + replicator?: { + replicationStatics?: WritableSource; + }; + API?: { + requestCount?: WritableSource; + responseCount?: WritableSource; + }; + }; +}; +type TestApp = { + plugins?: { + plugins?: { "obsidian-livesync"?: { core?: TestCore } }; + }; + vault?: { + getAbstractFileByPath?: (path: string) => unknown | null; + create?: (path: string, content: string) => Promise; + }; + workspace?: TestWorkspace; +}; +type TestWindow = typeof globalThis & { + app?: TestApp; + require?: (moduleName: string) => unknown; + __livesyncStatusFixtureFile?: unknown; + __livesyncStatusFirstLeaf?: TestLeaf; + __livesyncStatusSplitLeaf?: TestLeaf; + __livesyncStatusPopoutLeaf?: TestLeaf; + __livesyncStatusNode?: HTMLElement; +}; +type MainWindowControl = { + hidden: boolean; + method?: string; + reason?: string; + restore(): Promise; +}; + +function failUnless(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function replicationStats(checkedChanges: number, sent: number, arrived: number): TestReplicationStatics { + return { + sent, + arrived, + maxPullSeq: sent + 1, + maxPushSeq: sent + 1, + lastSyncPullSeq: 1, + lastSyncPushSeq: 1, + syncStatus: "CONNECTED", + couchDB: { + phase: "continuous", + pull: { checkedChanges, completedBatches: 0, checking: true }, + push: { checkedChanges: 0, completedBatches: 0, checking: false }, + }, + }; +} + +async function installFixtureFile(page: Page): Promise { + const result = await page.evaluate(async (path) => { + const app = (globalThis as TestWindow).app; + const workspace = app?.workspace; + const vault = app?.vault; + if (!workspace?.getLeaf || !workspace.setActiveLeaf || !vault?.create || !vault.getAbstractFileByPath) { + return { ready: false, reason: "Obsidian workspace APIs are unavailable." }; + } + let file = vault.getAbstractFileByPath(path); + if (!file) file = await vault.create(path, "Replication status renderer fixture.\n"); + const leaf = workspace.getLeaf(false); + if (!leaf.openFile) return { ready: false, reason: "The active workspace leaf cannot open a file." }; + await leaf.openFile(file); + workspace.setActiveLeaf(leaf, { focus: true }); + (globalThis as TestWindow).__livesyncStatusFixtureFile = file; + (globalThis as TestWindow).__livesyncStatusFirstLeaf = leaf; + return { ready: true }; + }, fixturePath); + if (!result.ready) + throw new Error("reason" in result ? result.reason : "Could not initialise the fixture editor leaf."); +} + +async function injectReplicationStats( + page: Page, + stats: TestReplicationStatics, + requestCount: number, + responseCount: number +): Promise { + const result = await page.evaluate( + ({ stats, requestCount, responseCount }) => { + const core = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const replicator = core?.services?.replicator; + const api = core?.services?.API; + if (!replicator?.replicationStatics || !api?.requestCount || !api.responseCount) { + return { ready: false, reason: "LiveSync replication status sources are unavailable." }; + } + replicator.replicationStatics.value = stats; + api.requestCount.value = requestCount; + api.responseCount.value = responseCount; + return { ready: true }; + }, + { stats, requestCount, responseCount } + ); + if (!result.ready) + throw new Error("reason" in result ? result.reason : "Could not inject replication status fixtures."); +} + +async function appendLiveSyncLog(page: Page, message: string): Promise { + const result = await page.evaluate((message) => { + const core = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const modules = core?.modules ?? []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + const candidate = entry as TestLogModule; + return typeof candidate.__addLog === "function" && candidate.statusLog !== undefined; + }); + if (!moduleLog?.__addLog) return { ready: false, reason: "LiveSync logging is unavailable." }; + moduleLog.__addLog(message); + return { ready: true }; + }, message); + if (!result.ready) throw new Error("reason" in result ? result.reason : "Could not append a LiveSync log entry."); +} + +async function waitForPartText(page: Page, key: string, expected: string): Promise { + const selector = `.livesync-status-statusline .livesync-stat-${key} .livesync-stat-value`; + await page.waitForFunction( + ({ selector, expected }) => document.querySelector(selector)?.textContent?.trim() === expected, + { selector, expected }, + { timeout: uiTimeoutMs } + ); +} + +async function waitForLogMessage(page: Page, expected: string, timeout = uiTimeoutMs): Promise { + await page.waitForFunction( + (message) => document.querySelector(".livesync-status-logmessage")?.textContent?.includes(message) === true, + expected, + { timeout } + ); +} + +async function assertSingleOverlay(contextPages: Page[], expectedCount = 1): Promise { + const counts = await Promise.all( + contextPages.map( + async (page) => + await page + .locator(".livesync-status") + .count() + .catch(() => 0) + ) + ); + const actualCount = counts.reduce((total, count) => total + count, 0); + failUnless( + actualCount === expectedCount, + `Expected ${expectedCount} LiveSync status overlay across Obsidian windows, found ${actualCount} (${counts.join(", ")}).` + ); +} + +async function waitForOverlayOnActiveLeaf(page: Page): Promise { + await page.waitForFunction( + () => { + const host = globalThis as TestWindow; + const status = host.__livesyncStatusNode; + const activeLeaf = host.app?.workspace?.activeLeaf; + return ( + status !== undefined && + document.querySelectorAll(".livesync-status").length === 1 && + document.querySelector(".livesync-status") === status && + activeLeaf?.view?.containerEl?.contains(status) === true + ); + }, + undefined, + { timeout: uiTimeoutMs } + ); +} + +async function getStatusDiagnostics(page: Page): Promise> { + return await page.evaluate(() => { + const app = (globalThis as TestWindow).app; + const core = app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const moduleLog = core?.modules?.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + const candidate = entry as TestLogModule; + return candidate.statusDiv !== undefined || candidate.statusLineRenderer !== undefined; + }); + const currentSettings = core?.services?.setting?.currentSettings?.(); + const status = document.querySelector(".livesync-status"); + const statusParent = status?.parentElement; + const activeLeaf = app?.workspace?.activeLeaf; + const recentLeaf = app?.workspace?.getLeaf?.(false); + const statusStyle = status ? getComputedStyle(status) : undefined; + const statusParentStyle = statusParent ? getComputedStyle(statusParent) : undefined; + const activeLeafContainer = activeLeaf?.view?.containerEl; + const recentLeafContainer = recentLeaf?.view?.containerEl; + const activeLeafStyle = activeLeafContainer ? getComputedStyle(activeLeafContainer) : undefined; + const recentLeafStyle = recentLeafContainer ? getComputedStyle(recentLeafContainer) : undefined; + return { + settings: currentSettings + ? { + isConfigured: currentSettings.isConfigured, + showStatusOnEditor: currentSettings.showStatusOnEditor, + showStatusOnStatusbar: currentSettings.showStatusOnStatusbar, + } + : undefined, + statusCount: document.querySelectorAll(".livesync-status").length, + status: status + ? { + tag: status.tagName, + className: status.className, + display: statusStyle?.display, + visibility: statusStyle?.visibility, + } + : undefined, + statusParent: statusParent + ? { + tag: statusParent.tagName, + className: statusParent.className, + display: statusParentStyle?.display, + visibility: statusParentStyle?.visibility, + } + : undefined, + moduleHasStatusDiv: moduleLog?.statusDiv !== undefined, + moduleStatusLine: moduleLog?.statusLine + ? { + tag: moduleLog.statusLine.tagName, + className: moduleLog.statusLine.className, + } + : undefined, + activeFile: app?.workspace?.getActiveFile?.()?.path, + activeLeafContainer: activeLeafContainer + ? { + tag: activeLeafContainer.tagName, + className: activeLeafContainer.className, + display: activeLeafStyle?.display, + visibility: activeLeafStyle?.visibility, + } + : undefined, + recentLeafContainer: recentLeafContainer + ? { + tag: recentLeafContainer.tagName, + className: recentLeafContainer.className, + display: recentLeafStyle?.display, + visibility: recentLeafStyle?.visibility, + } + : undefined, + statusInActiveLeaf: activeLeafContainer?.contains(status) ?? false, + statusInRecentLeaf: recentLeafContainer?.contains(status) ?? false, + }; + }); +} + +async function waitForPopoutPage(page: Page, marker: string): Promise { + const deadline = Date.now() + uiTimeoutMs; + const selector = `[data-e2e-status-popout="${marker}"]`; + while (Date.now() < deadline) { + for (const candidate of page.context().pages()) { + if ( + (await candidate + .locator(selector) + .count() + .catch(() => 0)) > 0 + ) + return candidate; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`Obsidian did not expose the popout renderer containing ${selector}.`); +} + +async function createPopoutLeaf(page: Page, marker: string): Promise { + const result = await page.evaluate(async (marker) => { + const host = globalThis as TestWindow; + const app = host.app; + const workspace = app?.workspace; + const file = host.__livesyncStatusFixtureFile; + if (!workspace?.openPopoutLeaf || !workspace.setActiveLeaf || file === undefined) { + return { ready: false, reason: "Obsidian popout workspace APIs are unavailable." }; + } + const leaf = workspace.openPopoutLeaf(); + if (!leaf.openFile || !leaf.containerEl) return { ready: false, reason: "The popout leaf is incomplete." }; + leaf.containerEl.dataset.e2eStatusPopout = marker; + await leaf.openFile(file); + workspace.setActiveLeaf(leaf, { focus: true }); + host.__livesyncStatusPopoutLeaf = leaf; + return { ready: true }; + }, marker); + if (!result.ready) throw new Error("reason" in result ? result.reason : "Could not open an Obsidian popout leaf."); +} + +async function switchToLeaf(page: Page, leafName: "first" | "split"): Promise { + const result = await page.evaluate((leafName) => { + const host = globalThis as TestWindow; + const workspace = host.app?.workspace; + const leaf = leafName === "first" ? host.__livesyncStatusFirstLeaf : host.__livesyncStatusSplitLeaf; + if (!workspace?.setActiveLeaf || !leaf) return { ready: false }; + workspace.setActiveLeaf(leaf, { focus: true }); + return { ready: true }; + }, leafName); + failUnless(result.ready, `Could not activate the ${leafName} editor leaf.`); + await waitForOverlayOnActiveLeaf(page); +} + +async function createSplitLeaf(page: Page): Promise { + const result = await page.evaluate(async () => { + const host = globalThis as TestWindow; + const workspace = host.app?.workspace; + const file = host.__livesyncStatusFixtureFile; + if (!workspace?.getLeaf || !workspace.setActiveLeaf || file === undefined) { + return { ready: false, reason: "Obsidian split workspace APIs are unavailable." }; + } + const leaf = workspace.getLeaf("split", "vertical"); + if (!leaf.openFile) return { ready: false, reason: "The split leaf cannot open a file." }; + await leaf.openFile(file); + workspace.setActiveLeaf(leaf, { focus: true }); + host.__livesyncStatusSplitLeaf = leaf; + return { ready: true }; + }); + if (!result.ready) throw new Error("reason" in result ? result.reason : "Could not create a split editor leaf."); +} + +async function observeRepeatedRendering(page: Page): Promise<{ mutationCount: number; elapsedMs: number }> { + return await page.evaluate(async () => { + const core = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core; + const modules = core?.modules ?? []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + const candidate = entry as TestLogModule; + return ( + typeof candidate.statusLineRenderer?.render === "function" && candidate.statusBarLabels !== undefined + ); + }); + const statusLine = document.querySelector(".livesync-status-statusline"); + if (!moduleLog?.statusLineRenderer?.render || !moduleLog.statusBarLabels || !statusLine) { + throw new Error("The LiveSync status renderer is unavailable for the idempotence observation."); + } + let mutationCount = 0; + const observer = new MutationObserver((records) => { + mutationCount += records.length; + }); + observer.observe(statusLine, { attributes: true, characterData: true, childList: true, subtree: true }); + const parts = moduleLog.statusBarLabels.value.parts; + const startedAt = performance.now(); + for (let iteration = 0; iteration < 1000; iteration++) { + moduleLog.statusLineRenderer.render(parts); + } + const elapsedMs = performance.now() - startedAt; + await new Promise((resolve) => setTimeout(resolve, 0)); + observer.disconnect(); + return { mutationCount, elapsedMs }; + }); +} + +async function measureNumericWidths(page: Page): Promise> { + const measurements = await page.evaluate((keys) => { + const result: Record = {}; + const statusLine = document.querySelector(".livesync-status-statusline"); + if (!statusLine) throw new Error("The LiveSync editor status line is unavailable."); + for (const key of keys) { + const value = statusLine.querySelector(`.livesync-stat-${key} .livesync-stat-value`); + if (!value || value.hidden) throw new Error(`The numeric status item ${key} is not visible.`); + if (getComputedStyle(value).fontVariantNumeric !== "tabular-nums") { + throw new Error(`The numeric status item ${key} does not use tabular figures.`); + } + result[key] = value.getBoundingClientRect().width; + } + return result; + }, numericKeys); + return measurements as Record<(typeof numericKeys)[number], number>; +} + +async function assertNumericWidths( + page: Page, + expectedValue: string +): Promise> { + for (const key of numericKeys) await waitForPartText(page, key, expectedValue); + const widths = await page.evaluate((keys) => { + const result: Record = {}; + const statusLine = document.querySelector(".livesync-status-statusline"); + if (!statusLine) throw new Error("The LiveSync editor status line is unavailable."); + for (const key of keys) { + const value = statusLine.querySelector(`.livesync-stat-${key} .livesync-stat-value`); + if (!value || value.hidden) throw new Error(`The numeric status item ${key} is not visible.`); + result[key] = { + width: value.style.getPropertyValue("--livesync-stat-width"), + value: value.textContent?.trim() ?? "", + }; + } + return result; + }, numericKeys); + for (const key of numericKeys) { + failUnless(widths[key]?.value === expectedValue, `Status item ${key} did not render ${expectedValue}.`); + failUnless(widths[key]?.width === "4ch", `Status item ${key} did not retain a 4ch width.`); + } + return await measureNumericWidths(page); +} + +async function measureStatusBarNetworkWidth( + page: Page, + expectedValue: string +): Promise<{ cssWidth: string; renderedWidth: number }> { + const selector = ".syncstatusbar .livesync-stat-network .livesync-stat-value"; + await page.waitForFunction( + ({ selector, expectedValue }) => { + const value = document.querySelector(selector); + return value !== null && !value.hidden && value.textContent?.trim() === expectedValue; + }, + { selector, expectedValue }, + { timeout: uiTimeoutMs } + ); + return await page.locator(selector).evaluate((element) => { + const value = element as HTMLElement; + return { + cssWidth: value.style.getPropertyValue("--livesync-stat-width"), + renderedWidth: value.getBoundingClientRect().width, + }; + }); +} + +async function hideMainWindow(page: Page): Promise { + const nativeWindow = await page.evaluate((action) => { + type ElectronWindow = { + isVisible?: () => boolean; + hide?: () => void; + show?: () => void; + focus?: () => void; + }; + type RemoteModule = { getCurrentWindow?: () => ElectronWindow }; + const host = globalThis as TestWindow; + let electronModule: unknown; + try { + electronModule = host.require?.("electron"); + } catch { + electronModule = undefined; + } + const electron = electronModule as { remote?: RemoteModule } | undefined; + let remoteModule: unknown = electron?.remote; + if (!remoteModule) { + try { + remoteModule = host.require?.("@electron/remote"); + } catch { + remoteModule = undefined; + } + } + const remote = remoteModule as RemoteModule | undefined; + const currentWindow = remote?.getCurrentWindow?.(); + if (!currentWindow?.isVisible || !currentWindow.hide || !currentWindow.show) { + return { + available: false, + visible: false, + changed: false, + reason: "Electron remote window controls are unavailable.", + }; + } + if (action === "hide") { + if (!currentWindow.isVisible()) { + return { + available: true, + visible: false, + changed: false, + reason: "The main Electron window was already hidden.", + }; + } + currentWindow.hide(); + const visible = currentWindow.isVisible(); + return { available: true, visible, changed: !visible }; + } + currentWindow.show(); + currentWindow.focus?.(); + return { available: true, visible: currentWindow.isVisible(), changed: false }; + }, "hide"); + if (nativeWindow.available && nativeWindow.changed && !nativeWindow.visible) { + return { + hidden: true, + method: "Electron remote", + async restore() { + const restored = await page.evaluate(() => { + type ElectronWindow = { + isVisible?: () => boolean; + hide?: () => void; + show?: () => void; + focus?: () => void; + }; + type RemoteModule = { getCurrentWindow?: () => ElectronWindow }; + const host = globalThis as TestWindow; + let electronModule: unknown; + try { + electronModule = host.require?.("electron"); + } catch { + electronModule = undefined; + } + const electron = electronModule as { remote?: RemoteModule } | undefined; + let remoteModule: unknown = electron?.remote; + if (!remoteModule) { + try { + remoteModule = host.require?.("@electron/remote"); + } catch { + remoteModule = undefined; + } + } + const remote = remoteModule as RemoteModule | undefined; + const currentWindow = remote?.getCurrentWindow?.(); + if (!currentWindow?.show) return false; + currentWindow.show(); + currentWindow.focus?.(); + return currentWindow.isVisible?.() === true; + }); + if (!restored) throw new Error("The hidden main Electron window could not be restored."); + }, + }; + } + return { + hidden: false, + reason: nativeWindow.reason ?? "Electron remote did not hide the main window.", + restore: async () => {}, + }; +} + +async function runStatusScenario(session: ObsidianLiveSyncSession): Promise { + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + let popoutMarker: string | undefined; + let popoutPage: Page | undefined; + let mainWindow: MainWindowControl | undefined; + try { + await installFixtureFile(page); + try { + await page.locator(".livesync-status").waitFor({ state: "attached", timeout: uiTimeoutMs }); + } catch (error) { + const diagnostics = await getStatusDiagnostics(page); + throw new Error( + `${error instanceof Error ? error.message : String(error)}\nLiveSync status setup: ${JSON.stringify(diagnostics)}` + ); + } + const setup = await getStatusDiagnostics(page); + const settings = setup.settings as { showStatusOnEditor?: unknown } | undefined; + failUnless( + setup.moduleHasStatusDiv === true && settings?.showStatusOnEditor === true, + `LiveSync editor status is not enabled in the isolated vault: ${JSON.stringify(setup)}` + ); + await injectReplicationStats(page, replicationStats(0, 0, 0), 0, 0); + try { + await page.locator(".livesync-status").waitFor({ state: "visible", timeout: uiTimeoutMs }); + } catch (error) { + const diagnostics = await getStatusDiagnostics(page); + throw new Error( + `${error instanceof Error ? error.message : String(error)}\nLiveSync status setup: ${JSON.stringify(diagnostics)}` + ); + } + await page.evaluate(() => { + const host = globalThis as TestWindow; + const status = document.querySelector(".livesync-status"); + if (!status) throw new Error("LiveSync editor status overlay was not created."); + host.__livesyncStatusNode = status; + }); + await waitForOverlayOnActiveLeaf(page); + + const initialStats = replicationStats(1000, 1000, 0); + const startedAt = Date.now(); + await injectReplicationStats(page, initialStats, 0, 0); + await waitForPartText(page, "arrived", "0"); + await waitForPartText(page, "ff", "1000"); + const syncIcon = await page.locator(".livesync-status-statusline .livesync-stat-sync").textContent(); + failUnless(syncIcon?.includes("⏊"), "The fast-forward checking icon was not independent of arrivals."); + + for (let update = 1; update <= 3; update++) { + await page.waitForTimeout(700); + await injectReplicationStats(page, replicationStats(1000, 1000 + update, 0), 0, 0); + } + const remainingLinger = Math.max(0, fastForwardLingerMs + 350 - (Date.now() - startedAt)); + if (remainingLinger > 0) await page.waitForTimeout(remainingLinger); + await page.waitForFunction( + () => + document.querySelector(".livesync-status-statusline .livesync-stat-ff")?.hidden === + true, + undefined, + { timeout: uiTimeoutMs } + ); + const iconAfterExpiry = await page.locator(".livesync-status-statusline .livesync-stat-sync").textContent(); + failUnless( + iconAfterExpiry?.includes("⏊"), + "The fast-forward checking icon disappeared when its count expired." + ); + + await injectReplicationStats(page, replicationStats(1001, 1001, 1001), 1001, 0); + const wideWidths = await assertNumericWidths(page, "1001"); + const wideStatusBarWidth = await measureStatusBarNetworkWidth(page, "1001"); + failUnless(wideStatusBarWidth.cssWidth === "4ch", "The status-bar network count did not reserve 4ch."); + await injectReplicationStats(page, replicationStats(7, 7, 7), 7, 0); + const narrowWidths = await assertNumericWidths(page, "7"); + const narrowStatusBarWidth = await measureStatusBarNetworkWidth(page, "7"); + for (const key of numericKeys) { + failUnless( + Math.abs(wideWidths[key] - narrowWidths[key]) < 0.01, + `Status item ${key} changed rendered width from ${wideWidths[key]}px to ${narrowWidths[key]}px.` + ); + } + failUnless( + narrowStatusBarWidth.cssWidth === "4ch" && + Math.abs(wideStatusBarWidth.renderedWidth - narrowStatusBarWidth.renderedWidth) < 0.01, + `The status-bar network count changed width from ${wideStatusBarWidth.renderedWidth}px to ${narrowStatusBarWidth.renderedWidth}px.` + ); + + const repeatedRender = await observeRepeatedRendering(page); + failUnless( + repeatedRender.mutationCount === 0, + `Repeated identical status rendering caused ${repeatedRender.mutationCount} DOM mutations.` + ); + console.log( + `Identical status render: 1000 iterations, 0 DOM mutations, ${repeatedRender.elapsedMs.toFixed(2)} ms.` + ); + + await createSplitLeaf(page); + await assertSingleOverlay(page.context().pages()); + await waitForOverlayOnActiveLeaf(page); + await switchToLeaf(page, "first"); + await assertSingleOverlay(page.context().pages()); + await switchToLeaf(page, "split"); + await assertSingleOverlay(page.context().pages()); + await switchToLeaf(page, "first"); + await assertSingleOverlay(page.context().pages()); + + popoutMarker = `livesync-status-${process.pid}-${Date.now()}`; + await createPopoutLeaf(page, popoutMarker); + popoutPage = await waitForPopoutPage(page, popoutMarker); + await popoutPage.locator(".livesync-status").waitFor({ state: "visible", timeout: uiTimeoutMs }); + await assertSingleOverlay(page.context().pages()); + await popoutPage.waitForFunction( + (marker) => { + const leaf = document.querySelector(`[data-e2e-status-popout="${marker}"]`); + const status = document.querySelector(".livesync-status"); + return leaf !== null && status !== null && leaf.contains(status); + }, + popoutMarker, + { timeout: uiTimeoutMs } + ); + + const popoutLog = `E2E popout log ${Date.now()}`; + await appendLiveSyncLog(page, popoutLog); + await waitForLogMessage(popoutPage, popoutLog); + + mainWindow = await hideMainWindow(page); + if (!mainWindow.hidden) { + console.warn( + `Skipped hidden-main-window assertion: ${mainWindow.reason ?? "minimisation unavailable"}` + ); + } else { + await popoutPage.bringToFront(); + const hiddenWindowLog = `E2E hidden-main log ${Date.now()}`; + await appendLiveSyncLog(page, hiddenWindowLog); + await waitForLogMessage(popoutPage, hiddenWindowLog, Math.min(uiTimeoutMs, 2500)); + console.log(`Popout log updated while the main window was hidden through ${mainWindow.method}.`); + } + await mainWindow.restore(); + mainWindow = undefined; + await page.bringToFront(); + await switchToLeaf(page, "first"); + await assertSingleOverlay(page.context().pages()); + await waitForOverlayOnActiveLeaf(page); + failUnless( + await page.evaluate( + () => (globalThis as TestWindow).__livesyncStatusNode === document.querySelector(".livesync-status") + ), + "The LiveSync status overlay node was replaced while moving between editor windows." + ); + const returnedLog = `E2E main-window log ${Date.now()}`; + await appendLiveSyncLog(page, returnedLog); + await waitForLogMessage(page, returnedLog); + console.log("Replication status, split-pane ownership, and popout status/log checks passed."); + } finally { + if (mainWindow) await mainWindow.restore(); + await page.bringToFront().catch(() => {}); + if (popoutMarker !== undefined) { + await page + .evaluate((marker) => { + const workspace = (globalThis as TestWindow).app?.workspace; + workspace?.iterateAllLeaves?.((leaf) => { + if (leaf.containerEl?.dataset.e2eStatusPopout === marker) leaf.detach?.(); + }); + }, popoutMarker) + .catch(() => {}); + } + if (popoutPage) await popoutPage.close().catch(() => {}); + } + }); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + const vault = await createTemporaryVault("obsidian-livesync-replication-status-"); + let session: ObsidianLiveSyncSession | undefined; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli.binary, + vault, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + pluginData: { + liveSync: false, + syncOnStart: false, + syncOnSave: false, + isConfigured: false, + remoteType: "", + showStatusOnEditor: true, + showOnlyIconsOnEditor: false, + showStatusOnStatusbar: true, + lessInformationInLog: false, + writeLogToTheFile: false, + }, + }); + failUnless(session.readiness.pluginId === "obsidian-livesync", "The LiveSync plug-in did not become ready."); + await runStatusScenario(session); + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 31a8f339..0d1bc3e2 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -21,6 +21,7 @@ const focusedScenarios = new Set([ "configuration-encryption-sync", "review-harness", "p2p-pane", + "replication-status", "vault-reflection", "couchdb-upload", "couchdb-http-cache", diff --git a/updates.md b/updates.md index 6a92c03f..e3c00366 100644 --- a/updates.md +++ b/updates.md @@ -14,6 +14,8 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Synchronisation +- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏊ status icon and a comparison count beside received items. An unchanged count disappears after three seconds. +- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. - We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) ## 1.0.40 From e7f7e62cfa87988caadf004f7790bbf16a2589ae Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 09:03:51 +0000 Subject: [PATCH 48/68] Refine status item placement and inactive styling --- README.md | 6 +- .../features/ReplicationStatusLabel.ts | 14 +- .../ReplicationStatusLabel.unit.spec.ts | 7 +- src/modules/features/StatusBarRenderer.ts | 33 +-- .../features/StatusBarRenderer.unit.spec.ts | 78 +++++- styles.css | 19 ++ test/e2e-obsidian/README.md | 2 +- .../scripts/replication-status.ts | 250 +++++++++++++++--- updates.md | 4 +- 9 files changed, 345 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 38c78d2c..8bdbacef 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Synchronisation status is shown in the status bar with the following icons. To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files. -When CouchDB change history is being compared, `↓ 0 (⏊ 1200)` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It disappears after three seconds without a change and reappears when the count changes. The ⏊ status icon follows comparison activity independently of that timer. +When CouchDB change history is being compared, `(⏊ 1200) ↑ 0 ↓ 0` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It appears before the upload count, disappears after three seconds without a change, and reappears when the count changes. The ⏊ status icon follows comparison activity independently of that timer. ### Customising status items with CSS snippets @@ -120,7 +120,9 @@ The status bar and editor overlay use individual spans with the common `livesync The item classes include `livesync-stat-network`, `livesync-stat-sync`, `livesync-stat-sent`, `livesync-stat-push-remaining`, `livesync-stat-arrived`, `livesync-stat-pull-remaining`, `livesync-stat-ff`, and `livesync-stat-p2p`. File-processing items use `livesync-stat-file-processing`, `livesync-stat-file-pending`, and `livesync-stat-file-batched`; queue items use `livesync-stat-queue-` followed by `replication`, `database`, `storage`, `chunk-initial`, `chunk-retrying`, `plugin-scan`, `hidden-files`, or `conflicts`. The skipped-file indicator uses `livesync-stat-file-skip`. -All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count until it is hidden, reducing movement as counts decrease. +All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count during an active period, reducing movement as counts decrease. The editor overlay uses wider gaps between items, while remaining counts stay beside their transfer totals. + +Inactive items receive `livesync-stat-inactive` instead of the HTML `hidden` attribute. The default CSS hides them immediately. They retain their last text, numeric width, and position amongst the other items so that CSS snippets can customise how they disappear. The numeric width starts afresh when an item becomes active again. ## Tips and Troubleshooting - If you want a faster and simpler initial replication when setting up subsequent devices, see the [Fast Setup Guide](docs/tips/fast-setup.md). diff --git a/src/modules/features/ReplicationStatusLabel.ts b/src/modules/features/ReplicationStatusLabel.ts index b36d4c95..8358127c 100644 --- a/src/modules/features/ReplicationStatusLabel.ts +++ b/src/modules/features/ReplicationStatusLabel.ts @@ -161,7 +161,7 @@ export function projectReplicationStatusLabel( fastForwardCount > 0 ? fastForwardCount : undefined; - const fastForwardSuffix = visibleFastForwardCount === undefined ? "" : " (⏊ " + visibleFastForwardCount + ")"; + const fastForwardLabel = visibleFastForwardCount === undefined ? "" : " (⏊ " + visibleFastForwardCount + ")"; const parts: StatusBarPart[] = []; if (networkStatus.trackedRequestCount > 0) { @@ -177,10 +177,6 @@ export function projectReplicationStatusLabel( parts.push({ key: "network", text: networkActivity }); } parts.push({ key: "sync", text: "Sync: " + icon }); - parts.push({ key: "sent", text: " ↑ ", value: sent }); - if (pushRemaining.part !== undefined) parts.push(pushRemaining.part); - parts.push({ key: "arrived", text: " ↓ ", value: arrived }); - if (pullRemaining.part !== undefined) parts.push(pullRemaining.part); if (visibleFastForwardCount !== undefined) { parts.push({ key: "ff", @@ -190,6 +186,10 @@ export function projectReplicationStatusLabel( title: "Compared CouchDB feed entries", }); } + parts.push({ key: "sent", text: " ↑ ", value: sent }); + if (pushRemaining.part !== undefined) parts.push(pushRemaining.part); + parts.push({ key: "arrived", text: " ↓ ", value: arrived }); + if (pullRemaining.part !== undefined) parts.push(pullRemaining.part); return { icon, @@ -197,13 +197,13 @@ export function projectReplicationStatusLabel( networkActivity + "Sync: " + icon + + fastForwardLabel + " ↑ " + sent + pushRemaining.text + " ↓ " + arrived + - pullRemaining.text + - fastForwardSuffix, + pullRemaining.text, parts, }; } diff --git a/src/modules/features/ReplicationStatusLabel.unit.spec.ts b/src/modules/features/ReplicationStatusLabel.unit.spec.ts index 6f322e8e..054d597e 100644 --- a/src/modules/features/ReplicationStatusLabel.unit.spec.ts +++ b/src/modules/features/ReplicationStatusLabel.unit.spec.ts @@ -126,8 +126,9 @@ describe("projectReplicationStatusLabel", () => { 1200 ); expect(transferred.icon).toBe("⚡"); - expect(transferred.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏊ 1200)"); - expect(transferred.parts[transferred.parts.length - 1]).toMatchObject({ + expect(transferred.message).toBe("Sync: ⚡ (⏊ 1200) ↑ 0 ↓ 7"); + expect(transferred.parts.map((part) => part.key)).toEqual(["sync", "ff", "sent", "arrived"]); + expect(transferred.parts[1]).toMatchObject({ key: "ff", text: " (⏊ ", value: 1200, @@ -144,7 +145,7 @@ describe("projectReplicationStatusLabel", () => { 1200 ); expect(idle.icon).toBe("⚡"); - expect(idle.message).toBe("Sync: ⚡ ↑ 0 ↓ 7 (⏊ 1200)"); + expect(idle.message).toBe("Sync: ⚡ (⏊ 1200) ↑ 0 ↓ 7"); }); it("keeps the error icon ahead of an active CouchDB checking indicator", () => { diff --git a/src/modules/features/StatusBarRenderer.ts b/src/modules/features/StatusBarRenderer.ts index 990cc9f5..ceeceafd 100644 --- a/src/modules/features/StatusBarRenderer.ts +++ b/src/modules/features/StatusBarRenderer.ts @@ -1,3 +1,5 @@ +const INACTIVE_CLASS = "livesync-stat-inactive"; + export type StatusBarPart = { key: string; text: string; @@ -22,7 +24,6 @@ type RenderedItem = { export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer { const items = new Map(); - const managedElements = new Set(); let disposed = false; const setText = (node: Text, text: string) => { @@ -33,6 +34,13 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer if (element.hidden !== hidden) element.hidden = hidden; }; + const setInactive = (element: HTMLElement, inactive: boolean) => { + if (element.classList.contains(INACTIVE_CLASS) === inactive) return; + element.classList.toggle(INACTIVE_CLASS, inactive); + if (inactive) element.setAttribute("aria-hidden", "true"); + else element.removeAttribute("aria-hidden"); + }; + const setWidth = (element: HTMLElement, width: string) => { if (element.style.getPropertyValue("--livesync-stat-width") !== width) { element.setCssProps({ "--livesync-stat-width": width }); @@ -58,7 +66,6 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer const item = { element, prefix, value, valueText, suffix, widestValueLength: 0 }; items.set(key, item); - managedElements.add(element); return item; }; @@ -89,20 +96,16 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer }; const reorderIfNeeded = (visibleItems: readonly RenderedItem[]) => { + const desiredOrder = visibleItems.map((item) => item.element); + const visibleElements = new Set(desiredOrder); const currentOrder = Array.from(parent.childNodes).filter((node): node is HTMLSpanElement => - managedElements.has(node as HTMLSpanElement) + visibleElements.has(node as HTMLSpanElement) ); - const visibleElements = new Set(visibleItems.map((item) => item.element)); - const desiredOrder = [ - ...visibleItems.map((item) => item.element), - ...currentOrder.filter((element) => !visibleElements.has(element)), - ]; if ( currentOrder.length === desiredOrder.length && currentOrder.every((node, index) => node === desiredOrder[index]) - ) { + ) return; - } let reference: ChildNode | null = currentOrder[0] ?? null; for (const element of desiredOrder) { @@ -110,7 +113,7 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer const childNodes = Array.from(parent.childNodes); const elementIndex = childNodes.indexOf(element); reference = - childNodes.slice(elementIndex + 1).find((node) => managedElements.has(node as HTMLSpanElement)) ?? null; + childNodes.slice(elementIndex + 1).find((node) => visibleElements.has(node as HTMLSpanElement)) ?? null; } }; @@ -123,17 +126,16 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer for (const part of parts) { let item = items.get(part.key); if (!item) item = createItem(part.key); + if (item.element.classList.contains(INACTIVE_CLASS)) item.widestValueLength = 0; updateItem(item, part); - setHidden(item.element, false); + setInactive(item.element, false); visibleItems.push(item); visibleKeys.add(part.key); } for (const [key, item] of items) { if (visibleKeys.has(key)) continue; - setHidden(item.element, true); - setWidth(item.value, ""); - item.widestValueLength = 0; + setInactive(item.element, true); } reorderIfNeeded(visibleItems); @@ -145,7 +147,6 @@ export function createStatusBarRenderer(parent: HTMLElement): StatusBarRenderer if (item.element.parentNode === parent) parent.removeChild(item.element); } items.clear(); - managedElements.clear(); }, }; } diff --git a/src/modules/features/StatusBarRenderer.unit.spec.ts b/src/modules/features/StatusBarRenderer.unit.spec.ts index 1f1be039..c3232f99 100644 --- a/src/modules/features/StatusBarRenderer.unit.spec.ts +++ b/src/modules/features/StatusBarRenderer.unit.spec.ts @@ -66,6 +66,33 @@ class TestElement { private currentClassName = ""; private currentHidden = false; private currentTitle = ""; + private attributes = new Map(); + readonly classList = { + contains: (token: string): boolean => this.currentClassName.split(" ").includes(token), + toggle: (token: string, force: boolean): boolean => { + const tokens = this.currentClassName.split(" ").filter(Boolean); + const present = tokens.includes(token); + if (present !== force) { + this.className = force + ? [...tokens, token].join(" ") + : tokens.filter((item) => item !== token).join(" "); + } + return force; + }, + }; + + getAttribute(name: string): string | null { + return this.attributes.get(name) ?? null; + } + + setAttribute(name: string, value: string) { + this.document.mutations++; + this.attributes.set(name, value); + } + + removeAttribute(name: string) { + if (this.attributes.delete(name)) this.document.mutations++; + } constructor( private readonly document: TestDocument, @@ -173,7 +200,9 @@ function findValue(item: TestElement): TestElement { } function visibleItems(root: TestElement): TestElement[] { - return root.children.filter((child) => !child.hidden && child.className.split(" ").includes("livesync-stat-item")); + return root.children.filter( + (child) => !child.classList.contains("livesync-stat-inactive") && child.classList.contains("livesync-stat-item") + ); } describe("createStatusBarRenderer", () => { @@ -195,8 +224,10 @@ describe("createStatusBarRenderer", () => { expect(findValue(queue).className).toBe("livesync-stat-value"); renderer.render([{ key: "queue", text: " (queued: ", value: 3, suffix: ")" }]); - expect(sync.hidden).toBe(true); - expect(queue.hidden).toBe(false); + expect(sync.classList.contains("livesync-stat-inactive")).toBe(true); + expect(sync.hidden).toBe(false); + expect(sync.getAttribute("aria-hidden")).toBe("true"); + expect(queue.classList.contains("livesync-stat-inactive")).toBe(false); expect(queue.textContent).toBe(" (queued: 3)"); renderer.render([{ key: "queue", text: "Queue idle" }]); expect(queue.textContent).toBe("Queue idle"); @@ -239,7 +270,7 @@ describe("createStatusBarRenderer", () => { renderer.dispose(); }); - it("resets numeric width while omitted and reuses the same item when it returns", () => { + it("retains omitted content and width, then resets width when the same item returns", () => { const { root, parent } = createFixture(); const renderer = createStatusBarRenderer(parent); @@ -249,16 +280,49 @@ describe("createStatusBarRenderer", () => { expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); renderer.render([]); - expect(item.hidden).toBe(true); - expect(value.style.getPropertyValue("--livesync-stat-width")).toBe(""); + expect(item.classList.contains("livesync-stat-inactive")).toBe(true); + expect(item.hidden).toBe(false); + expect(item.getAttribute("aria-hidden")).toBe("true"); + expect(item.textContent).toBe(" 1234"); + expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); renderer.render([{ key: "count", text: " ", value: 5 }]); expect(findItem(root, "count")).toBe(item); - expect(item.hidden).toBe(false); + expect(item.classList.contains("livesync-stat-inactive")).toBe(false); + expect(item.getAttribute("aria-hidden")).toBeNull(); expect(value.style.getPropertyValue("--livesync-stat-width")).toBe("1ch"); renderer.dispose(); }); + it("keeps inactive items in place when other items appear and does not repeat inactive mutations", () => { + const { document, root, parent } = createFixture(); + const renderer = createStatusBarRenderer(parent); + const first = { key: "first", text: "first " }; + const count = { key: "count", text: "count ", value: 1234 }; + const last = { key: "last", text: "last" }; + renderer.render([first, count, last]); + const initialOrder = [...root.children]; + const omitted = findItem(root, "count"); + + renderer.render([first, last]); + expect(root.children.map((item) => initialOrder.indexOf(item))).toEqual([0, 1, 2]); + expect(visibleItems(root)).toEqual([initialOrder[0], initialOrder[2]]); + const mutations = document.mutations; + renderer.render([first, last]); + expect(document.mutations).toBe(mutations); + + const added = { key: "added", text: "added " }; + renderer.render([first, added, last]); + expect(root.children.map((item) => item.textContent)).toEqual(["first ", "count 1234", "added ", "last"]); + expect(omitted.textContent).toBe("count 1234"); + expect(findValue(omitted).style.getPropertyValue("--livesync-stat-width")).toBe("4ch"); + + renderer.render([first, count, added, last]); + expect(findItem(root, "count")).toBe(omitted); + expect(visibleItems(root)).toEqual(root.children); + renderer.dispose(); + }); + it("reorders existing nodes without replacing them", () => { const { root, parent } = createFixture(); const renderer = createStatusBarRenderer(parent); diff --git a/styles.css b/styles.css index 4a4b0d61..e957358b 100644 --- a/styles.css +++ b/styles.css @@ -844,6 +844,25 @@ body.is-mobile .livesync-compatibility-review-notice { font-variant-numeric: tabular-nums; } +.livesync-stat-item.livesync-stat-inactive { + display: none; +} + +.syncstatusbar .livesync-stat-item { + white-space: pre; +} + +.livesync-status-statusline + > .livesync-stat-item:not(.livesync-stat-inactive) + ~ .livesync-stat-item:not( + .livesync-stat-push-remaining, + .livesync-stat-pull-remaining, + .livesync-stat-ff, + .livesync-stat-p2p + ) { + margin-inline-start: 1em; +} + .livesync-stat-value { display: inline-block; width: var(--livesync-stat-width, auto); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index a13018fb..72155f9b 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -88,7 +88,7 @@ npm run test:e2e:obsidian:focused -- replication-status The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. -`replication-status` uses an isolated Vault without a remote service. It injects synthetic replication statistics into the real plug-in to check the ⏊ icon, the comparison count and its three-second expiry, independently styled numeric items, stable rendered digit widths, and unchanged DOM nodes for repeated values. It switches between split panes, opens a popout editor, checks log updates there, and returns to the main window. Where Electron exposes native window control, it also hides the main window and verifies that popout logs still update before restoring it. This checks display behaviour; Commonlib owns the tests which establish that PouchDB produces the underlying statistics. +`replication-status` uses an isolated Vault without a remote service. It injects synthetic replication statistics into the real plug-in to check the ⏊ icon, the comparison count and its three-second expiry, independently styled numeric items, stable rendered digit widths, and unchanged DOM nodes for repeated values. It verifies the inactive class, retained inactive content and width, and the ability to override the default hiding behaviour through CSS. A separate session enters mobile emulation before loading the plug-in and measures the editor item gaps at a 390px viewport width. The desktop checks also ensure that the status bar keeps its compact spacing. The mobile spacing fixture uses the existing overlay placement method before measurement; it does not verify Android window events. It switches between split panes, opens a popout editor, checks log updates there, and returns to the main window. Where Electron exposes native window control, it also hides the main window and verifies that popout logs still update before restoring it. This checks display behaviour; Commonlib owns the tests which establish that PouchDB produces the underlying statistics. `database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including visible radio choices for original names and pseudonyms, their default and keyboard navigation, the default-on integrity-only remote option, retained local findings when the remote is unavailable, cancellation during inspection with earlier rows retained, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice and result dialogues in mobile emulation, checking layout at 800×600 and 375×667, radio choices for available formats, their touch targets, and export controls. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. diff --git a/test/e2e-obsidian/scripts/replication-status.ts b/test/e2e-obsidian/scripts/replication-status.ts index 1ef380a1..8445d5c2 100644 --- a/test/e2e-obsidian/scripts/replication-status.ts +++ b/test/e2e-obsidian/scripts/replication-status.ts @@ -3,6 +3,7 @@ import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../r import { withObsidianPage } from "../runner/ui.ts"; import { createTemporaryVault } from "../runner/vault.ts"; import type { Page } from "playwright"; +import { setObsidianMobileTestModeBeforePluginStart } from "../runner/mobileUi.ts"; const fixturePath = "replication-status-e2e.md"; const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_UI_TIMEOUT_MS ?? 10000); @@ -40,10 +41,12 @@ type TestWorkspace = { getActiveFile?: () => { path?: string } | null; }; type TestLogModule = { + adjustStatusDivPosition?: () => void; statusLog?: WritableSource; statusDiv?: HTMLElement; statusLine?: HTMLElement; statusLineRenderer?: { render?: (parts: readonly unknown[]) => void }; + statusBarRenderer?: { render?: (parts: readonly unknown[]) => void }; statusBarLabels?: WritableSource<{ parts: readonly unknown[] }>; __addLog?: (message: unknown, level?: number, key?: string) => void; }; @@ -171,7 +174,7 @@ async function appendLiveSyncLog(page: Page, message: string): Promise { } async function waitForPartText(page: Page, key: string, expected: string): Promise { - const selector = `.livesync-status-statusline .livesync-stat-${key} .livesync-stat-value`; + const selector = `.livesync-status-statusline .livesync-stat-${key}:not(.livesync-stat-inactive) .livesync-stat-value`; await page.waitForFunction( ({ selector, expected }) => document.querySelector(selector)?.textContent?.trim() === expected, { selector, expected }, @@ -442,11 +445,152 @@ async function assertNumericWidths( return await measureNumericWidths(page); } +async function assertEditorItemSpacing(page: Page, mobile: boolean): Promise { + const spacing = await page.evaluate(() => { + const modules = (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core?.modules ?? []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + return typeof (entry as TestLogModule).statusLineRenderer?.render === "function"; + }); + if (!moduleLog?.statusLineRenderer?.render || !moduleLog.statusBarLabels) { + throw new Error("The LiveSync editor status renderer is unavailable."); + } + const original = moduleLog.statusBarLabels.value.parts; + const parts = [ + { key: "queue-replication", text: " đŸ“Ĩ", value: 1 }, + { key: "queue-database", text: " 📄", value: 12 }, + { key: "queue-storage", text: " 💾", value: 3 }, + ]; + const results: { selector: string; gapsEm: number[] }[] = []; + try { + moduleLog.statusLineRenderer.render(parts); + moduleLog.statusBarRenderer?.render?.(parts); + for (const selector of [".livesync-status-statusline", ".syncstatusbar"]) { + const root = document.querySelector(selector); + if (!root || root.getBoundingClientRect().width === 0) continue; + const firstItem = root.querySelector(`.livesync-stat-${parts[0].key}`); + if (!firstItem || Number.parseFloat(getComputedStyle(firstItem).marginInlineStart) !== 0) { + throw new Error("Inactive predecessors left a gap before the first active status item."); + } + const fontSize = Number.parseFloat(getComputedStyle(root).fontSize); + const gapsEm: number[] = []; + for (let i = 1; i < parts.length; i++) { + const previous = root.querySelector( + `.livesync-stat-${parts[i - 1].key} .livesync-stat-value` + ); + const next = root.querySelector(`.livesync-stat-${parts[i].key}`)?.firstChild; + if (!previous || next?.nodeType !== Node.TEXT_NODE) + throw new Error("A spacing fixture item is missing."); + const range = document.createRange(); + range.setStart(next, next.textContent?.search(/\S/u) ?? 0); + range.setEnd(next, next.textContent?.length ?? 0); + const nextBox = range.getBoundingClientRect(); + const previousBox = previous.getBoundingClientRect(); + if (Math.abs(nextBox.top - previousBox.top) > 2) + throw new Error("The spacing fixture unexpectedly wrapped."); + gapsEm.push((nextBox.left - previousBox.right) / fontSize); + } + results.push({ selector, gapsEm }); + } + } finally { + moduleLog.statusLineRenderer.render(original); + moduleLog.statusBarRenderer?.render?.(original); + } + return results; + }); + const editor = spacing.find((entry) => entry.selector === ".livesync-status-statusline"); + failUnless(editor?.gapsEm.length === 2, "The editor status gaps could not be measured."); + for (const gap of editor.gapsEm) { + failUnless(gap >= 1 && gap <= 1.5, `Editor status items need a 1–1.5em gap; found ${gap.toFixed(3)}em.`); + } + if (!mobile) { + const statusBar = spacing.find((entry) => entry.selector === ".syncstatusbar"); + failUnless(statusBar?.gapsEm.length === 2, "The status-bar gaps could not be measured."); + failUnless( + statusBar.gapsEm.every((gap) => gap < 1), + "The status-bar items unexpectedly acquired the wider editor gaps." + ); + } + console.log( + `${mobile ? "Mobile" : "Desktop"} editor item gaps: ${editor.gapsEm.map((gap) => gap.toFixed(3)).join(", ")}em.` + ); +} + +async function assertStatusBarItemSpacing(page: Page): Promise { + const collapsedItems = await page.evaluate(() => { + const items = document.querySelectorAll(".syncstatusbar > .livesync-stat-item"); + const collapsed: string[] = []; + for (const item of items) { + if (item.classList.contains("livesync-stat-inactive")) continue; + const prefix = item.firstChild; + if (prefix?.nodeType !== Node.TEXT_NODE || !prefix.textContent?.startsWith(" ")) continue; + const range = document.createRange(); + range.setStart(prefix, 0); + range.setEnd(prefix, 1); + if (range.getBoundingClientRect().width <= 0) collapsed.push(item.className); + } + return collapsed; + }); + failUnless( + collapsedItems.length === 0, + `Status-bar item separators lost their visible spaces: ${collapsedItems.join(", ")}.` + ); +} + +async function assertInactiveItemStyling(page: Page): Promise { + const roots = [".livesync-status-statusline", ".syncstatusbar"]; + for (const root of roots) { + const state = await page.locator(`${root} .livesync-stat-ff`).evaluate((element) => { + const item = element as HTMLElement; + return { + inactive: item.classList.contains("livesync-stat-inactive"), + hidden: item.hidden, + ariaHidden: item.getAttribute("aria-hidden"), + display: getComputedStyle(item).display, + text: item.textContent, + width: item + .querySelector(".livesync-stat-value") + ?.style.getPropertyValue("--livesync-stat-width"), + precedesUploads: item.nextElementSibling?.classList.contains("livesync-stat-sent"), + }; + }); + failUnless( + state.inactive && !state.hidden && state.ariaHidden === "true", + `Inactive state was not exposed through the class in ${root}.` + ); + failUnless(state.display === "none", `Inactive items were not hidden by the default CSS in ${root}.`); + failUnless( + state.text?.includes("1000") && state.width === "4ch" && state.precedesUploads, + `An inactive item lost its content, width, or position in ${root}.` + ); + } + + const snippet = await page.addStyleTag({ + content: ".livesync-stat-item.livesync-stat-inactive { display: inline-block; }", + }); + try { + for (const root of roots) { + failUnless( + await page.locator(`${root} .livesync-stat-ff`).isVisible(), + `A CSS override could not reveal the inactive item in ${root}.` + ); + } + } finally { + await snippet.evaluate((element) => element.parentNode?.removeChild(element)); + } + for (const root of roots) { + failUnless( + await page.locator(`${root} .livesync-stat-ff`).isHidden(), + `The default inactive style was not restored in ${root}.` + ); + } +} + async function measureStatusBarNetworkWidth( page: Page, expectedValue: string ): Promise<{ cssWidth: string; renderedWidth: number }> { - const selector = ".syncstatusbar .livesync-stat-network .livesync-stat-value"; + const selector = ".syncstatusbar .livesync-stat-network:not(.livesync-stat-inactive) .livesync-stat-value"; await page.waitForFunction( ({ selector, expectedValue }) => { const value = document.querySelector(selector); @@ -606,6 +750,13 @@ async function runStatusScenario(session: ObsidianLiveSyncSession): Promise 0) await page.waitForTimeout(remainingLinger); await page.waitForFunction( () => - document.querySelector(".livesync-status-statusline .livesync-stat-ff")?.hidden === - true, + document + .querySelector(".livesync-status-statusline .livesync-stat-ff") + ?.classList.contains("livesync-stat-inactive") === true, undefined, { timeout: uiTimeoutMs } ); + await assertInactiveItemStyling(page); const iconAfterExpiry = await page.locator(".livesync-status-statusline .livesync-stat-sync").textContent(); failUnless( iconAfterExpiry?.includes("⏊"), @@ -735,32 +888,69 @@ async function main(): Promise { const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); - const vault = await createTemporaryVault("obsidian-livesync-replication-status-"); - let session: ObsidianLiveSyncSession | undefined; - try { - session = await startObsidianLiveSyncSession({ - binary, - cliBinary: cli.binary, - vault, - startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), - pluginData: { - liveSync: false, - syncOnStart: false, - syncOnSave: false, - isConfigured: false, - remoteType: "", - showStatusOnEditor: true, - showOnlyIconsOnEditor: false, - showStatusOnStatusbar: true, - lessInformationInLog: false, - writeLogToTheFile: false, - }, - }); - failUnless(session.readiness.pluginId === "obsidian-livesync", "The LiveSync plug-in did not become ready."); - await runStatusScenario(session); - } finally { - if (session) await session.app.stop(); - await vault.dispose(); + for (const mobile of [false, true]) { + const vault = await createTemporaryVault("obsidian-livesync-replication-status-"); + let session: ObsidianLiveSyncSession | undefined; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli.binary, + vault, + lifecycle: mobile + ? { + beforePluginStart: async ({ remoteDebuggingPort }) => { + await setObsidianMobileTestModeBeforePluginStart(remoteDebuggingPort, true, uiTimeoutMs); + }, + } + : undefined, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + pluginData: { + liveSync: false, + syncOnStart: false, + syncOnSave: false, + isConfigured: false, + remoteType: "", + showStatusOnEditor: true, + showOnlyIconsOnEditor: false, + showStatusOnStatusbar: true, + lessInformationInLog: false, + writeLogToTheFile: false, + }, + }); + failUnless( + session.readiness.pluginId === "obsidian-livesync", + "The LiveSync plug-in did not become ready." + ); + if (!mobile) await runStatusScenario(session); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await installFixtureFile(page); + if (mobile) { + await page.evaluate(() => { + const modules = + (globalThis as TestWindow).app?.plugins?.plugins?.["obsidian-livesync"]?.core?.modules ?? + []; + const moduleLog = modules.find((entry): entry is TestLogModule => { + if (typeof entry !== "object" || entry === null) return false; + return typeof (entry as TestLogModule).adjustStatusDivPosition === "function"; + }); + moduleLog?.adjustStatusDivPosition?.(); + }); + } + try { + await page.locator(".livesync-status").waitFor({ state: "attached", timeout: uiTimeoutMs }); + await injectReplicationStats(page, replicationStats(0, 0, 0), 0, 0); + await page.locator(".livesync-status").waitFor({ state: "visible", timeout: uiTimeoutMs }); + } catch (error) { + throw new Error( + `${String(error)}; spacing setup: ${JSON.stringify(await getStatusDiagnostics(page))}` + ); + } + await assertEditorItemSpacing(page, mobile); + }); + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + } } } diff --git a/updates.md b/updates.md index e3c00366..0929f2e0 100644 --- a/updates.md +++ b/updates.md @@ -14,8 +14,8 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Synchronisation -- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏊ status icon and a comparison count beside received items. An unchanged count disappears after three seconds. -- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. +- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏊ status icon and a comparison count before the upload count. An unchanged count disappears after three seconds. +- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. Wider gaps between editor status items make the counts easier to read. The `livesync-stat-inactive` class lets CSS snippets control how inactive items disappear. - We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) ## 1.0.40 From 62b2edba9dde87d6d154457fa49faeeaa9e4161d Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 25 Oct 2026 13:26:02 +0000 Subject: [PATCH 49/68] fix: await optional file database saves --- ...local_file_save_completion_and_batching.md | 31 +++ package.json | 1 + .../CmdConfigSync.completion.unit.spec.ts | 220 +++++++++++++++ src/features/ConfigSync/CmdConfigSync.ts | 100 ++++++- src/features/ConfigSync/PendingConfigSaves.ts | 49 ++++ .../PendingConfigSaves.unit.spec.ts | 84 ++++++ .../CmdHiddenFileSync.completion.unit.spec.ts | 260 ++++++++++++++++++ .../HiddenFileSync/CmdHiddenFileSync.ts | 155 ++++++++++- test/e2e-obsidian/README.md | 2 + .../scripts/optional-file-save-completion.ts | 230 ++++++++++++++++ updates.md | 1 + 11 files changed, 1113 insertions(+), 20 deletions(-) create mode 100644 src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts create mode 100644 src/features/ConfigSync/PendingConfigSaves.ts create mode 100644 src/features/ConfigSync/PendingConfigSaves.unit.spec.ts create mode 100644 src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/optional-file-save-completion.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md index 8abef973..27e6e4f6 100644 --- a/docs/adr/2026_10_local_file_save_completion_and_batching.md +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -85,6 +85,37 @@ Ordinary file saving by Obsidian remains independent of database saving. Pending and processing indicators must include work in progress and unresolved failures, so an empty timer map cannot imply that local processing is complete. +### Optional file ownership and completion + +Hidden File Sync and Customisation Sync register the new +`processOptionalFileEventWithResult` contract together. `firstHandled` tries +providers in order until one returns `completed` or `veto`; only +`not-applicable` advances to the next provider. Exceptions propagate unchanged. +`HandlerOutcome` gives completion data and refusal reasons +independent types. These saves use `void` completion data and string reasons. + +A provider returns `completed` after its database operation succeeds or the +current state is already represented. Failed writes and deletions throw, and +the manager retains the event. Hidden File Sync acknowledges its processed +fingerprint after successful processing. Customisation Sync waits for the real +save after its existing 100ms quiet period. All notifications coalesced into a +V1 group receive that save's result; V2 files retain separate save groups. + +A finite flush resolves each captured event's `whenFlushRequested` signal. This +releases its customisation delay and freezes that group; later groups do not +extend its completion. During start-up replay or suspension, providers use +physical database readiness rather than waiting for application readiness or +resume. Hidden File Sync loads its existing per-database caches before replay +and reuses them during the following initialisation hook. + +The existing boolean optional API remains a legacy notification contract for +other callers and older hosts. Once a host registers typed providers, the +manager dispatches through that chain exclusively. Without a typed provider, +a resolved legacy notification completes that dispatch only; its boolean does +not establish a database save. Snapshot-write failures remain visible under +both contracts. No persistent journal, new snapshot schema, or automatic retry +loop is introduced. + ### 2. Keep a recoverable record through processing The event manager owns each operation from queue admission through completion. diff --git a/package.json b/package.json index cd383dde..5b35668b 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", "test:e2e:obsidian:hidden-file-selectors": "tsx test/e2e-obsidian/scripts/hidden-file-selectors.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", + "test:e2e:obsidian:optional-file-save-completion": "tsx test/e2e-obsidian/scripts/optional-file-save-completion.ts", "test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts", "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", "test:e2e:obsidian:internal-metadata-migration": "tsx test/e2e-obsidian/scripts/internal-metadata-migration.ts", diff --git a/src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts new file mode 100644 index 00000000..147ee857 --- /dev/null +++ b/src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts @@ -0,0 +1,220 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + diff_match_patch: class DiffMatchPatch {}, + normalizePath: vi.fn((path: string) => path), + parseYaml: vi.fn(), + Platform: {}, +})); +vi.mock("./PluginDialogModal.ts", () => ({ + PluginDialogModal: class PluginDialogModal {}, +})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ + JsonResolveModal: class JsonResolveModal {}, +})); +vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({ + ConflictResolveModal: class ConflictResolveModal {}, +})); +// The readiness helpers come from the real LiveSyncCommands, so only its log view dependency is replaced. +vi.mock("@/modules/features/ModuleLog.ts", () => ({ + MARK_DONE: "", +})); +vi.mock("@/common/types.ts", () => ({ + ICXHeader: "ix:", + PERIODIC_PLUGIN_SWEEP: 60, +})); +vi.mock("@/common/utils.ts", () => ({ + cancelTask: vi.fn(), + EVEN: Symbol("even"), + isCustomisationSyncMetadata: vi.fn(), + isPluginMetadata: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ + PeriodicProcessor: class PeriodicProcessor {}, +})); +vi.mock("@/common/events.ts", () => ({ + EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync", + eventHub: { + onEvent: vi.fn(), + }, +})); +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ + getObsidianCommunityPluginManager: vi.fn(), +})); + +import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { getDocData } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { ConfigSync } from "./CmdConfigSync"; +import { createOptionalFileSyncDatabase } from "../../../test/fixtures/optionalFileSyncDatabase"; + +const path = ".obsidian/snippets/test.css" as FilePath; +const fixtures: Awaited>[] = []; + +async function createConfigSync(useV2 = false) { + const fixture = await createOptionalFileSyncDatabase(); + fixtures.push(fixture); + fixture.settings.usePluginSyncV2 = useV2; + const files = new Map([ + [path, { content: "body { color: red; }", mtime: 2000 }], + ]); + const storageAccess = { + statHidden: vi.fn(async (file: string) => { + const entry = files.get(file); + return entry + ? { type: "file" as const, ctime: 1000, mtime: entry.mtime, size: entry.content.length } + : null; + }), + readHiddenFileBinary: vi.fn(async (file: string) => { + const entry = files.get(file); + if (!entry) throw new Error(`Missing fixture: ${file}`); + return new TextEncoder().encode(entry.content).buffer; + }), + }; + const services = { + ...fixture.services, + appLifecycle: { isReady: () => false, isSuspended: () => true }, + database: { ...fixture.services.database, isDatabaseReady: vi.fn(() => true) }, + path: { + ...fixture.services.path, + isMarkedAsSameChanges: vi.fn(), + markChangesAreSame: vi.fn(), + }, + }; + const config = Object.create(ConfigSync.prototype) as ConfigSync; + Object.assign(config, { + core: { settings: fixture.settings, localDatabase: fixture.localDatabase, storageAccess, services }, + _log: vi.fn(), + updatePluginList: vi.fn(async () => undefined), + updatePluginListV2: vi.fn(async () => undefined), + recentProcessedInternalFiles: [], + }); + const databasePath = useV2 ? config.filenameWithUnifiedKey(path) : config.filenameToUnifiedKey(path); + return { fixture, config, files, services, storageAccess, databasePath }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + for (const fixture of fixtures.splice(0)) await fixture.dispose(); +}); + +describe("Customisation Sync optional save completion", () => { + const flush = { whenFlushRequested: Promise.resolve() }; + + it("waits for the database write while the application is suspended or still starting", async () => { + const { fixture, config } = await createConfigSync(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const put = fixture.localDatabase.putDBEntry.bind(fixture.localDatabase); + const spy = vi.spyOn(fixture.localDatabase, "putDBEntry").mockImplementation(async (...args) => { + await gate; + return await put(...args); + }); + let completed = false; + const result = config._processOptionalFileEventWithResult(path, flush).then((outcome) => { + completed = true; + return outcome; + }); + await vi.waitFor(() => expect(spy).toHaveBeenCalledOnce()); + expect(completed).toBe(false); + release(); + await expect(result).resolves.toEqual({ kind: "completed" }); + }); + + it.each([false, true])("confirms identical contents and stores a same-time edit (V2: %s)", async (useV2) => { + const { fixture, config, databasePath, files } = await createConfigSync(useV2); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + const first = await fixture.database.get(databasePath); + await config._processOptionalFileEventWithResult(path, flush); + expect((await fixture.database.get(databasePath))._rev).toBe(first._rev); + files.get(path)!.content = "body { color: tan; }"; + await config._processOptionalFileEventWithResult(path, flush); + expect((await fixture.database.get(databasePath))._rev).not.toBe(first._rev); + }); + + it.each([ + { useV2: false, throws: false }, + { useV2: true, throws: false }, + { useV2: false, throws: true }, + { useV2: true, throws: true }, + ])("rejects a failed write and permits the unchanged notification to retry: %j", async ({ useV2, throws }) => { + const { fixture, config, databasePath } = await createConfigSync(useV2); + const failure = new Error("temporary write failure"); + const put = vi.spyOn(fixture.localDatabase, "putDBEntry"); + if (throws) put.mockRejectedValueOnce(failure); + else put.mockResolvedValueOnce(false); + const failed = config._processOptionalFileEventWithResult(path, flush); + if (throws) await expect(failed).rejects.toBe(failure); + else await expect(failed).rejects.toThrow("Could not save customisation"); + await expect(fixture.localDatabase.getDBEntry(databasePath)).resolves.toBe(false); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + expect(put).toHaveBeenCalledTimes(2); + expect(await fixture.localDatabase.getDBEntry(databasePath)).not.toBe(false); + }); + + it.each([false, true])("confirms deletion and treats a repeated deletion as complete (V2: %s)", async (useV2) => { + const { fixture, config, databasePath, files } = await createConfigSync(useV2); + await config._processOptionalFileEventWithResult(path, flush); + files.delete(path); + const put = vi.spyOn(fixture.localDatabase, "putRaw"); + put.mockResolvedValueOnce({ ok: false, id: "failed", rev: "" }); + await expect(config._processOptionalFileEventWithResult(path, flush)).rejects.toThrow( + "Could not delete customisation" + ); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + const deleted = await fixture.database.get(databasePath); + expect("deleted" in deleted && deleted.deleted).toBe(true); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" }); + expect((await fixture.database.get(databasePath))._rev).toBe(deleted._rev); + }); + + it("distinguishes a different owner and unmet prerequisites", async () => { + const { config, services, storageAccess } = await createConfigSync(); + await expect(config._processOptionalFileEventWithResult("notes.md" as FilePath, flush)).resolves.toEqual({ + kind: "not-applicable", + }); + services.database.isDatabaseReady.mockReturnValue(false); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ + kind: "veto", + value: "database-not-ready", + }); + services.database.isDatabaseReady.mockReturnValue(true); + vi.spyOn(services.setting, "getDeviceAndVaultName").mockReturnValue(""); + await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ + kind: "veto", + value: "device-name-not-configured", + }); + expect(storageAccess.readHiddenFileBinary).not.toHaveBeenCalled(); + }); + + it("preserves every V2 file when neighbouring plug-in files arrive together", async () => { + const { fixture, config, files } = await createConfigSync(true); + const main = ".obsidian/plugins/example/main.js" as FilePath; + const style = ".obsidian/plugins/example/styles.css" as FilePath; + files.set(main, { content: "module.exports = {};", mtime: 3000 }); + files.set(style, { content: "body {}", mtime: 3000 }); + await Promise.all([ + config._processOptionalFileEventWithResult(main, flush), + config._processOptionalFileEventWithResult(style, flush), + ]); + expect(await fixture.localDatabase.getDBEntry(config.filenameWithUnifiedKey(main))).not.toBe(false); + expect(await fixture.localDatabase.getDBEntry(config.filenameWithUnifiedKey(style))).not.toBe(false); + }); + + it("does not acknowledge a database deletion which was not accepted", async () => { + const { fixture, config, databasePath } = await createConfigSync(); + await config.storeCustomizationFiles(path); + const original = await fixture.localDatabase.getDBEntry(databasePath); + expect(original).not.toBe(false); + vi.spyOn(fixture.localDatabase, "putRaw").mockResolvedValueOnce({ ok: false, id: "failed", rev: "" }); + await expect(config.deleteConfigOnDatabase(databasePath)).resolves.toBe(false); + const unchanged = await fixture.localDatabase.getDBEntry(databasePath); + expect(unchanged && getDocData(unchanged.data)).toBe(original && getDocData(original.data)); + }); +}); diff --git a/src/features/ConfigSync/CmdConfigSync.ts b/src/features/ConfigSync/CmdConfigSync.ts index 0e1412ee..cbdad936 100644 --- a/src/features/ConfigSync/CmdConfigSync.ts +++ b/src/features/ConfigSync/CmdConfigSync.ts @@ -69,6 +69,10 @@ import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError" import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts"; +import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; +import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import { PendingConfigSaves } from "./PendingConfigSaves"; + const d = "\u200b"; const d2 = "\n"; const UPDATED_CONFIGURATION_NOTICE_KEY = "config-sync:updated-configuration"; @@ -1206,7 +1210,7 @@ export class ConfigSync extends LiveSyncCommands { } recentProcessedInternalFiles = [] as string[]; - async makeEntryFromFile(path: FilePath): Promise { + async makeEntryFromFile(path: FilePath, requireCompletion = false): Promise { const stat = await this.core.storageAccess.statHidden(path); let version: string | undefined; let displayName: string | undefined; @@ -1240,6 +1244,7 @@ export class ConfigSync extends LiveSyncCommands { } catch (ex) { this._log(`The file ${path} could not be encoded`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } const mtime = stat.mtime; @@ -1254,6 +1259,15 @@ export class ConfigSync extends LiveSyncCommands { } async storeCustomisationFileV2(path: FilePath, term: string, force = false) { + return await this.storeCustomisationFileV2Core(path, term, force, false); + } + + private async storeCustomisationFileV2Core( + path: FilePath, + term: string, + force: boolean, + requireCompletion: boolean + ) { const vf = this.filenameWithUnifiedKey(path, term); return await serialized(`plugin-${vf}`, async () => { const prefixedFileName = vf; @@ -1261,7 +1275,8 @@ export class ConfigSync extends LiveSyncCommands { const id = await this.path2id(prefixedFileName); const stat = await this.core.storageAccess.statHidden(path); if (!stat) { - return false; + if (!requireCompletion) return false; + return await this.deleteConfigOnDatabaseCore(vf, force, true); } const mtime = stat.mtime; const content = await this.core.storageAccess.readHiddenFileBinary(path); @@ -1285,7 +1300,10 @@ export class ConfigSync extends LiveSyncCommands { eden: {}, }; } else { - if (this.services.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN) { + if ( + !requireCompletion && + this.services.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN + ) { this._log( `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, LOG_LEVEL_DEBUG @@ -1300,7 +1318,7 @@ export class ConfigSync extends LiveSyncCommands { const dataStart = dataSrc.indexOf(DUMMY_END); const oldContent = dataSrc.substring(dataStart + DUMMY_END.length); const oldContentArray = base64ToArrayBuffer(oldContent); - if (await isDocContentSame(oldContentArray, content)) { + if (!old.deleted && (await isDocContentSame(oldContentArray, content))) { this._log( `STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, LOG_LEVEL_VERBOSE @@ -1320,24 +1338,36 @@ export class ConfigSync extends LiveSyncCommands { }; } const ret = await this.localDatabase.putDBEntry(saveData); + if (requireCompletion && (!ret || !ret.ok)) + throw new LiveSyncError(`Could not save customisation: ${path}`); this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); fireAndForget(() => this.updatePluginListV2(false, this.filenameWithUnifiedKey(path))); return ret; } catch (ex) { this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } }); } async storeCustomizationFiles(path: FilePath, termOverRide?: string) { + return await this.storeCustomizationFilesCore(path, termOverRide, false); + } + + private async storeCustomizationFilesCore( + path: FilePath, + termOverRide: string | undefined, + requireCompletion: boolean + ) { const term = termOverRide || this.services.setting.getDeviceAndVaultName(); if (term == "") { this._log($msg("We have to configure the device name"), LOG_LEVEL_NOTICE); + if (requireCompletion) throw new LiveSyncError("The device name is not configured"); return; } if (this.useV2) { - return await this.storeCustomisationFileV2(path, term); + return await this.storeCustomisationFileV2Core(path, term, false, requireCompletion); } const vf = this.filenameToUnifiedKey(path, term); // console.warn(`Storing ${path} to ${bareVF} :--> ${keyedVF}`); @@ -1378,7 +1408,7 @@ export class ConfigSync extends LiveSyncCommands { fileTargets = ["manifest.json", "theme.css"].map((e) => `${parentPath}/${e}` as FilePath); } for (const target of fileTargets) { - const data = await this.makeEntryFromFile(target); + const data = await this.makeEntryFromFile(target, requireCompletion); if (data == false) { this._log(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE); continue; @@ -1398,9 +1428,9 @@ export class ConfigSync extends LiveSyncCommands { // this._log(`Configuration saving: ${prefixedFileName}`); if (dt.files.length == 0) { this._log(`Nothing left: deleting.. ${path}`); - await this.deleteConfigOnDatabase(prefixedFileName); + const deleted = await this.deleteConfigOnDatabaseCore(prefixedFileName, false, requireCompletion); await this.updatePluginList(false, prefixedFileName); - return; + return requireCompletion ? deleted : undefined; } const content = createTextBlob(serialize(dt)); @@ -1422,7 +1452,7 @@ export class ConfigSync extends LiveSyncCommands { eden: {}, }; } else { - if (old.mtime == mtime) { + if (!requireCompletion && old.mtime == mtime) { // this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE); return true; } @@ -1443,7 +1473,7 @@ export class ConfigSync extends LiveSyncCommands { } }); const isSame = (await Promise.all(diffs)).every((e) => e == true); - if (isSame) { + if (isSame && !old.deleted) { this._log( `STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, LOG_LEVEL_VERBOSE @@ -1464,16 +1494,51 @@ export class ConfigSync extends LiveSyncCommands { }; } const ret = await this.localDatabase.putDBEntry(saveData); + if (requireCompletion && (!ret || !ret.ok)) + throw new LiveSyncError(`Could not save customisation: ${path}`); await this.updatePluginList(false, saveData.path); this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`); return ret; } catch (ex) { this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } }); } + private pendingOptionalSaves?: PendingConfigSaves; + + async _processOptionalFileEventWithResult( + path: FilePath, + options?: OptionalFileEventOptions + ): Promise> { + if (!this.isThisModuleEnabled() || !this.isTargetPath(path)) return { kind: "not-applicable" }; + const configDir = normalizePath(this.configDir); + const handledElsewhere = Object.values(this.settings.pluginSyncExtendedSetting) + .filter((entry) => entry.mode !== MODE_SELECTIVE && entry.mode !== MODE_SHINY) + .flatMap((entry) => entry.files) + .some((file) => `${configDir}/${file}`.toLowerCase().startsWith(path.toLowerCase())); + if (handledElsewhere) return { kind: "not-applicable" }; + if (!this._isDatabaseReady()) return { kind: "veto", value: "database-not-ready" }; + const term = this.services.setting.getDeviceAndVaultName(); + if (!term) return { kind: "veto", value: "device-name-not-configured" }; + const stat = await this.core.storageAccess.statHidden(path); + if (stat && stat.type !== "file") return { kind: "not-applicable" }; + + // V1 saves a complete file group; V2 persists each file separately. + const key = this.useV2 ? this.filenameWithUnifiedKey(path, term) : this.filenameToUnifiedKey(path, term); + this.pendingOptionalSaves ??= new PendingConfigSaves(); + await this.pendingOptionalSaves.enqueue( + key, + async () => { + await this.storeCustomizationFilesCore(path, term, true); + }, + options?.whenFlushRequested + ); + return { kind: "completed" }; + } + async _anyProcessOptionalFileEvent(path: FilePath): Promise { return await this.watchVaultRawEventsAsync(path); } @@ -1615,6 +1680,14 @@ export class ConfigSync extends LiveSyncCommands { } async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false) { + return await this.deleteConfigOnDatabaseCore(prefixedFileName, forceWrite, false); + } + + private async deleteConfigOnDatabaseCore( + prefixedFileName: FilePathWithPrefix, + forceWrite: boolean, + requireCompletion: boolean + ) { // const id = await this.path2id(prefixedFileName); const mtime = new Date().getTime(); return await serialized("file-x-" + prefixedFileName, async () => { @@ -1640,13 +1713,15 @@ export class ConfigSync extends LiveSyncCommands { type: "newnote", }; } - await this.localDatabase.putRaw(saveData); + const result = await this.localDatabase.putRaw(saveData); + if (!result.ok) throw new LiveSyncError(`Could not delete customisation: ${prefixedFileName}`); await this.updatePluginList(false, prefixedFileName); this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Done`); return true; } catch (ex) { this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (requireCompletion) throw ex; return false; } }); @@ -1756,6 +1831,9 @@ export class ConfigSync extends LiveSyncCommands { } override onBindFunction(core: LiveSyncCore, services: InjectableServiceHub): void { services.fileProcessing.processOptionalFileEvent.addHandler(this._anyProcessOptionalFileEvent.bind(this)); + services.fileProcessing.processOptionalFileEventWithResult?.addHandler( + this._processOptionalFileEventWithResult.bind(this) + ); services.conflict.getOptionalConflictCheckMethod.addHandler(this._anyGetOptionalConflictCheckMethod.bind(this)); services.replication.processVirtualDocument.addHandler(this._anyModuleParsedReplicationResultItem.bind(this)); services.setting.onRealiseSetting.addHandler(this._everyRealizeSettingSyncMode.bind(this)); diff --git a/src/features/ConfigSync/PendingConfigSaves.ts b/src/features/ConfigSync/PendingConfigSaves.ts new file mode 100644 index 00000000..6243fe93 --- /dev/null +++ b/src/features/ConfigSync/PendingConfigSaves.ts @@ -0,0 +1,49 @@ +import { compatGlobal, type CompatTimeoutHandle } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; + +type PendingSave = { + save: () => Promise; + completion: Promise; + resolve: () => void; + reject: (error: unknown) => void; + timer?: CompatTimeoutHandle; + started: boolean; +}; + +/** Owns the quiet-period delay and completion of coalesced customisation saves. */ +export class PendingConfigSaves { + private readonly pending = new Map(); + + enqueue(key: string, save: () => Promise, whenFlushRequested?: Promise): Promise { + let group = this.pending.get(key); + if (!group) { + let resolve!: () => void; + let reject!: (error: unknown) => void; + const completion = new Promise((onResolve, onReject) => { + resolve = onResolve; + reject = onReject; + }); + group = { save, completion, resolve, reject, started: false }; + this.pending.set(key, group); + } else { + group.save = save; + if (group.timer !== undefined) compatGlobal.clearTimeout(group.timer); + } + const captured = group; + group.timer = compatGlobal.setTimeout(() => this.run(key, captured), 100); + // Capture this group: a late flush of an earlier event must not release + // a newer group which happens to use the same customisation key. + void whenFlushRequested?.then( + () => this.run(key, captured), + () => {} + ); + return group.completion; + } + + private run(key: string, group: PendingSave): void { + if (group.started) return; + group.started = true; + if (group.timer !== undefined) compatGlobal.clearTimeout(group.timer); + this.pending.delete(key); + void Promise.resolve().then(group.save).then(group.resolve, group.reject); + } +} diff --git a/src/features/ConfigSync/PendingConfigSaves.unit.spec.ts b/src/features/ConfigSync/PendingConfigSaves.unit.spec.ts new file mode 100644 index 00000000..2b38e159 --- /dev/null +++ b/src/features/ConfigSync/PendingConfigSaves.unit.spec.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { PendingConfigSaves } from "./PendingConfigSaves"; + +function deferred() { + let resolve!: () => void; + let reject!: (error: unknown) => void; + const promise = new Promise((onResolve, onReject) => { + resolve = onResolve; + reject = onReject; + }); + return { promise, resolve, reject }; +} + +afterEach(() => vi.useRealTimers()); + +describe("pending customisation saves", () => { + it("retains the quiet period and completes every coalesced notification after the actual save", async () => { + vi.useFakeTimers(); + const pending = new PendingConfigSaves(); + const save = deferred(); + const firstSave = vi.fn(async () => {}); + const lastSave = vi.fn(() => save.promise); + const first = pending.enqueue("plugin", firstSave); + await vi.advanceTimersByTimeAsync(80); + const second = pending.enqueue("plugin", lastSave); + let completed = false; + const all = Promise.all([first, second]).then(() => { + completed = true; + }); + await vi.advanceTimersByTimeAsync(99); + expect(lastSave).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(firstSave).not.toHaveBeenCalled(); + expect(lastSave).toHaveBeenCalledOnce(); + expect(completed).toBe(false); + save.resolve(); + await all; + expect(completed).toBe(true); + }); + + it("flushes the captured group without waiting for or releasing later notifications", async () => { + vi.useFakeTimers(); + const pending = new PendingConfigSaves(); + const flush = deferred(); + const lateFlush = deferred(); + const firstSave = deferred(); + const work = vi.fn(() => firstSave.promise); + const first = pending.enqueue("plugin", work, flush.promise); + const coalesced = pending.enqueue("plugin", work, lateFlush.promise); + flush.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(work).toHaveBeenCalledOnce(); + const nextWork = vi.fn(async () => {}); + const later = pending.enqueue("plugin", nextWork); + lateFlush.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(nextWork).not.toHaveBeenCalled(); + firstSave.resolve(); + await Promise.all([first, coalesced]); + expect(nextWork).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(100); + await later; + expect(nextWork).toHaveBeenCalledOnce(); + }); + + it("rejects every coalesced notification with the original failure and permits a later retry", async () => { + vi.useFakeTimers(); + const pending = new PendingConfigSaves(); + const failure = new Error("database write failed"); + const save = async () => { + throw failure; + }; + const first = pending.enqueue("plugin", save); + const second = pending.enqueue("plugin", save); + const outcomes = Promise.allSettled([first, second]); + await vi.advanceTimersByTimeAsync(100); + expect(await outcomes).toEqual([ + { status: "rejected", reason: failure }, + { status: "rejected", reason: failure }, + ]); + const retry = pending.enqueue("plugin", async () => {}, Promise.resolve()); + await retry; + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts new file mode 100644 index 00000000..5ef0e4de --- /dev/null +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts @@ -0,0 +1,260 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; + +vi.mock("@/deps.ts", () => ({})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("./configureHiddenFileSyncMode.ts", () => ({ configureHiddenFileSyncMode: vi.fn() })); + +import type { FilePath, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { readContent } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { HiddenFileSync } from "./CmdHiddenFileSync"; +import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase"; + +const path = ".obsidian/plugins/example/main.js" as FilePath; +const content = "module.exports = class ExamplePlugin {};"; +const fixtureOwners: Awaited>[] = []; + +async function getText(entry: Parameters[0]) { + const value = await readContent(entry); + return typeof value === "string" ? value : new TextDecoder().decode(value); +} + +async function createHiddenFileSync() { + const fixture = await createOptionalFileSyncDatabase(); + fixtureOwners.push(fixture); + fixture.settings.usePluginSync = false; + fixture.settings.syncInternalFilesTargetPatterns = "" as typeof fixture.settings.syncInternalFilesTargetPatterns; + fixture.settings.syncInternalFilesIgnorePatterns = "" as typeof fixture.settings.syncInternalFilesIgnorePatterns; + let fileContent = content; + let stat: UXStat | null = { + ctime: 1_000, + mtime: 2_000, + size: fileContent.length, + type: "file", + }; + const values = new Map(); + const kvDB = { + get: vi.fn(async (key: string) => values.get(key) as T | undefined), + set: vi.fn(async (key: string, value: T) => { + values.set(key, value); + }), + }; + const storageAccess = { + statHidden: vi.fn(async () => stat), + readHiddenFileAuto: vi.fn(async () => fileContent), + writeHiddenFileAuto: vi.fn(async () => undefined), + removeHidden: vi.fn(async () => true), + isExistsIncludeHidden: vi.fn(async () => true), + ensureDir: vi.fn(async () => undefined), + triggerHiddenFile: vi.fn(async () => undefined), + setFile(nextContent: string | null, mtime = 2_000) { + fileContent = nextContent ?? ""; + stat = nextContent === null ? null : { ctime: 1_000, mtime, size: fileContent.length, type: "file" }; + }, + }; + const databaseService = { isDatabaseReady: vi.fn(() => true) }; + const appLifecycle = { isReady: vi.fn(() => false), isSuspended: vi.fn(() => true) }; + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + core: { + settings: fixture.settings, + localDatabase: fixture.localDatabase, + kvDB, + storageAccess, + services: { + ...fixture.services, + database: databaseService, + appLifecycle, + path: { + ...fixture.services.path, + markChangesAreSame: vi.fn(), + unmarkChanges: vi.fn(), + }, + vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) }, + }, + }, + cacheFileRegExps: new Map(), + cacheCustomisationSyncIgnoredFiles: new Map(), + _fileInfoLastProcessed: new Map(), + _databaseInfoLastProcessed: new Map(), + _fileInfoLastKnown: new Map(), + semaphore: Semaphore(10), + _log: vi.fn(), + }); + return { fixture, hiddenFileSync, storageAccess, kvDB, databaseService, appLifecycle }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + for (const fixture of fixtureOwners.splice(0)) await fixture.dispose(); +}); + +describe("Hidden File Sync optional save outcomes", () => { + it("retries a failed database write when the storage fingerprint is unchanged", async () => { + const { fixture, hiddenFileSync } = await createHiddenFileSync(); + await expect(hiddenFileSync.isTargetFile(path)).resolves.toBe(true); + vi.spyOn(fixture.localDatabase, "putDBEntry").mockRejectedValueOnce(new Error("temporary database failure")); + + await expect(hiddenFileSync.trackStorageFileModification(path)).resolves.toBe(false); + await expect(hiddenFileSync.trackStorageFileModification(path)).resolves.toBe(true); + + expect(fixture.localDatabase.putDBEntry).toHaveBeenCalledTimes(2); + const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(saved).not.toBe(false); + }); + + it("rejects a failed strict write and retries the same fingerprint without delaying readiness", async () => { + const { fixture, hiddenFileSync, appLifecycle } = await createHiddenFileSync(); + const putDBEntry = vi + .spyOn(fixture.localDatabase, "putDBEntry") + .mockRejectedValueOnce(new Error("temporary database failure")); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow( + "temporary database failure" + ); + expect(hiddenFileSync.getLastProcessedFileKey(path)).toBeUndefined(); + + await expect( + hiddenFileSync._processOptionalFileEventWithResult(path, { + whenFlushRequested: new Promise(() => undefined), + }) + ).resolves.toMatchObject({ + kind: "completed", + }); + expect(putDBEntry).toHaveBeenCalledTimes(2); + const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(saved).not.toBe(false); + if (saved) expect(await getText(saved)).toBe(content); + expect(appLifecycle.isReady()).toBe(false); + expect(appLifecycle.isSuspended()).toBe(true); + }); + + it("rejects a false database-write result and can store a later edit after a same-content event", async () => { + const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync(); + const putDBEntry = vi + .spyOn(fixture.localDatabase, "putDBEntry") + .mockImplementationOnce(async () => false as never); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow( + `Failed to store hidden file in the database: ${path}` + ); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + const first = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(first).not.toBe(false); + if (first) { + const firstRevision = first._rev; + hiddenFileSync._fileInfoLastProcessed.delete(path); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + const unchanged = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(unchanged).not.toBe(false); + if (unchanged) expect(unchanged._rev).toBe(firstRevision); + } + + storageAccess.setFile("module.exports = class UpdatedExamplePlugin {};", 3_000); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + const updated = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(updated).not.toBe(false); + if (updated) expect(await getText(updated)).toBe("module.exports = class UpdatedExamplePlugin {};"); + expect(putDBEntry).toHaveBeenCalledTimes(3); + }); + + it("vetoes when the physical database is unavailable and ignores disabled or filtered events", async () => { + const { fixture, hiddenFileSync, databaseService } = await createHiddenFileSync(); + + databaseService.isDatabaseReady.mockReturnValue(false); + await expect( + hiddenFileSync._processOptionalFileEventWithResult(path, { + whenFlushRequested: new Promise(() => undefined), + }) + ).resolves.toMatchObject({ kind: "veto", value: expect.any(String) }); + databaseService.isDatabaseReady.mockReturnValue(true); + + fixture.settings.syncInternalFiles = false; + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "not-applicable", + }); + fixture.settings.syncInternalFiles = true; + fixture.settings.syncInternalFilesTargetPatterns = + "^\\.obsidian/snippets/" as typeof fixture.settings.syncInternalFilesTargetPatterns; + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "not-applicable", + }); + }); + + it("treats an absent replayed deletion as complete and leaves cache initialisation intact", async () => { + const { hiddenFileSync, storageAccess, kvDB } = await createHiddenFileSync(); + storageAccess.setFile(null); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + expect(hiddenFileSync.getLastProcessedFileKey(path)).toBe("0-0"); + const lifecycle = hiddenFileSync as unknown as { + _everyOnDatabaseInitialized(showNotice: boolean): Promise; + }; + hiddenFileSync.core.settings.syncInternalFiles = false; + await lifecycle._everyOnDatabaseInitialized(false); + expect(kvDB.get).toHaveBeenCalledTimes(3); + expect(hiddenFileSync.getLastProcessedFileKey(path)).toBe("0-0"); + }); + + it("rejects actual deletion failures and completes duplicate deletions without another write", async () => { + const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync(); + await fixture.localDatabase.putDBEntry(optionalFileEntry(`i:${path}`, content)); + storageAccess.setFile(null); + const putRaw = vi + .spyOn(fixture.localDatabase, "putRaw") + .mockRejectedValueOnce(new Error("temporary deletion failure")); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow( + "temporary deletion failure" + ); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + hiddenFileSync.resetLastProcessedFile([path]); + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + expect(putRaw).toHaveBeenCalledTimes(2); + }); + + it("recreates an empty hidden file when its database entry is logically deleted", async () => { + const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync(); + await fixture.localDatabase.putDBEntry(optionalFileEntry(`i:${path}`, "")); + await expect(hiddenFileSync.deleteInternalFileOnDatabase(path)).resolves.toBe(true); + const deletedBase = await fixture.localDatabase.getDBEntry( + `i:${path}` as FilePath, + undefined, + false, + true, + true + ); + expect(deletedBase).not.toBe(false); + if (deletedBase) expect(deletedBase.deleted).toBe(true); + if (!deletedBase) throw new Error("The logically deleted test entry could not be loaded"); + vi.spyOn(fixture.localDatabase, "getDBEntry").mockResolvedValueOnce(deletedBase); + + storageAccess.setFile("", 4_000); + const putDBEntry = vi.spyOn(fixture.localDatabase, "putDBEntry"); + + await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({ + kind: "completed", + }); + + expect(putDBEntry).toHaveBeenCalledTimes(1); + const recreated = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(recreated).not.toBe(false); + if (recreated) { + expect(recreated.deleted).not.toBe(true); + expect(await getText(recreated)).toBe(""); + } + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.ts index 5b945c69..5abc7027 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.ts @@ -54,6 +54,8 @@ import { EVENT_SETTING_SAVED, eventHub } from "@/common/events.ts"; import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; import type { LiveSyncCore } from "@/main.ts"; import { tryGetFilePath } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc"; +import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; +import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import { configureHiddenFileSyncMode, type ConfigureHiddenFileSyncResult } from "./configureHiddenFileSyncMode.ts"; import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts"; @@ -161,9 +163,7 @@ export class HiddenFileSync extends LiveSyncCommands { // return true; // } private async _everyOnDatabaseInitialized(showNotice: boolean) { - this._fileInfoLastProcessed = await autosaveCache(this.kvDB, "hidden-file-lastProcessed"); - this._databaseInfoLastProcessed = await autosaveCache(this.kvDB, "hidden-file-lastProcessed-database"); - this._fileInfoLastKnown = await autosaveCache(this.kvDB, "hidden-file-lastKnown"); + await this.ensureProcessedCaches(); if (this.isThisModuleEnabled()) { if (this._fileInfoLastProcessed.size == 0 && this._fileInfoLastProcessed.size == 0) { this._log(`No cache found. Performing startup scan.`, LOG_LEVEL_VERBOSE); @@ -249,6 +249,26 @@ export class HiddenFileSync extends LiveSyncCommands { return false; } + async _processOptionalFileEventWithResult( + path: FilePath, + _options?: OptionalFileEventOptions + ): Promise> { + if (!this.isThisModuleEnabled()) return { kind: "not-applicable" }; + if (!(await this.isTargetFile(path))) return { kind: "not-applicable" }; + if (!this._isDatabaseReady()) { + return { kind: "veto", value: "The local database is not ready" }; + } + + await this.ensureProcessedCaches(); + if (!this.isThisModuleEnabled() || !(await this.isTargetFile(path))) { + return { kind: "not-applicable" }; + } + if (!(await this.trackStorageFileModificationStrict(path))) { + return { kind: "not-applicable" }; + } + return { kind: "completed" }; + } + _anyGetOptionalConflictCheckMethod(path: FilePathWithPrefix): Promise { if (isInternalMetadata(path)) { this.queueConflictCheck(path); @@ -317,6 +337,51 @@ export class HiddenFileSync extends LiveSyncCommands { _fileInfoLastKnown!: MapLike; _databaseInfoLastProcessed!: MapLike; + private processedCacheStore: unknown; + private processedCacheStoreReady = false; + private processedCacheLoadStore: unknown; + private processedCacheLoad?: Promise; + + private async ensureProcessedCaches() { + const store = this.kvDB; + if (!store) throw new Error("The Hidden File Sync cache store is unavailable"); + if (this.processedCacheStoreReady && this.processedCacheStore === store) return; + if (this.processedCacheLoad && this.processedCacheLoadStore === store) { + await this.processedCacheLoad; + if (!this.processedCacheStoreReady || this.processedCacheStore !== this.kvDB) { + throw new Error("The Hidden File Sync cache store changed while loading its processing state"); + } + return; + } + + let load!: Promise; + load = Promise.all([ + autosaveCache(store, "hidden-file-lastProcessed"), + autosaveCache(store, "hidden-file-lastProcessed-database"), + autosaveCache(store, "hidden-file-lastKnown"), + ]) + .then(([fileInfoLastProcessed, databaseInfoLastProcessed, fileInfoLastKnown]) => { + if (this.processedCacheLoad !== load || this.kvDB !== store) return; + this._fileInfoLastProcessed = fileInfoLastProcessed; + this._databaseInfoLastProcessed = databaseInfoLastProcessed; + this._fileInfoLastKnown = fileInfoLastKnown; + this.processedCacheStore = store; + this.processedCacheStoreReady = true; + }) + .finally(() => { + if (this.processedCacheLoad === load) { + this.processedCacheLoad = undefined; + this.processedCacheLoadStore = undefined; + } + }); + this.processedCacheLoadStore = store; + this.processedCacheLoad = load; + await load; + if (!this.processedCacheStoreReady || this.processedCacheStore !== this.kvDB) { + throw new Error("The Hidden File Sync cache store changed while loading its processing state"); + } + } + statToKey(stat: UXStat | null) { return `${stat?.mtime ?? 0}-${stat?.size ?? 0}`; } @@ -600,7 +665,6 @@ Offline Changed files: ${processFiles.length}`; this._log(`Hidden file:${path} is changed.`, LOG_LEVEL_VERBOSE); stat = cache.stat; } - this.updateLastProcessedFile(path, stat!); const lastIsNotFound = !lastKey || lastKey.endsWith("-0-0"); const nowIsNotFound = cache.deleted; const type = lastIsNotFound && nowIsNotFound ? "invalid" : nowIsNotFound ? "delete" : "modified"; @@ -651,6 +715,27 @@ Offline Changed files: ${processFiles.length}`; return true; } + private async trackStorageFileModificationStrict(path: FilePath): Promise { + return await this.serializedForEvent(path, async () => { + if (!this.isThisModuleEnabled() || !(await this.isTargetFile(path))) return false; + const stat = await this.core.storageAccess.statHidden(path); + if (stat != null && stat.type != "file") return true; + const key = await this.fileToStatKey(path, stat); + if (this.getLastProcessedFileKey(path) === key) { + this._log(`${path} Already processed.`, LOG_LEVEL_DEBUG); + return true; + } + + const fileInfo = await this.loadFileWithInfo(path); + if (fileInfo.deleted) { + await this.deleteInternalFileOnDatabaseStrict(path); + return true; + } + await this.storeInternalFileToDatabaseStrict(fileInfo); + return true; + }); + } + // --> Conflict processing // Keep one in-flight conflict check per path so repeated sync events do not close the active merge dialogue. @@ -1500,6 +1585,18 @@ Offline Changed files: ${files.length}`; // --> Storage To Database Functions async __loadBaseSaveData(file: FilePath, includeContent = true): Promise { + return await this.loadBaseSaveData(file, includeContent, false); + } + + private async loadBaseSaveDataStrict(file: FilePath, includeContent = true): Promise { + return await this.loadBaseSaveData(file, includeContent, true); + } + + private async loadBaseSaveData( + file: FilePath, + includeContent: boolean, + strict: boolean + ): Promise { const prefixedFileName = addPrefix(file, ICHeader); const id = await this.path2id(prefixedFileName, ICHeader); try { @@ -1525,6 +1622,7 @@ Offline Changed files: ${files.length}`; return old; } } catch (ex) { + if (strict) throw ex; this._log(`Getting base save data failed`); this._log(ex, LOG_LEVEL_VERBOSE); return false; @@ -1552,6 +1650,18 @@ Offline Changed files: ${files.length}`; } async storeInternalFileToDatabase(file: InternalFileInfo | UXFileInfo, forceWrite = false) { + return await this.storeInternalFileToDatabaseCore(file, forceWrite, false); + } + + private async storeInternalFileToDatabaseStrict(file: InternalFileInfo | UXFileInfo, forceWrite = false) { + return await this.storeInternalFileToDatabaseCore(file, forceWrite, true); + } + + private async storeInternalFileToDatabaseCore( + file: InternalFileInfo | UXFileInfo, + forceWrite: boolean, + strict: boolean + ) { const storeFilePath = stripAllPrefixes(file.path); const storageFilePath = file.path; if (await this.services.vault.isIgnoredByIgnoreFile(storageFilePath)) { @@ -1565,9 +1675,11 @@ Offline Changed files: ${files.length}`; if (fileInfo.deleted) { throw new Error(`Hidden file:${storeFilePath} is deleted. This should not be occurred.`); } - const baseData = await this.__loadBaseSaveData(storeFilePath, true); + const baseData = strict + ? await this.loadBaseSaveDataStrict(storeFilePath, true) + : await this.__loadBaseSaveData(storeFilePath, true); if (baseData === false) throw new Error("Failed to load base data"); - if (baseData._rev && !forceWrite) { + if (baseData._rev && !baseData.deleted && !baseData._deleted && !forceWrite) { // Not newly created, we should check the content is actually modified. const isSame = await isDocContentSame(readAsBlob(baseData), fileInfo.body); if (isSame) { @@ -1592,11 +1704,15 @@ Offline Changed files: ${files.length}`; this.updateLastProcessed(storeFilePath, saveData, fileInfo.stat); } const success = ret && ret.ok; + if (!success && strict) { + throw new Error(`Failed to store hidden file in the database: ${storeFilePath}`); + } this._log(`STORAGE --> DB:${storageFilePath}: (hidden) ${success ? "Done" : "Failed"}`); return success; } catch (ex) { this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (strict) throw ex; return false; } }); @@ -1676,6 +1792,14 @@ Offline Changed files: ${files.length}`; } async deleteInternalFileOnDatabase(filenameSrc: FilePath, forceWrite = false) { + return await this.deleteInternalFileOnDatabaseCore(filenameSrc, forceWrite, false); + } + + private async deleteInternalFileOnDatabaseStrict(filenameSrc: FilePath, forceWrite = false) { + return await this.deleteInternalFileOnDatabaseCore(filenameSrc, forceWrite, true); + } + + private async deleteInternalFileOnDatabaseCore(filenameSrc: FilePath, _forceWrite: boolean, strict: boolean) { const storeFilePath = filenameSrc; const storageFilePath = filenameSrc; const displayFileName = filenameSrc; @@ -1686,18 +1810,26 @@ Offline Changed files: ${files.length}`; } return await serialized("file-" + prefixedFileName, async () => { try { - const baseData = await this.__loadBaseSaveData(storeFilePath, false); + const baseData = strict + ? await this.loadBaseSaveDataStrict(storeFilePath, false) + : await this.__loadBaseSaveData(storeFilePath, false); if (baseData === false) throw new Error("Failed to load base data during deleting"); if (baseData._conflicts !== undefined) { for (const conflictRev of baseData._conflicts) { - await this.localDatabase.removeRevision(baseData._id, conflictRev); + const removed = await this.localDatabase.removeRevision(baseData._id, conflictRev); + if (strict && removed === false) { + throw new Error(`Failed to remove conflicted hidden-file revision ${conflictRev}`); + } this._log( `STORAGE -x> DB: ${displayFileName}: (hidden) conflict removed ${baseData._rev} => ${conflictRev}`, LOG_LEVEL_VERBOSE ); } } - if (baseData.deleted) { + const alreadyDeleted = strict + ? !baseData._rev || baseData.deleted || baseData._deleted + : baseData.deleted; + if (alreadyDeleted) { this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) already deleted`, LOG_LEVEL_VERBOSE); this.updateLastProcessedDeletion(storeFilePath, baseData); return true; @@ -1718,11 +1850,13 @@ Offline Changed files: ${files.length}`; return true; } else { this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`); + if (strict) throw new Error(`Failed to delete hidden file from the database: ${storeFilePath}`); return false; } } catch (ex) { this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`); this._log(ex, LOG_LEVEL_VERBOSE); + if (strict) throw ex; return false; } }); @@ -2084,6 +2218,9 @@ Offline Changed files: ${files.length}`; // services.databaseEvents.handleOnDatabaseInitialisation(this._everyOnInitializeDatabase.bind(this)); services.appLifecycle.onSettingLoaded.addHandler(this._everyOnloadAfterLoadSettings.bind(this)); services.fileProcessing.processOptionalFileEvent.addHandler(this._anyProcessOptionalFileEvent.bind(this)); + services.fileProcessing.processOptionalFileEventWithResult?.addHandler( + this._processOptionalFileEventWithResult.bind(this) + ); services.conflict.getOptionalConflictCheckMethod.addHandler(this._anyGetOptionalConflictCheckMethod.bind(this)); services.replication.processOptionalSynchroniseResult.addHandler(this._anyProcessOptionalSyncFiles.bind(this)); services.setting.onRealiseSetting.addHandler(this._everyRealizeSettingSyncMode.bind(this)); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 5f04c8c8..59d1f437 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -256,6 +256,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow. +`test:e2e:obsidian:optional-file-save-completion` exercises Hidden File Sync and both Customisation Sync formats in one isolated real Obsidian Vault. It holds a local database write to verify that its event stays in the snapshot and that flush waits for the write, then checks unchanged notifications, subsequent edits, and repeated deletions. Injected save failures remain pending and must replay successfully before the next start reaches readiness. This local scenario disables replication and requires no remote service. It is outside `test:e2e:obsidian:local-suite`. + `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`. `test:e2e:obsidian:remote-feature-change` starts real Obsidian with continuous CouchDB replication, then changes the remote version document from generation 12 to generation 13 with an unknown feature. It waits for the control document to reach the local database and the active Replicator to retire, checks that another replication is refused, and verifies that an already accepted Vault note remains intact. After restarting the same Vault, the current remote declaration still blocks finite replication and the actual continuous connection attempt. No KV feature history is involved. It is a focused test outside `test:e2e:obsidian:local-suite`; recovery with a future compatible client remains a separate validation boundary. diff --git a/test/e2e-obsidian/scripts/optional-file-save-completion.ts b/test/e2e-obsidian/scripts/optional-file-save-completion.ts new file mode 100644 index 00000000..f6590eca --- /dev/null +++ b/test/e2e-obsidian/scripts/optional-file-save-completion.ts @@ -0,0 +1,230 @@ +/** Exercises optional file admission, actual save completion, and replay in a real Obsidian Vault. */ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const vault = await createTemporaryVault("obsidian-optional-save-"); + let session: ObsidianLiveSyncSession | undefined; + const observations: unknown[] = []; + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const settings = createE2eCouchDbPluginData( + // These scenarios use only the local database; every replication trigger is disabled. + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "optional-save-local" }, + { + syncInternalFiles: true, + usePluginSync: true, + usePluginSyncV2: true, + deviceAndVaultName: "optional-save-device", + watchInternalFileChanges: true, + syncInternalFilesTargetPatterns: "^\\.obsidian(?:$|/(?:optional-save|snippets)(?:/|$))", + syncInternalFilesIgnorePatterns: "", + autoSweepPlugins: false, + autoSweepPluginsPeriodic: false, + syncInternalFilesInterval: 0, + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + showVerboseLog: true, + } + ); + + async function start(restart = false): Promise { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cliBinary, + vault, + pluginStartup: restart ? "natural" : undefined, + pluginData: restart ? undefined : settings, + localStorageEntries: restart ? undefined : createE2eObsidianDeviceLocalState(vault.name), + lifecycle: { + beforeLaunch: async () => { + const registryPath = join(vault.userDataPath, "obsidian.json"); + const registry = JSON.parse(await readFile(registryPath, "utf8")); + registry.updateDisabled = true; + await writeFile(registryPath, JSON.stringify(registry)); + }, + }, + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + } + + async function evaluate(body: string): Promise { + if (!session) throw new Error("The Obsidian session is not running"); + return await evalObsidianJson( + cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + const manager=core.serviceModules.storageAccess.vaultManager; + const config=core.getAddOn('ConfigSync'); + const sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms)); + const require=(condition,message)=>{if(!condition)throw new Error(message);}; + const waitFor=async (predicate,message)=>{ + const until=Date.now()+10000; + while(!(await predicate())) { + if(Date.now()>until)throw new Error(message); + await sleep(20); + } + }; + const pending=async path=>(await manager.adapter.persistence.loadSnapshot()??[]) + .some(event=>event.args?.file?.path===path); + const entry=async path=>await core.localDatabase.getDBEntry(path,undefined,false,true); + const contents=doc=>Array.isArray(doc.data)?doc.data.join(''):doc.data; + const represents=(doc,content)=>doc&& !doc.deleted && + (contents(doc).includes(content)||contents(doc).includes(btoa(content))); + const notify=async path=>{ + await manager._watchVaultRawEvents(path); + await sleep(30); + }; + ${body} + })()`, + session.cliEnv + ); + } + + try { + await start(); + await evaluate(` + require(core.services.fileProcessing.processOptionalFileEventWithResult?.hasHandlers, + 'Typed optional providers were not registered'); + core.services.setting.setDeviceAndVaultName('optional-save-device'); + await core.services.setting.saveSettingData(); + for(const path of ['.obsidian/optional-save','.obsidian/snippets']) { + if(!(await app.vault.adapter.exists(path)))await app.vault.adapter.mkdir(path); + } + return JSON.stringify({ready:true}); + `); + + for (const mode of ["hidden", "config-v1", "config-v2"]) { + observations.push( + await evaluate(` + const mode=${JSON.stringify(mode)}; + core.settings.usePluginSyncV2=mode!=='config-v1'; + await core.services.setting.saveSettingData(); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Initial flush failed'); + const path=mode==='hidden'?'.obsidian/optional-save/content.json': + '.obsidian/snippets/optional-save-'+mode+'.css'; + const databasePath=mode==='hidden'?'i:'+path:mode==='config-v1'? + config.filenameToUnifiedKey(path):config.filenameWithUnifiedKey(path); + const content='optional save '+mode+' initial'; + const original=core.localDatabase.putDBEntry.bind(core.localDatabase); + let release; + const gate=new Promise(resolve=>{release=resolve;}); + let started=false; + core.localDatabase.putDBEntry=async data=>{ + if(data.path===databasePath){started=true;await gate;} + return await original(data); + }; + try { + await app.vault.adapter.write(path,content); + await notify(path); + await waitFor(()=>started,'The optional database save did not start: '+mode); + require(await pending(path),'The in-flight optional save was missing from the snapshot: '+mode); + let flushed=false; + const flush=core.services.fileProcessing.commitPendingFileEvents().then(result=>{flushed=true;return result;}); + await sleep(30); + require(!flushed,'Flush returned before the actual database save: '+mode); + release(); + require(await flush,'Flush failed after the write was released: '+mode); + } finally { + release(); + core.localDatabase.putDBEntry=original; + } + const first=await entry(databasePath); + require(represents(first,content),'The completed save did not contain the expected content: '+mode); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Unchanged notification failed: '+mode); + require((await entry(databasePath))._rev===first._rev,'Unchanged notification added a revision: '+mode); + const updated=content+' with a later edit'; + await app.vault.adapter.write(path,updated); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Later edit was blocked: '+mode); + require(represents(await entry(databasePath),updated),'Later edit was not saved: '+mode); + await app.vault.adapter.remove(path); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Deletion failed: '+mode); + const deleted=await core.localDatabase.getDBEntryMeta(databasePath,undefined,true); + require(deleted&&deleted.deleted,'Deletion was not represented: '+mode); + await notify(path); + require(await core.services.fileProcessing.commitPendingFileEvents(),'Repeated deletion failed: '+mode); + require((await core.localDatabase.getDBEntryMeta(databasePath,undefined,true))._rev===deleted._rev, + 'Repeated deletion added a revision: '+mode); + return JSON.stringify({mode,saveWaited:true,unchangedPreservedRevision:true,laterEditSaved:true,deletionReplayed:true}); + `) + ); + } + + const replay = [ + { path: ".obsidian/optional-save/replay.json", content: "hidden save retained across restart" }, + { + path: ".obsidian/snippets/optional-save-replay.css", + content: "customisation save retained across restart", + }, + ]; + observations.push( + await evaluate(` + const fixtures=${JSON.stringify(replay)}; + core.settings.usePluginSyncV2=true; + await core.services.setting.saveSettingData(); + const failures=new Set(fixtures.map(({path})=>path.endsWith('.json')?'i:'+path:config.filenameWithUnifiedKey(path))); + const original=core.localDatabase.putDBEntry.bind(core.localDatabase); + let failed=0; + core.localDatabase.putDBEntry=async data=>{ + if(failures.delete(data.path)){failed++;return false;} + return await original(data); + }; + try { + for(const {path,content} of fixtures) { + await app.vault.adapter.write(path,content); + await notify(path); + } + await waitFor(()=>failed===fixtures.length,'Injected optional failures were not reached'); + require(!(await core.services.fileProcessing.commitPendingFileEvents()),'Failed saves were acknowledged'); + for(const {path} of fixtures)require(await pending(path),'Failed save was lost: '+path); + } finally { + core.localDatabase.putDBEntry=original; + } + return JSON.stringify({failedSavesRetained:failed}); + `) + ); + await session!.app.stop(); + session = undefined; + await start(true); + observations.push( + await evaluate(` + const fixtures=${JSON.stringify(replay)}; + for(const {path,content} of fixtures) { + const databasePath=path.endsWith('.json')?'i:'+path:config.filenameWithUnifiedKey(path); + require(represents(await entry(databasePath),content),'Start-up did not recover '+path); + require(!(await pending(path)),'Recovered save is still pending: '+path); + } + return JSON.stringify({replayedBeforeReadiness:fixtures.length,ready:core.services.appLifecycle.isReady()}); + `) + ); + console.log(JSON.stringify({ ok: true, bundleHash, observations }, null, 2)); + } finally { + await session?.app.stop(); + await vault.dispose(); + } +} + +await main(); diff --git a/updates.md b/updates.md index 9467ef89..94859d63 100644 --- a/updates.md +++ b/updates.md @@ -24,6 +24,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - Local changes remain available for recovery while their database save is running, including when the application stops after a batch delay ends. - Manual synchronisation and settings application now wait for the accepted local saves to finish and report failures. Pending events retain the existing snapshot format. + - Hidden File Sync and Customisation Sync now report completion after saving, including delayed customisation updates. Failed saves remain pending, while unchanged notifications allow later edits to proceed. - Start-up reconciliation checks content when modification times are equal, so an edit within the same timestamp interval is not mistaken for an unchanged file. - Existing revision provenance preserves the correct edit branch. Unreadable or continually changing content remains unresolved rather than being reported as successfully reconciled. - We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) From b449dfd2bb8e2863f8ed396b931f57acfd9f7c0b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 26 Oct 2026 12:18:52 +0000 Subject: [PATCH 50/68] Prepare 1.0.41 notes, dependencies, and recovery coverage Align release metadata and audited development dependencies, and consolidate release notes with contributor acknowledgements. Cover upgrades, start-up integrity, text reflection, and interrupted saves in real Obsidian. Document the accepted provenance persistence interval: identical replayed content converges, safe merge retains the later remote content and ancestry, and overlapping edits remain available for review. --- ...local_file_save_completion_and_batching.md | 39 +- docs/specs_conflict_resolution.md | 43 ++ manifest.json | 2 +- package-lock.json | 409 ++++++++---------- package.json | 13 +- src/apps/cli/package.json | 6 +- src/apps/webapp/package.json | 4 +- src/apps/webpeer/package.json | 4 +- test/e2e-obsidian/README.md | 14 +- .../scripts/issue1207-livesync-batch.ts | 284 +++++++++++- test/e2e-obsidian/scripts/run-focused.ts | 3 + .../e2e-obsidian/scripts/startup-integrity.ts | 396 +++++++++++++++++ .../scripts/third-party-chunks.ts | 269 ++++++++++++ .../scripts/upgrade-livesync-batching.ts | 312 +++++++++++++ updates.md | 74 ++-- versions.json | 3 +- 16 files changed, 1583 insertions(+), 292 deletions(-) create mode 100644 test/e2e-obsidian/scripts/startup-integrity.ts create mode 100644 test/e2e-obsidian/scripts/third-party-chunks.ts create mode 100644 test/e2e-obsidian/scripts/upgrade-livesync-batching.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md index 27e6e4f6..33ee9b8c 100644 --- a/docs/adr/2026_10_local_file_save_completion_and_batching.md +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -143,9 +143,30 @@ policy-approved skip before the manager can acknowledge either safely. A crash after the database write but before snapshot removal can replay an operation. Replay must check current storage and revision provenance, retain -the existing protection against stale restored events, and avoid duplicate -revisions or deletion of a newer same-path file. It must be safe to interrupt -replay again. Existing saved snapshots remain readable. +the existing protection against stale restored events, and never delete a +newer same-path file. Once both the database write and provenance write have +completed, replay must not add another revision. If the database write +succeeded before provenance was recorded, replay may first create an identical +sibling even when no remote descendant has arrived. In the tested case +without a remote edit, replay creates that sibling. Duplicate handling then +tombstones one of the identical branches and converges to one live leaf with +the exact saved content. +Another restart must add no revision. Replay must remain safe to interrupt; +existing saved snapshots remain readable. + +The database write and the device-local provenance write are separate, +sequential operations. A stop between them can leave the previous provenance +record in place. This pre-provenance gap is the known exception to the +no-extra-revision result after both writes complete: replay can first create a +duplicate sibling; one of the identical branches is then tombstoned while +one live leaf retains the saved bytes when no remote descendant has arrived. If continuous +replication advances the local database during the gap, replay can instead +expose the local content as a sibling of that descendant. The [conflict +resolution specification](../specs_conflict_resolution.md#known-potentially-problematic-behaviour) +records the required mergeable and manual-resolution outcomes. The per-file +lock does not fence replication writes. The interruption window is expected +to be very rare, but its field frequency is unmeasured; this decision adds no +transaction or second journal to close the interval. Boolean failure, an exception, and an unavailable comparison are not successful completion. Keep unresolved work recoverable and report the failure through the @@ -202,6 +223,14 @@ outcomes through the ordinary non-forced file handler and its provenance rules: | Vault content has no provable origin and matches no current live leaf | Preserve it according to the existing independent-branch rules. Do not attach it to an arbitrary database winner. | | Required content or provenance cannot be read, or state changes during comparison | Retain unresolved work and report or re-evaluate it. Do not mark it reconciled merely because the timestamp comparison is `EVEN`. | +The table describes stable-state reconciliation, not atomic database and +provenance persistence. Replay in the pre-provenance gap may create an +identical sibling; with no remote descendant, one of the identical branches +is tombstoned, leaving one live leaf with the exact saved content. If replication has +already added a descendant, preserve its exact content through an existing +safe merge, or retain both branches for manual resolution. See [Known +potentially problematic behaviour](../specs_conflict_resolution.md#known-potentially-problematic-behaviour). + Revalidate the relevant file and database state before acting on a comparison. Reuse the existing content and provenance checks; avoid a separate conflict algorithm in the scanner. Metadata continues to reference Chunks containing the @@ -333,7 +362,9 @@ matrix is: | Scenario | Required observation | | -------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | | Stop after dequeue, timer release, or during a database write | Pending work remains recoverable; restart eventually represents the saved Vault edit. | -| Database write succeeds before snapshot cleanup; restart twice | Replay creates no duplicate revision and preserves later edits. | +| Database and provenance writes are complete before replay; restart twice | Replay adds no extra revision. | +| Stop after the database write but before provenance; no remote descendant; replay twice | The tested replay creates an identical sibling and tombstones one of the two branches; one live leaf retains exact saved content, and the second restart adds no revision. | +| A remote descendant arrives during the pre-provenance gap | The mergeable case preserves its exact content and ancestry; the overlapping case retains both bodies for manual resolution. | | Old and new events share a path; snapshot writes overlap | Completing old work cannot remove new work or regress the durable snapshot. | | Handler returns false, throws, or snapshot persistence fails | Flush reports failure; pending work is retained; unrelated recoverable work can continue. | | Buffered or in-progress work exists with no waiting timer | Flush still waits for actual completion. | diff --git a/docs/specs_conflict_resolution.md b/docs/specs_conflict_resolution.md index aabf5807..6f29fedb 100644 --- a/docs/specs_conflict_resolution.md +++ b/docs/specs_conflict_resolution.md @@ -166,6 +166,49 @@ For ordinary saves and incoming reflection, a missing or unreadable recorded bas Provenance keys follow the selected [Unicode path identity](design_docs/unicode_path_identity.md#state-which-follows-the-selected-identity). In NFC mode, equivalent path spellings address the same record; the content evidence continues to describe file bytes, independently of that path spelling. +## Known potentially problematic behaviour + +Saving a local file writes its revision to the local database, then writes +device-local provenance in a separate operation. The writes are not atomic. +If the process stops between them, provenance can still identify the revision +which preceded the local edit. Continuous replication which was already +running can add a remote descendant to the local database during this +interval. The per-file lock covers file handling and Vault reflection; it +does not block raw replication writes to the database. + +For example, local content `2-B` may have been saved as a child of +displayed revision `1`, while provenance still identifies `1`. +On restart, replay of the pending save can treat `2-B` as an edit of +`1` and write sibling `2-D`. The tested case demonstrates that this +pre-provenance gap alone can produce the duplicate. Without a remote +descendant, replay creates this sibling; duplicate handling tombstones one +of the identical branches and converges to one live leaf whose exact content +matches the saved local file. A second restart must add no revision. + +If replication has already added descendant `3-C` during the gap, replay +can expose an additional branch. When the revisions have a complete shared +base and their text changes can be combined unambiguously, sensible merge can retain the +exact content of the incoming descendant. In the tested fixture, merge +produced revision `4` as a child of `3-C`; its file bytes were exactly +those of `3-C`, `3-C` remained in its ancestry, and sibling +`2-D` was tombstoned. The acceptance condition is retaining +`3-C`'s exact content; it need not remain the current leaf. Both Vaults +converged to those bytes. This observed outcome is limited to that mergeable +fixture and is not a universal no-data-loss guarantee. + +When changes cannot be combined safely, as in the tested competing whole-line +replacements, sensible merge does not combine the bodies. Both versions remain available for the existing +manual conflict dialogue, even when sensible merge is enabled. + +An interruption in the database/provenance interval is expected to be very +rare, especially when combined with a remote descendant arriving in that +interval; the field frequency is unmeasured. Start-up replays restored work, +scans, and commits pending local file work before marking the plug-in ready; +ordinary replication proceeds after that ready boundary. This does not claim +remote synchronisation before start-up readiness. The accepted design adds no +cross-store transaction, second journal, or new persistent state to close the +interval. + ## Operations while a conflict exists - Editing a file writes a child of its recorded or uniquely reconstructed displayed revision. diff --git a/manifest.json b/manifest.json index 267d35b5..efe56799 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.40", + "version": "1.0.41", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index e17dd5e7..5393a72f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.40", + "version": "1.0.41", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.40", + "version": "1.0.41", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -55,7 +55,7 @@ "@types/pouchdb-replication": "^6.4.7", "@types/transform-pouch": "^1.0.6", "@typescript-eslint/parser": "8.69.0", - "@vitest/coverage-v8": "^4.1.11", + "@vitest/coverage-v8": "^4.1.12", "@vrtmrz/obsidian-test-session": "0.2.6", "dotenv-cli": "^11.0.0", "esbuild": "0.28.1", @@ -90,8 +90,8 @@ "tsx": "^4.21.0", "typescript": "6.0.3", "typescript-eslint": "8.69.0", - "vite": "^8.0.16", - "vitest": "^4.1.11", + "vite": "^8.1.6", + "vitest": "^4.1.12", "yaml": "^2.8.2" } }, @@ -1945,22 +1945,25 @@ } }, "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", - "integrity": "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.6.tgz", + "integrity": "sha512-mRyDmEw9/fCDXCGfL1sbgIw/GUK0PVxRW4YwgPN61F3LQ+fSyoBFarJVzgGyhF6kRzEQZ0FejDqm4WLXSeqQmw==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@tybys/wasm-util": "^0.10.2" + "@tybys/wasm-util": "^0.10.3" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" }, "funding": { "type": "github", "url": "https://github.com/sponsors/Brooooooklyn" }, "peerDependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1" + "@emnapi/core": "^1.7.1 || 2.0.0-beta.2", + "@emnapi/runtime": "^1.7.1 || 2.0.0-beta.2" } }, "node_modules/@noble/curves": { @@ -2012,9 +2015,9 @@ "license": "MIT" }, "node_modules/@oxc-project/types": { - "version": "0.133.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", - "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", + "version": "0.139.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", + "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", "dev": true, "license": "MIT", "funding": { @@ -2222,9 +2225,9 @@ } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", - "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", + "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", "cpu": [ "arm64" ], @@ -2239,9 +2242,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", - "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", + "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", "cpu": [ "arm64" ], @@ -2256,9 +2259,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", - "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", + "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", "cpu": [ "x64" ], @@ -2273,9 +2276,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", - "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", + "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", "cpu": [ "x64" ], @@ -2290,9 +2293,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", - "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", + "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", "cpu": [ "arm" ], @@ -2307,13 +2310,16 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", - "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", + "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2324,13 +2330,16 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", - "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", + "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2341,13 +2350,16 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", - "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", + "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2358,13 +2370,16 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", - "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", + "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2375,13 +2390,16 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", - "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", + "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2392,13 +2410,16 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", - "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", + "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2409,9 +2430,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", - "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", + "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", "cpu": [ "arm64" ], @@ -2426,9 +2447,9 @@ } }, "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", - "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", + "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", "cpu": [ "wasm32" ], @@ -2436,52 +2457,18 @@ "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.10.0", - "@emnapi/runtime": "1.10.0", - "@napi-rs/wasm-runtime": "^1.1.4" + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" }, "engines": { "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", - "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", + "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", "cpu": [ "arm64" ], @@ -2496,9 +2483,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", - "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", + "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", "cpu": [ "x64" ], @@ -3347,9 +3334,9 @@ "license": "MIT" }, "node_modules/@tybys/wasm-util": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", - "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", + "integrity": "sha512-W3c4gRigFS0T/Ma4qIYF3GDAc5AQdHb1yL5znJT1Zv1YaD9Kitx656wBjvr19qbiosmZT8lWDM5BEMynUqX65A==", "dev": true, "license": "MIT", "optional": true, @@ -3963,14 +3950,14 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.11.tgz", - "integrity": "sha512-8MVGEFnJIcdGjcbfKmeq8z0pZHH0JlVtoVZH9Q/qwUp6wyFnEJUBMrw9DCaj+ra3vShGmhavjalMIhPNxZAUcw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.12.tgz", + "integrity": "sha512-6S7/iVknKglLAxMFxQGNppBI0iqJS2NgM6FBxavitj7KaDq/vywOWGf6dGu7F2L1Qy0G6S3uTp7aGshHrzK8JA==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", @@ -3984,8 +3971,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.1.11", - "vitest": "4.1.11" + "@vitest/browser": "4.1.12", + "vitest": "4.1.12" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -3994,16 +3981,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", - "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.12.tgz", + "integrity": "sha512-vFlTt+YEu9WBpNjdGayNmlMVzrWPi1+sv853/7t2FU7hpO5gAFBKHzQmkCq5Xjpr+evegF2994LQT3hF3mRlqQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -4012,13 +3999,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", - "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.12.tgz", + "integrity": "sha512-fKShNpKZzupWZuY0z7egydvL0sqW4INEZBEExMBMAa6izlENEVHwoXgdaTCaBZ5fwlr4xf2o1wXC83n9RpO+NA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.11", + "@vitest/spy": "4.1.12", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -4039,9 +4026,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", - "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.12.tgz", + "integrity": "sha512-XW9O8+Mxx4VpCtkYdMJLZQQwsVryP9wk7cPKDxtXFfwSak6yRh5+2fep/Y0XF/ElBvJHq8bcYdi6aeuai0opZA==", "dev": true, "license": "MIT", "dependencies": { @@ -4052,13 +4039,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", - "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.12.tgz", + "integrity": "sha512-NKlVKCr5QnCe8jijdMnmkpHHU+WTqiLFR6G0pHji9uRjXK9xJl1k1nTw6F3pOOj2PDcsDt2GKQMI7wIsaT491w==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "pathe": "^2.0.3" }, "funding": { @@ -4066,14 +4053,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", - "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.12.tgz", + "integrity": "sha512-ko1y1Qgb/slzQ9mr+b89Cuqud8WG1TKtsPD8r7ZUBdvVOGt85Af2HwCqHujrhwJq95hs/9RM7fcAPg57o1vlzw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/pretty-format": "4.1.12", + "@vitest/utils": "4.1.12", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -4082,9 +4069,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", - "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.12.tgz", + "integrity": "sha512-RGq9x4JNGNTZHEJxvN0oSiJjeSCExm0n2gZKKJZHY9cX5AVA6M7VJILBT3ZJ/tJ/0J9uWT7fETV9/btoi//vSg==", "dev": true, "license": "MIT", "funding": { @@ -4092,13 +4079,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", - "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.12.tgz", + "integrity": "sha512-Q3awXrwOfEV+qc3WmO5ONyE0yO7Q1gQ9/oyHxGLZ/pl7N3NO/ZztqQwKLuwMkV+CA1JYSIeO3gkHnRDIo4AVaA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", + "@vitest/pretty-format": "4.1.12", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -6377,9 +6364,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", - "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", "dev": true, "funding": [ { @@ -8814,6 +8801,19 @@ "dev": true, "license": "ISC" }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -9586,13 +9586,13 @@ } }, "node_modules/rolldown": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", - "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", + "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.133.0", + "@oxc-project/types": "=0.139.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -9602,21 +9602,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.3", - "@rolldown/binding-darwin-arm64": "1.0.3", - "@rolldown/binding-darwin-x64": "1.0.3", - "@rolldown/binding-freebsd-x64": "1.0.3", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", - "@rolldown/binding-linux-arm64-gnu": "1.0.3", - "@rolldown/binding-linux-arm64-musl": "1.0.3", - "@rolldown/binding-linux-ppc64-gnu": "1.0.3", - "@rolldown/binding-linux-s390x-gnu": "1.0.3", - "@rolldown/binding-linux-x64-gnu": "1.0.3", - "@rolldown/binding-linux-x64-musl": "1.0.3", - "@rolldown/binding-openharmony-arm64": "1.0.3", - "@rolldown/binding-wasm32-wasi": "1.0.3", - "@rolldown/binding-win32-arm64-msvc": "1.0.3", - "@rolldown/binding-win32-x64-msvc": "1.0.3" + "@rolldown/binding-android-arm64": "1.1.5", + "@rolldown/binding-darwin-arm64": "1.1.5", + "@rolldown/binding-darwin-x64": "1.1.5", + "@rolldown/binding-freebsd-x64": "1.1.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", + "@rolldown/binding-linux-arm64-gnu": "1.1.5", + "@rolldown/binding-linux-arm64-musl": "1.1.5", + "@rolldown/binding-linux-ppc64-gnu": "1.1.5", + "@rolldown/binding-linux-s390x-gnu": "1.1.5", + "@rolldown/binding-linux-x64-gnu": "1.1.5", + "@rolldown/binding-linux-x64-musl": "1.1.5", + "@rolldown/binding-openharmony-arm64": "1.1.5", + "@rolldown/binding-wasm32-wasi": "1.1.5", + "@rolldown/binding-win32-arm64-msvc": "1.1.5", + "@rolldown/binding-win32-x64-msvc": "1.1.5" } }, "node_modules/sade": { @@ -10465,19 +10465,6 @@ } } }, - "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/tinyrainbow": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", @@ -10847,16 +10834,16 @@ } }, "node_modules/vite": { - "version": "8.0.16", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", - "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", + "version": "8.1.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.6.tgz", + "integrity": "sha512-aRAHIxx5SW5zc1sMqFwJU/t5qaFfXb5Hj0J7W0VXihu/X99+346tz6080tO1T1xVIJORX/b3zVTXGFqUXh35uw==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.32.0", - "picomatch": "^4.0.4", - "postcss": "^8.5.15", - "rolldown": "1.0.3", + "picomatch": "^4.0.5", + "postcss": "^8.5.17", + "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "bin": { @@ -10873,7 +10860,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.1.18", + "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -10939,19 +10926,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/vite/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/vitefu": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.2.tgz", @@ -10973,19 +10947,19 @@ } }, "node_modules/vitest": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", - "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.12.tgz", + "integrity": "sha512-5eWdz2yb3ajxWF3C8C4TjkXl7FGm3n2E/VlazALrQHzK7nYMSGFnCZto0bbPNl77AwkOZVLk0YnBAejUu0NiWA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.11", - "@vitest/mocker": "4.1.11", - "@vitest/pretty-format": "4.1.11", - "@vitest/runner": "4.1.11", - "@vitest/snapshot": "4.1.11", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/expect": "4.1.12", + "@vitest/mocker": "4.1.12", + "@vitest/pretty-format": "4.1.12", + "@vitest/runner": "4.1.12", + "@vitest/snapshot": "4.1.12", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -11013,12 +10987,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.11", - "@vitest/browser-preview": "4.1.11", - "@vitest/browser-webdriverio": "4.1.11", - "@vitest/coverage-istanbul": "4.1.11", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/browser-playwright": "4.1.12", + "@vitest/browser-preview": "4.1.12", + "@vitest/browser-webdriverio": "^4.1.11", + "@vitest/coverage-istanbul": "4.1.12", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" @@ -11069,19 +11043,6 @@ "dev": true, "license": "MIT" }, - "node_modules/vitest/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/vuvuzela": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/vuvuzela/-/vuvuzela-1.0.3.tgz", @@ -11408,7 +11369,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.40-cli", + "version": "1.0.41-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11427,13 +11388,13 @@ }, "devDependencies": { "typescript": "6.0.3", - "vite": "^8.0.16", - "vitest": "^4.1.11" + "vite": "^8.1.6", + "vitest": "^4.1.12" } }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.40-webapp", + "version": "1.0.41-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11441,11 +11402,11 @@ "@sveltejs/vite-plugin-svelte": "^7.1.2", "svelte": "5.56.3", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } }, "src/apps/webpeer": { - "version": "1.0.40-webpeer", + "version": "1.0.41-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11456,7 +11417,7 @@ "svelte": "5.56.3", "svelte-check": "^4.6.0", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } } } diff --git a/package.json b/package.json index 1c1bea1a..d5927927 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.40", + "version": "1.0.41", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", @@ -127,7 +127,10 @@ "test:docker-all:start": "npm run test:docker-all:up && sleep 5 && npm run test:docker-all:init", "test:docker-all:stop": "npm run test:docker-all:down", "update-workspaces": "node update-workspaces.mjs", - "version": "node version-bump.mjs && node update-workspaces.mjs && npm run pretty:json && git add manifest.json versions.json src/apps/cli/package.json src/apps/webpeer/package.json src/apps/webapp/package.json" + "version": "node version-bump.mjs && node update-workspaces.mjs && npm run pretty:json && git add manifest.json versions.json src/apps/cli/package.json src/apps/webpeer/package.json src/apps/webapp/package.json", + "test:e2e:obsidian:startup-integrity": "tsx test/e2e-obsidian/scripts/startup-integrity.ts", + "test:e2e:obsidian:third-party-chunks": "tsx test/e2e-obsidian/scripts/third-party-chunks.ts", + "test:e2e:obsidian:upgrade-livesync-batching": "tsx test/e2e-obsidian/scripts/upgrade-livesync-batching.ts" }, "keywords": [], "author": "vorotamoroz", @@ -151,7 +154,7 @@ "@types/pouchdb-replication": "^6.4.7", "@types/transform-pouch": "^1.0.6", "@typescript-eslint/parser": "8.69.0", - "@vitest/coverage-v8": "^4.1.11", + "@vitest/coverage-v8": "^4.1.12", "@vrtmrz/obsidian-test-session": "0.2.6", "dotenv-cli": "^11.0.0", "esbuild": "0.28.1", @@ -186,8 +189,8 @@ "tsx": "^4.21.0", "typescript": "6.0.3", "typescript-eslint": "8.69.0", - "vite": "^8.0.16", - "vitest": "^4.1.11", + "vite": "^8.1.6", + "vitest": "^4.1.12", "yaml": "^2.8.2" }, "dependencies": { diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index e8d7ca9c..7f73f7f1 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.40-cli", + "version": "1.0.41-cli", "main": "dist/index.cjs", "type": "module", "scripts": { @@ -52,7 +52,7 @@ }, "devDependencies": { "typescript": "6.0.3", - "vite": "^8.0.16", - "vitest": "^4.1.11" + "vite": "^8.1.6", + "vitest": "^4.1.12" } } diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index 487af8bf..d0f4f5ce 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.40-webapp", + "version": "1.0.41-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { @@ -21,6 +21,6 @@ "@sveltejs/vite-plugin-svelte": "^7.1.2", "svelte": "5.56.3", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } } diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index 22594247..c7654ed5 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.40-webpeer", + "version": "1.0.41-webpeer", "type": "module", "scripts": { "dev": "vite", @@ -24,6 +24,6 @@ "svelte": "5.56.3", "svelte-check": "^4.6.0", "typescript": "6.0.3", - "vite": "^8.0.16" + "vite": "^8.1.6" } } diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index fd543794..ae4790b3 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -98,6 +98,10 @@ The same scenario checks the five actions in Reporting Tools, the separate Check `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. +`startup-integrity` uses the registered Hatch integrity-check event with real Obsidian storage and a local database. It verifies that healthy files are inspected without reading their contents, that a controlled stale stat followed by matching real bytes does not produce a repair dialogue, and that a changed final stat leaves the check incomplete until a settled retry. The stat variations are injected; this scenario exercises the checker used during start-up rather than restarting the process. Natural configured start-up remains covered by `startup-scan`. + +`third-party-chunks` reflects synthetic externally generated text through the installed Commonlib and the real Obsidian file handler. It supplies a split surrogate pair at the file-access read boundary, with size checking enabled, then verifies exact Vault and on-disk text, unchanged Metadata, and matching persisted provenance. Complete-pair text and a binary attachment provide controls. The underlying Metadata and Chunks are real local database entries; this does not establish that CouchDB or a particular third-party application generates this chunk layout. No remote service is required. + `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. The principal entry points are: @@ -265,7 +269,13 @@ The workflow also opens the Customisation Sync dialogue. `--case=visibility` che `--case=automatic` separately checks that choosing **Automatic** while Hidden File Sync is disabled displays a Notice without changing modes, files, Metadata revisions, or processed markers. The fixture starts with a saved Automatic selection and requires it to survive start-up and both rejected actions. It also checks disabling the feature after opening **Initial Action**. After enablement, selecting a whole plug-in must initialise both **MAIN** and **DATA**, and transfer their files to the other Vault. Separate temporary Vaults exercise **Overwrite Remote**, **Overwrite Local**, and **Use newer**. The latter two start with different local and database contents; deterministic per-file modification times require **Use newer** to keep the newer database MAIN content and the newer local DATA content. -`test:e2e:obsidian:livesync-batch` uses two real Obsidian Vaults and a disposable local CouchDB to compare batching disabled, the older `batchSave` choice alone, and the explicit LiveSync batching choice. It covers repeated edits in both directions, template-like writes, deletion and recreation, incoming changes, maximum delay, concurrent paths, manual flush, and restart recovery before and after a database write. Restarts use the saved profile with natural plug-in start-up. `ISSUE1207_CASE_FILTER` selects case names; `ISSUE1207_RESTART_BUCKET=same|exact|different` exercises interrupted processing at the selected timestamp condition. `ISSUE1207_RESTART_SAVE_AFTER_EDIT=true` separately checks a settings save after the pending edit. Use `npm run test:e2e:obsidian:focused -- livesync-batch` to build and run the default matrix. This scenario requires a localhost CouchDB and is outside the complete local suite. +`test:e2e:obsidian:livesync-batch` uses two real Obsidian Vaults and a disposable local CouchDB to compare batching disabled, the older `batchSave` choice alone, and the explicit LiveSync batching choice. It covers repeated edits in both directions, template-like writes, deletion and recreation, incoming changes, maximum delay, concurrent paths, manual flush, and restart recovery before and after a local database write. Restarts use the saved profile with natural plug-in start-up. + +`ISSUE1207_CASE_FILTER` selects case names; `ISSUE1207_RESTART_BUCKET=same|exact|different` exercises interrupted processing at the selected timestamp condition. `ISSUE1207_RESTART_SAVE_AFTER_EDIT=true` separately checks a settings save after the pending edit. + +`ISSUE1207_RESTART_BEFORE_PROVENANCE=true` deliberately stops after the local database write and before device-local provenance is recorded. The case without a remote edit demonstrates replay creating an identical sibling and tombstoning one of the two branches; it must converge to one live leaf with exact saved content, and a second restart must add no revision. + +`ISSUE1207_RESTART_REMOTE_EDIT=mergeable|conflicting` requires `ISSUE1207_RESTART_BEFORE_PROVENANCE=true`. These cases persist the enabled test settings before editing so continuous replication remains active, receive a remote descendant before stopping, and restart the same profile. Both remote cases enable sensible merge. The pause is deliberately injected to observe the ordering; it is not a measure of field frequency. The `mergeable` case requires the final bytes to equal the incoming descendant exactly, with that revision retained in ancestry and further restart stable. The `conflicting` case uses a whole-line replacement and requires both bodies to remain for the manual conflict dialogue. The cases are named `enabled-before-provenance-mergeable-restart` and `enabled-before-provenance-conflicting-restart`. Use `npm run test:e2e:obsidian:focused -- livesync-batch` to build and run the default matrix. This scenario requires a localhost CouchDB and is outside the complete local suite. `test:e2e:obsidian:optional-file-save-completion` exercises Hidden File Sync and both Customisation Sync formats in one isolated real Obsidian Vault. It holds a local database write to verify that its event stays in the snapshot and that flush waits for the write, then checks unchanged notifications, subsequent edits, and repeated deletions. Injected save failures remain pending and must replay successfully before the next start reaches readiness. This local scenario disables replication and requires no remote service. It is outside `test:e2e:obsidian:local-suite`. @@ -281,6 +291,8 @@ The workflow also opens the Customisation Sync dialogue. `--case=visibility` che `test:e2e:obsidian:setting-markdown-roundtrip` generates Markdown with and without credentials through the real export command, copies each original file into a separate fresh Vault, and imports it through **Parse setting file** and the visible **Apply settings** dialogue. Each receiver starts with default plug-in settings and no connection profiles, with its own configuration encryption passphrase provided independently of the source through localStorage. The scenario verifies ordinary settings, conditional credentials and complete profiles, encrypted local persistence, and restoration after restarting the same Vault and profile without reseeding settings or localStorage. Imports without credentials may create new profiles through the existing legacy migration; those profiles must contain none of the omitted source credentials. This scenario requires no remote service and is included in the focused wrapper and local suite. +`test:e2e:obsidian:upgrade-livesync-batching` verifies the introduction of the separate LiveSync batching choice using the pinned, published 1.0.40 artefacts. It first saves and restarts the old release with legacy batching enabled and LiveSync mode selected. Replacing only the plug-in artefacts must retain the Vault, local database, device identity, remote profile, and existing Metadata revision, while leaving the new LiveSync batching choice disabled. The upgraded Vault and an independent current-version Vault exchange notes through CouchDB. Natural restarts retain both the disabled LiveSync choice and the original batching choice when switching to periodic mode. The source release is downloaded into the verified release cache; `E2E_LIVESYNC_TARGET_ARTIFACT_ROOT` selects an exact target build. This tests the update behaviour, while installation through BRAT remains a separate acceptance step. + `test:e2e:obsidian:upgrade-from-stable` is the release-acceptance upgrade workflow. It installs the exact published 0.25.83 artefacts into an isolated Vault, verifies their pinned SHA-256 values, and then replaces only the plug-in artefacts with the current target while retaining the same Vault and isolated Obsidian profile. The first run downloads the old release into the ignored `_testdata/releases` cache; every later run verifies the cached bytes before use. The workflow first exercises a non-empty legacy settings document which has no `isConfigured` or file-name case value. It verifies that 0.25.83 treats a default-equivalent document as unconfigured. That release can persist the inferred boolean during a later, unrelated settings-save event, so the runner accepts either an absent value or the inferred `false` on disk, then restores the same minimal pre-flag document deliberately before installing 1.0. The target independently proves its direct migration: the Vault remains unconfigured instead of receiving new-Vault recommendations, case-insensitive handling becomes explicit, no compatibility pause or acknowledgement marker is created while onboarding remains pending, and a second 1.0 start is idempotent. The absent marker is deliberately deferred while onboarding is pending; a later configured start records the current version if no other review is required. This fixture rewrite is limited to the missing-flag boundary; the configured transport upgrades use only state created and saved by 0.25.83 itself. diff --git a/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts index 6b249b38..1ccc0ab0 100644 --- a/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts +++ b/test/e2e-obsidian/scripts/issue1207-livesync-batch.ts @@ -180,12 +180,17 @@ async function waitFor( throw new Error(`State did not settle: ${JSON.stringify(state)}`); } -async function settled(pair: Pair, path: string, body: string): Promise<[Snapshot, Snapshot]> { +async function settled( + pair: Pair, + path: string, + body: string, + allowDeletedBranches = false +): Promise<[Snapshot, Snapshot]> { const ready = (state: Snapshot) => state.content === body && liveLeaves(state).length === 1 && liveLeaves(state)[0].body === body && - state.leaves.length === 1 && + (allowDeletedBranches || state.leaves.length === 1) && !state.waiting.some((item) => item.path === path); const deadline = Date.now() + 20000; let states: [Snapshot, Snapshot]; @@ -234,6 +239,15 @@ async function main(): Promise { await assertCouchDbReachable(config); const dbName = makeUniqueDatabaseName(config.dbPrefix, "live-batch"); const couch = { ...config, dbName }; + const remoteRestartEdit = process.env.ISSUE1207_RESTART_REMOTE_EDIT; + check( + !remoteRestartEdit || ["mergeable", "conflicting"].includes(remoteRestartEdit), + "Unknown remote edit for the provenance restart case" + ); + check( + !remoteRestartEdit || process.env.ISSUE1207_RESTART_BEFORE_PROVENANCE === "true", + "A remote restart edit requires the provenance gate" + ); const overrides = { doctorProcessedVersion: "1.0.0", notifyThresholdOfRemoteStorageSize: -1, @@ -248,7 +262,7 @@ async function main(): Promise { syncAfterMerge: false, checkConflictOnlyOnOpen: true, showMergeDialogOnlyOnActive: true, - disableMarkdownAutoMerge: true, + disableMarkdownAutoMerge: !remoteRestartEdit, resolveConflictsByNewerFile: false, writeDocumentsIfConflicted: false, writeLogToTheFile: false, @@ -259,8 +273,6 @@ async function main(): Promise { evidence.commonlibVersion = JSON.parse( await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8") ).version; - evidence.sourceBaseRevision = "73a14d4e4b0b414a1acfe170d01500abb3dca2e9"; - evidence.unreleasedImplementation = true; if (process.env.ISSUE1207_COMMONLIB_ARTEFACT) { evidence.commonlibArtefactSha256 = createHash("sha256") .update(new Uint8Array(readFileSync(process.env.ISSUE1207_COMMONLIB_ARTEFACT))) @@ -370,6 +382,31 @@ async function main(): Promise { type:item.type,path:item.args?.file?.path,oldPath:item.args?.oldPath}))};` ), }); + if (name === "enabled-settings-save-restart" || name.startsWith("enabled-before-provenance")) { + observations.push({ + name, + device, + revisionEvidence: await evaluate( + page, + ` + const path=${JSON.stringify(`${prefix}restart.md`)}; + const id=await core.services.path.path2id(path); + const leaves=await core.localDatabase.getRaw(id,{open_revs:'all',revs:true}); + const revisions=new Set(); + for(const leaf of leaves){const h=leaf.ok?._revisions; + for(let i=0;i<(h?.ids.length??0);i++)revisions.add((h.start-i)+'-'+h.ids[i]);} + const result=[]; + for(const rev of revisions){ + const raw=await core.localDatabase.getRaw(id,{rev,revs:true}).catch(error=>({error:String(error)})); + const doc=await core.localDatabase.getDBEntry(path,{rev,revs:true},false,true,true).catch(()=>false); + result.push({rev,deleted:raw._deleted??false,ctime:raw.ctime,mtime:raw.mtime, + size:raw.size,error:raw.error,body:doc?(Array.isArray(doc.data)?doc.data.join(''):doc.data):null}); + } + return {revisions:result,provenance:await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(path)}; + ` + ), + }); + } await page.screenshot({ path: join(diagnostics, `${name}-${device}-failure.png`) }); try { await evaluate(page, "await manager.waitForIdle();"); @@ -663,27 +700,61 @@ async function main(): Promise { const saveAfterEdit = process.env.ISSUE1207_RESTART_SAVE_AFTER_EDIT === "true"; const restartBucket = process.env.ISSUE1207_RESTART_BUCKET; + const stopBeforeProvenance = process.env.ISSUE1207_RESTART_BEFORE_PROVENANCE === "true"; + check(!stopBeforeProvenance || !restartBucket, "The provenance and timestamp gates are separate cases"); check( !restartBucket || ["same", "exact", "different"].includes(restartBucket), "Unknown restart timestamp condition" ); - const restartName = restartBucket - ? `enabled-inflight-restart-${restartBucket}` - : saveAfterEdit - ? "enabled-settings-save-restart" - : "enabled-restart"; + const restartName = stopBeforeProvenance + ? `enabled-before-provenance${remoteRestartEdit ? `-${remoteRestartEdit}` : ""}-restart` + : restartBucket + ? `enabled-inflight-restart-${restartBucket}` + : saveAfterEdit + ? "enabled-settings-save-restart" + : "enabled-restart"; await run(restartName, async () => { const path = `${prefix}restart.md`; + const beforeContent = + remoteRestartEdit === "mergeable" + ? "Heading\n\nLocal: original\n\nRemote: original\n\nFooter\n" + : "Before restart\n"; + const localContent = + remoteRestartEdit === "mergeable" + ? "Heading\n\nLocal: saved\n\nRemote: original\n\nFooter\n" + : "Retained after restart\n"; + const remoteContent = + remoteRestartEdit === "mergeable" + ? "Heading\n\nLocal: saved\n\nRemote: changed remotely\n\nFooter\n" + : "Remote edit after the durable save\n"; const beforeRestart = await withPair(async (pair) => { await options(pair.a, "enabled-batch", 5, 60); await options(pair.b, "normal"); - if (!saveAfterEdit) + // Remote cases keep continuous replication running while the local save is held. + if (!saveAfterEdit || remoteRestartEdit) await evaluate( pair.a, "await core.services.setting.saveSettingData(); await core.services.control.applySettings();" ); - await create(pair.b, path, "Before restart\n"); - const initial = await settled(pair, path, "Before restart\n"); + await create(pair.b, path, beforeContent); + const initial = await settled(pair, path, beforeContent); + if (stopBeforeProvenance) { + await evaluate( + pair.a, + ` + const access=core.serviceModules.databaseFileAccess; + const original=access.storeWithBaseRevision.bind(access); + access.storeWithBaseRevision=async(file,...args)=>{ + const revision=await original(file,...args); + if(file.path===${JSON.stringify(path)}&&revision){ + globalThis.__issue1207Batch.beforeProvenanceRevision=revision; + await new Promise(resolve=>setTimeout(resolve,30000)); + } + return revision; + }; + ` + ); + } if (restartBucket) { check(initial[0].databaseMtime !== null, "Missing initial database mtime"); const mtime = @@ -703,11 +774,27 @@ async function main(): Promise { }; await app.vault.modify(app.vault.getFileByPath(${JSON.stringify(path)}),"Retained after restart\\n",{mtime:${mtime}});` ); - } else await modify(pair.a, path, "Retained after restart\n"); + } else await modify(pair.a, path, localContent); const pending = await waitFor(pair.a, path, (state) => state.waiting.some((item) => item.path === path) ); - if (saveAfterEdit) await evaluate(pair.a, "await core.services.setting.saveSettingData();"); + if (saveAfterEdit && !remoteRestartEdit) { + await evaluate(pair.a, "await core.services.setting.saveSettingData();"); + observations.push({ + phase: "settings-save-before-stop", + initial, + pending, + current: await Promise.all([snapshot(pair.a, path), snapshot(pair.b, path)]), + sourceProbe: await evaluate( + pair.a, + `return { + events:globalThis.__issue1207Batch.events, + changes:globalThis.__issue1207Batch.changes, + provenance:await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(${JSON.stringify(path)}) + };` + ), + }); + } if (restartBucket) await evaluate( pair.a, @@ -715,6 +802,82 @@ async function main(): Promise { while(!globalThis.__issue1207Batch.inflightPaused&&Date.now()setTimeout(resolve,10)); if(!globalThis.__issue1207Batch.inflightPaused) throw new Error('The in-flight test gate did not start');` ); + let durableBeforeProvenance: Snapshot | undefined; + let remoteAdvanced: Snapshot | undefined; + if (stopBeforeProvenance) { + await evaluate( + pair.a, + ` + const deadline=Date.now()+core.services.setting.currentSettings().batchSaveMinimumDelay*1000+5000; + while(!globalThis.__issue1207Batch.beforeProvenanceRevision&&Date.now()setTimeout(resolve,10)); + if(!globalThis.__issue1207Batch.beforeProvenanceRevision)throw new Error('The durable-write provenance gate did not start'); + ` + ); + durableBeforeProvenance = await snapshot(pair.a, path); + const provenance = await evaluate<{ revision: string }>( + pair.a, + `return await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(${JSON.stringify(path)});` + ); + check( + durableBeforeProvenance.rev !== initial[0].rev && + liveLeaves(durableBeforeProvenance)[0]?.body === localContent, + "The gated database write is not durable" + ); + check( + provenance?.revision === initial[0].rev, + "The fixture did not stop before provenance was updated" + ); + observations.push({ + phase: "durable-before-provenance", + state: durableBeforeProvenance, + provenance, + }); + if (remoteRestartEdit) { + const receivedOriginal = await waitFor( + pair.b, + path, + (state) => state.rev === durableBeforeProvenance!.rev && state.content === localContent + ); + await modify(pair.b, path, remoteContent); + remoteAdvanced = await waitFor( + pair.b, + path, + (state) => + liveLeaves(state).length === 1 && + liveLeaves(state)[0].body === remoteContent && + state.rev === liveLeaves(state)[0].rev && + state.content === remoteContent + ); + check( + liveLeaves(remoteAdvanced)[0].ancestry.includes(durableBeforeProvenance.rev!), + "The remote edit did not descend from the already durable local save" + ); + const localDbAdvanced = await waitFor( + pair.a, + path, + (state) => + state.rev === remoteAdvanced!.rev && + liveLeaves(state).length === 1 && + liveLeaves(state)[0].body === remoteContent + ); + const stillOldProvenance = await evaluate<{ revision: string }>( + pair.a, + `return await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(${JSON.stringify(path)});` + ); + check( + localDbAdvanced.content === localContent && stillOldProvenance.revision === initial[0].rev, + "The incoming DB change did not precede Vault reflection and provenance persistence" + ); + observations.push({ + phase: "remote-edit-local-db-ahead-of-storage", + receivedOriginal, + remoteAdvanced, + localDbAdvanced, + stillOldProvenance, + }); + } + } await evaluate(pair.a, "await manager._takeSnapshot();"); check(pending.rev === initial[0].rev, "Restart test did not retain a pending edit"); if (restartBucket) { @@ -746,16 +909,18 @@ async function main(): Promise { sameBucket, }); } - if (!saveAfterEdit && !restartBucket) { + if (stopBeforeProvenance || (!saveAfterEdit && !restartBucket)) { const strictState = await snapshot(pair.a, path); const saved = await evaluate>( pair.a, `return (await manager.adapter.persistence.loadSnapshot()).map(item=>({path:item.args?.file?.path}));` ); - check( - strictState.rev === initial[0].rev && strictState.waiting.some((item) => item.path === path), - "Restart fixture flushed before shutdown" - ); + if (!stopBeforeProvenance) + check( + strictState.rev === initial[0].rev && + strictState.waiting.some((item) => item.path === path), + "Restart fixture flushed before shutdown" + ); check( saved.some((item) => item.path === path), "Restart fixture did not persist the pending event" @@ -778,12 +943,12 @@ async function main(): Promise { const settings=core.services.setting.currentSettings(); return {initialized:await core.kvDB.get('initialized'),restorationRequested:manager.snapShotRestored!==null, settings:{isConfigured:settings.isConfigured,liveSync:settings.liveSync,batchSave:settings.batchSave,batchSaveInLiveSync:settings.batchSaveInLiveSync, - suspendFileWatching:settings.suspendFileWatching}, + suspendFileWatching:settings.suspendFileWatching,disableMarkdownAutoMerge:settings.disableMarkdownAutoMerge}, events:events?.map(item=>({type:item.type,path:item.args?.file?.path, stat:item.args?.file?.stat,skipBatchWait:item.skipBatchWait}))};` ), }); - return { initial, pending }; + return { initial, pending, durableBeforeProvenance, remoteAdvanced }; }); await stop(slots[0]); const persistedSettings = JSON.parse( @@ -817,11 +982,82 @@ async function main(): Promise { events:events?.map(item=>({type:item.type,path:item.args?.file?.path,stat:item.args?.file?.stat}))};` ), }); - const result = await settled(pair, path, "Retained after restart\n"); + if (remoteRestartEdit === "conflicting") { + const preservesBothBodies = (state: Snapshot) => { + const leaves = liveLeaves(state); + return ( + leaves.length === 2 && + leaves.some((leaf) => leaf.body === localContent) && + leaves.some( + (leaf) => leaf.body === remoteContent && leaf.rev === beforeRestart.remoteAdvanced!.rev + ) + ); + }; + const result = await Promise.all([ + waitFor(pair.a, path, preservesBothBodies), + waitFor(pair.b, path, preservesBothBodies), + ]); + check( + result[0].content === localContent && result[1].content === remoteContent, + "An unresolved conflict overwrote one of the preserved Vault contents" + ); + const manualDialogue = pair.a.locator(".modal").filter({ hasText: "Conflicting changes" }); + await manualDialogue.waitFor({ state: "visible", timeout: 10000 }); + await pair.a.screenshot({ path: join(diagnostics, `${restartName}-manual-conflict.png`) }); + return { result, manualConflict: true, restoredGuard: result[0].shouldBatchSave }; + } + const expectedContent = remoteRestartEdit ? remoteContent : localContent; + const result = await settled(pair, path, expectedContent, saveAfterEdit || stopBeforeProvenance); check(result[0].shouldBatchSave, "The saved LiveSync batching choice was not restored"); + if (beforeRestart.remoteAdvanced) { + for (const state of result) + check( + liveLeaves(state)[0].ancestry.includes(beforeRestart.remoteAdvanced.rev!), + "Sensible merge lost the later remote revision's ancestry" + ); + check( + (await pair.a.locator(".modal").filter({ hasText: "Conflicting changes" }).count()) === 0, + "The mergeable recovery left a manual conflict dialogue" + ); + } + for (const [index, page] of [pair.a, pair.b].entries()) { + const provenance = await evaluate<{ revision: string }>( + page, + `const store=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1'); + const deadline=Date.now()+5000; + let record; + do { + record=await store.get(${JSON.stringify(path)}); + if(record?.revision===${JSON.stringify(result[index].rev)})return record; + await new Promise(resolve=>setTimeout(resolve,10)); + }while(Date.now() { + const result = await settled(pair, path, remoteRestartEdit ? remoteContent : localContent, true); + check( + result[0].rev === afterRestart.result[0].rev, + "A repeated restart created another revision after recovery had converged" + ); + check( + result[0].leaves.length === afterRestart.result[0].leaves.length, + "A repeated restart added another branch after recovery had converged" + ); + return result; + }); + } + return { beforeRestart, afterRestart, repeatedRestart }; }); await run("enabled-after-write-restart", async () => { diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 0a08e03d..8e394ccd 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -23,6 +23,7 @@ const focusedScenarios = new Set([ "p2p-pane", "replication-status", "vault-reflection", + "third-party-chunks", "couchdb-upload", "couchdb-http-cache", "chunk-fetch-retry", @@ -36,6 +37,7 @@ const focusedScenarios = new Set([ "p2p-setup-uri-workflow", "partial-startup-file-failure", "startup-scan", + "startup-integrity", "stale-file-restart", "folder-batch", "setup-uri-workflow", @@ -52,6 +54,7 @@ const focusedScenarios = new Set([ "setting-markdown-export", "setting-markdown-roundtrip", "upgrade-from-stable", + "upgrade-livesync-batching", ]); function usage(): string { diff --git a/test/e2e-obsidian/scripts/startup-integrity.ts b/test/e2e-obsidian/scripts/startup-integrity.ts new file mode 100644 index 00000000..f30bdd10 --- /dev/null +++ b/test/e2e-obsidian/scripts/startup-integrity.ts @@ -0,0 +1,396 @@ +/** Checks incomplete-document integrity against real Vault storage in Obsidian. */ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +const filePath = "startup-integrity.md"; +const fileContent = "startup integrity remains healthy\n"; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const vault = await createTemporaryVault("obsidian-startup-integrity-"); + let session: ObsidianLiveSyncSession | undefined; + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const settings = createE2eCouchDbPluginData( + // No replication is enabled; this scenario uses only the local database. + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "startup-integrity-local" }, + { + deviceAndVaultName: "startup-integrity-device", + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + } + ); + + async function evaluate(body: string): Promise { + if (!session) throw new Error("The Obsidian session is not running"); + return await evalObsidianJson( + cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync']?.core; + const sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms)); + const require=(condition,message)=>{if(!condition)throw new Error(message);}; + const waitFor=async(predicate,message,timeoutMs=15000)=>{ + const deadline=Date.now()+timeoutMs; + while(Date.now()(` + require(await core.services.fileProcessing.commitPendingFileEvents(), 'Initial local flush failed'); + const path=${JSON.stringify(filePath)}; + const expectedContent=${JSON.stringify(fileContent)}; + const storage=core.serviceModules.storageAccess; + const fileHandler=core.serviceModules.fileHandler; + const keyValueDB=core.services.keyValueDB.kvDB; + const confirm=core.services.UI.confirm; + const noticeGroups=core.services.context.noticeGroups; + const events=core.services.context.events; + require(storage&&typeof storage.statHidden==='function'&& + typeof storage.readHiddenFileBinary==='function', + 'The real StorageAccess methods were not available'); + require(fileHandler&&typeof fileHandler.storeFileToDB==='function', + 'The real file handler was not available'); + + const snapshot=async()=>{ + const meta=await core.localDatabase.getDBEntryMeta(path,undefined,true); + require(meta&&meta._rev,'The integrity fixture Metadata was missing'); + const doc=await core.localDatabase.getDBEntryFromMeta(meta); + require(doc&&doc.data!==undefined,'The integrity fixture Chunks could not be loaded'); + const content=Array.isArray(doc.data)?doc.data.join(''):doc.data; + require(typeof content==='string','The Markdown fixture did not load as text'); + const bytes=Array.from(new TextEncoder().encode(content)); + require(meta.size===bytes.length,'Metadata size did not match the stored bytes'); + require(doc.size===bytes.length,'Loaded document size did not match the stored bytes'); + return {revision:meta._rev,metadataSize:meta.size,documentSize:doc.size,content,bytes}; + }; + const assertUnchanged=(before,after,label)=>{ + require(before.revision===after.revision,label+': the Metadata revision changed'); + require(before.metadataSize===after.metadataSize,label+': the Metadata size changed'); + require(before.content===after.content,label+': the stored content changed'); + require(JSON.stringify(before.bytes)===JSON.stringify(after.bytes), + label+': the stored bytes changed'); + }; + const activeBytes=async()=>{ + const content=await app.vault.adapter.read(path); + return {content,bytes:Array.from(new TextEncoder().encode(content))}; + }; + const marker=async()=>await keyValueDB.get('checkIncompleteDocs')===true; + const activeState={current:null}; + const originalStat=storage.statHidden; + const originalRead=storage.readHiddenFileBinary; + const originalStore=fileHandler.storeFileToDB; + const originalDialogue=confirm.askSelectStringDialogue; + const ownStat=Object.prototype.hasOwnProperty.call(storage,'statHidden'); + const ownRead=Object.prototype.hasOwnProperty.call(storage,'readHiddenFileBinary'); + const ownStore=Object.prototype.hasOwnProperty.call(fileHandler,'storeFileToDB'); + const ownDialogue=Object.prototype.hasOwnProperty.call(confirm,'askSelectStringDialogue'); + const unhandledErrors=[]; + const onUnhandled=event=>{ + const reason=event.reason; + unhandledErrors.push(String(reason&&reason.message?reason.message:reason)); + event.preventDefault(); + }; + let installedStat=false; + let installedRead=false; + let installedStore=false; + let installedDialogue=false; + const restore=(owner,key,hadOwn,original)=>{ + if(hadOwn)owner[key]=original; + else delete owner[key]; + }; + + const onStat=async function(candidatePath){ + const actual=await originalStat.call(this,candidatePath); + const current=activeState.current; + if(!current||candidatePath!==path)return actual; + current.statCalls++; + require(actual&&actual.type==='file','StorageAccess did not report the fixture as a file'); + current.actualStats.push({size:actual.size,mtime:actual.mtime,ctime:actual.ctime}); + if(current.mode==='stat-read-race'&¤t.statCalls===1){ + const oldStat={...actual,size:actual.size+1}; + current.returnedStats.push({size:oldStat.size,mtime:oldStat.mtime,ctime:oldStat.ctime}); + return oldStat; + } + if(current.mode==='unstable-final-stat'&¤t.statCalls===2){ + const changedStat={...actual,mtime:actual.mtime+1}; + current.returnedStats.push({size:changedStat.size,mtime:changedStat.mtime,ctime:changedStat.ctime}); + return changedStat; + } + current.returnedStats.push({size:actual.size,mtime:actual.mtime,ctime:actual.ctime}); + return actual; + }; + const onRead=async function(candidatePath){ + const bytes=await originalRead.call(this,candidatePath); + const current=activeState.current; + if(current&&candidatePath===path){ + current.readCalls++; + current.readBytes=Array.from(new Uint8Array(bytes)); + } + return bytes; + }; + const onStore=async function(file,...args){ + const current=activeState.current; + if(current&&file?.path===path)current.repairCalls++; + return await originalStore.call(this,file,...args); + }; + const onDialogue=async function(message,buttons,...args){ + const current=activeState.current; + if(current){ + current.dialogueCalls++; + current.dialogueMessages.push(String(message)); + return Array.isArray(buttons)&&buttons.length>0?buttons[0]:false; + } + return await originalDialogue.call(this,message,buttons,...args); + }; + + const runRegisteredCheck=async current=>{ + const originalFinish=noticeGroups.finish; + const ownFinish=Object.prototype.hasOwnProperty.call(noticeGroups,'finish'); + let finished=false; + noticeGroups.finish=function(...args){ + try{return originalFinish.apply(this,args);} + finally{ + if(args[0]==='startup-integrity-check')finished=true; + } + }; + activeState.current=current; + try{ + events.emitEvent('request-run-fix-incomplete'); + await waitFor(()=>finished,'The registered incomplete-document check did not finish'); + }finally{ + activeState.current=null; + restore(noticeGroups,'finish',ownFinish,originalFinish); + } + }; + const makeObservation=mode=>({ + mode, + statCalls:0, + actualStats:[], + returnedStats:[], + readCalls:0, + readBytes:[], + dialogueCalls:0, + dialogueMessages:[], + repairCalls:0, + }); + + window.addEventListener('unhandledrejection',onUnhandled); + try{ + storage.statHidden=onStat; + installedStat=true; + storage.readHiddenFileBinary=onRead; + installedRead=true; + fileHandler.storeFileToDB=onStore; + installedStore=true; + confirm.askSelectStringDialogue=onDialogue; + installedDialogue=true; + + const firstStorageStat=await storage.statHidden(path); + require(firstStorageStat&&firstStorageStat.type==='file', + 'The real Vault file was not visible through StorageAccess'); + const firstVault=await activeBytes(); + const first=snapshot(); + const healthyBefore=await first; + require(firstVault.content===expectedContent&&healthyBefore.content===expectedContent, + 'The fixture content did not match between the Vault and Metadata'); + require(firstStorageStat.size===healthyBefore.bytes.length, + 'The real Vault size did not match the Metadata bytes'); + + await keyValueDB.set('checkIncompleteDocs',false); + const healthy=makeObservation('healthy'); + const healthyUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(healthy); + const healthyAfter=await snapshot(); + const healthyMarker=await marker(); + const healthyVaultAfter=await activeBytes(); + require(healthy.statCalls===2,'The healthy fixture was not checked with both StorageAccess stats'); + require(healthy.readCalls===0,'The healthy fixture was read only to determine its size'); + require(healthy.dialogueCalls===0,'The healthy fixture opened a repair dialogue'); + require(healthy.repairCalls===0,'The healthy fixture invoked repair'); + require(healthyMarker,'The healthy integrity check did not set its completion marker'); + require(unhandledErrors.length===healthyUnhandledBefore, + 'The healthy integrity check raised an unhandled error'); + require(healthyVaultAfter.content===expectedContent,'The healthy Vault file changed'); + assertUnchanged(healthyBefore,healthyAfter,'Healthy scan'); + const observations=[ + { + case:'healthy', + revision:healthyAfter.revision, + storedBytes:healthyAfter.bytes.length, + readCalls:healthy.readCalls, + statCalls:healthy.statCalls, + dialogueCalls:healthy.dialogueCalls, + repairCalls:healthy.repairCalls, + completionMarker:healthyMarker, + storedBytesUnchanged:true, + }, + ]; + + const raceBefore=await snapshot(); + await keyValueDB.set('checkIncompleteDocs',false); + const race=makeObservation('stat-read-race'); + const raceUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(race); + const raceAfter=await snapshot(); + const raceMarker=await marker(); + const raceVaultAfter=await activeBytes(); + require(race.statCalls===1,'The controlled stat/read race did not use one initial stat'); + require(race.actualStats[0]?.size===raceBefore.bytes.length, + 'The real StorageAccess stat did not see the Metadata size'); + require(race.returnedStats[0]?.size===raceBefore.bytes.length+1, + 'The checker did not see the controlled old mismatched size'); + require(race.readCalls===1,'The controlled mismatch did not read real Vault bytes'); + require(JSON.stringify(race.readBytes)===JSON.stringify(raceBefore.bytes), + 'The actual StorageAccess read did not match the database bytes'); + require(race.dialogueCalls===0,'The settled stat/read race opened a repair dialogue'); + require(race.repairCalls===0,'The settled stat/read race invoked repair'); + require(raceMarker,'The settled stat/read race did not set its completion marker'); + require(unhandledErrors.length===raceUnhandledBefore, + 'The settled stat/read race raised an unhandled error'); + require(raceVaultAfter.content===expectedContent,'The Vault file changed during the stat/read race'); + assertUnchanged(raceBefore,raceAfter,'Settled stat/read race'); + observations.push({ + case:'stat-read-race', + revision:raceAfter.revision, + storedBytes:raceAfter.bytes.length, + observedStatSize:race.returnedStats[0]?.size, + readCalls:race.readCalls, + readMatchedDatabase:true, + dialogueCalls:race.dialogueCalls, + repairCalls:race.repairCalls, + completionMarker:raceMarker, + storedBytesUnchanged:true, + }); + + const unstableBefore=await snapshot(); + await keyValueDB.set('checkIncompleteDocs',false); + const unstable=makeObservation('unstable-final-stat'); + const unstableUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(unstable); + await waitFor(()=>unhandledErrors.length>unstableUnhandledBefore, + 'The changed final stat did not reject the incomplete-document check'); + const unstableError=unhandledErrors.slice(unstableUnhandledBefore).join(' | '); + const unstableAfter=await snapshot(); + const unstableMarker=await marker(); + require(unstable.statCalls===2,'The changed final stat was not checked'); + require(unstable.returnedStats[1]?.mtime!==unstable.actualStats[1]?.mtime, + 'The final StorageAccess stat was not changed by the race fixture'); + require(unstable.readCalls===0,'The healthy path was read during the final-stat race'); + require(unstableError.includes('Storage changed during the incomplete document check'), + 'The final-stat race did not report the expected incomplete-check error'); + require(unstableMarker===false,'The incomplete check was marked complete after storage changed'); + require(unstable.dialogueCalls===0,'The final-stat race opened a repair dialogue'); + require(unstable.repairCalls===0,'The final-stat race invoked repair'); + assertUnchanged(unstableBefore,unstableAfter,'Final-stat race'); + observations.push({ + case:'unstable-final-stat', + revision:unstableAfter.revision, + statCalls:unstable.statCalls, + readCalls:unstable.readCalls, + reportedError:unstableError, + completionMarker:unstableMarker, + storedBytesUnchanged:true, + }); + + await keyValueDB.set('checkIncompleteDocs',false); + const retry=makeObservation('settled-retry'); + const retryUnhandledBefore=unhandledErrors.length; + await runRegisteredCheck(retry); + const retryAfter=await snapshot(); + const retryMarker=await marker(); + require(retry.statCalls===2,'The settled retry did not complete both StorageAccess stats'); + require(retry.readCalls===0,'The settled retry read healthy storage'); + require(retry.dialogueCalls===0,'The settled retry opened a repair dialogue'); + require(retry.repairCalls===0,'The settled retry invoked repair'); + require(retryMarker,'The settled retry did not set its completion marker'); + require(unhandledErrors.length===retryUnhandledBefore, + 'The settled retry raised an unhandled error'); + assertUnchanged(unstableBefore,retryAfter,'Settled retry'); + observations.push({ + case:'settled-retry', + revision:retryAfter.revision, + statCalls:retry.statCalls, + readCalls:retry.readCalls, + dialogueCalls:retry.dialogueCalls, + repairCalls:retry.repairCalls, + completionMarker:retryMarker, + storedBytesUnchanged:true, + }); + return JSON.stringify(observations); + }finally{ + window.removeEventListener('unhandledrejection',onUnhandled); + if(installedDialogue)restore(confirm,'askSelectStringDialogue',ownDialogue,originalDialogue); + if(installedStore)restore(fileHandler,'storeFileToDB',ownStore,originalStore); + if(installedRead)restore(storage,'readHiddenFileBinary',ownRead,originalRead); + if(installedStat)restore(storage,'statHidden',ownStat,originalStat); + } + `); + + console.log( + JSON.stringify( + { + ok: true, + bundleHash, + trigger: "registered-request-run-fix-incomplete-event", + naturalRestart: false, + observations, + }, + null, + 2 + ) + ); + } finally { + try { + await session?.app.stop(); + } finally { + await vault.dispose(); + } + } +} + +await main(); diff --git a/test/e2e-obsidian/scripts/third-party-chunks.ts b/test/e2e-obsidian/scripts/third-party-chunks.ts new file mode 100644 index 00000000..e781d74e --- /dev/null +++ b/test/e2e-obsidian/scripts/third-party-chunks.ts @@ -0,0 +1,269 @@ +/** Exercises synthetic split-surrogate input through real Obsidian file reflection. */ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + assertEqual, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const textPath = "synthetic-split-pair.md"; + const controlPath = "complete-pair.md"; + const binaryPath = "attachment.bin"; + const textContent = "prefix 🙂 suffix\n" + "normal chunk payload\n".repeat(50_000); + const controlContent = "prefix 🙂 suffix"; + const binaryBytes = Buffer.from([0, 255, 31, 128]); + const textSize = Buffer.byteLength(textContent, "utf8"); + const textHash = `sha256:${createHash("sha256").update(textContent, "utf8").digest("hex")}`; + const controlSize = Buffer.byteLength(controlContent, "utf8"); + const controlHash = `sha256:${createHash("sha256").update(controlContent, "utf8").digest("hex")}`; + const settings = createE2eCouchDbPluginData( + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "third-party-chunks-local" }, + { + customChunkSize: 0, + processSizeMismatchedFiles: false, + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + } + ); + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const vault = await createTemporaryVault("obsidian-third-party-chunks-"); + let session: ObsidianLiveSyncSession | undefined; + + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + pluginData: settings, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + lifecycle: { + beforeLaunch: async () => { + const registryPath = join(vault.userDataPath, "obsidian.json"); + const registry = JSON.parse(await readFile(registryPath, "utf8")); + registry.updateDisabled = true; + await writeFile(registryPath, JSON.stringify(registry)); + }, + }, + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + + const reflection = await evalObsidianJson<{ + sizeCheckingEnabled: boolean; + split: { + reflected: boolean; + metadataRevisionUnchanged: boolean; + chunkCount: number; + provenanceRevision: string; + contentSize: number; + contentHash: string; + }; + control: { + reflected: boolean; + content: string; + metadataRevisionUnchanged: boolean; + provenanceRevision: string; + contentSize: number; + contentHash: string; + }; + binary: { reflected: boolean; bytes: number[]; metadataRevisionUnchanged: boolean }; + }>( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const require=(condition,message)=>{if(!condition)throw new Error(message);};", + "const textPath='synthetic-split-pair.md';", + "const controlPath='complete-pair.md';", + "const binaryPath='attachment.bin';", + "const textContent='prefix 🙂 suffix\\n'+'normal chunk payload\\n'.repeat(50000);", + "const controlContent='prefix 🙂 suffix';", + "const binaryBytes=[0,255,31,128];", + "const store=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1');", + "require(core.settings.processSizeMismatchedFiles===false,'Size mismatch checking must remain enabled');", + "const seed=async(path,blob,type,datatype)=>{", + " require(!app.vault.getAbstractFileByPath(path),'Fixture already exists in the Vault: '+path);", + " const id=await core.services.path.path2id(path);", + " const now=Date.now();", + " const saved=await core.localDatabase.putDBEntry({_id:id,path,data:blob,ctime:now,mtime:now,", + " size:blob.size,children:[],datatype,type,eden:{} });", + " require(saved?.ok,'Could not create Metadata and Chunks: '+path);", + " const meta=await core.localDatabase.getDBEntryMeta(path,undefined,true);", + " require(meta&&meta._rev===saved.rev,'Could not reload the created Metadata: '+path);", + " return meta;", + "};", + "const textMeta=await seed(textPath,new Blob([textContent],{type:'text/plain'}),'plain','plain');", + "require((textMeta.children??[]).length>0,'The primary fixture did not create normal Chunks');", + "const savedText=await core.localDatabase.getDBEntry(textPath,{rev:textMeta._rev,revs:true},false,true,true);", + "const storedText=Array.isArray(savedText?.data)?savedText.data.join(''):savedText?.data;", + "require(storedText===textContent,'The seeded local database content did not match the joined text');", + "const controlMeta=await seed(controlPath,new Blob([controlContent],{type:'text/plain'}),'plain','plain');", + "const binaryMeta=await seed(binaryPath,new Blob([new Uint8Array(binaryBytes)],{type:'application/octet-stream'}),'newnote','newnote');", + "const initialTextRevision=textMeta._rev;", + "const initialControlRevision=controlMeta._rev;", + "const database=core.serviceModules.databaseFileAccess;", + "require(typeof database?.fetchEntryFromMeta==='function','The database file-access provider is missing');", + "const descriptor=Object.getOwnPropertyDescriptor(database,'fetchEntryFromMeta');", + "const original=database.fetchEntryFromMeta;", + "let supplied=0;", + "const pairIndex=textContent.indexOf('🙂');", + "const syntheticChunks=[textContent.slice(0,pairIndex+1),textContent.slice(pairIndex+1)];", + "require(syntheticChunks[0]==='prefix \\uD83D','The synthetic first chunk did not end with the high surrogate');", + "require(syntheticChunks[1].startsWith('\\uDE42 suffix'),'The synthetic second chunk did not start with the low surrogate');", + "database.fetchEntryFromMeta=async function(meta,...args){", + " const loaded=await original.call(database,meta,...args);", + " if(meta?._id!==textMeta._id||meta?.path!==textPath||!loaded)return loaded;", + " supplied++;", + " return {...loaded,data:syntheticChunks};", + "};", + "let reflectedText=false;", + "try{reflectedText=await core.serviceModules.fileHandler.dbToStorage(textMeta,null,true);}", + "finally{if(descriptor)Object.defineProperty(database,'fetchEntryFromMeta',descriptor);else delete database.fetchEntryFromMeta;}", + "require(supplied===1,'The synthetic provider was not used exactly once');", + "require(reflectedText,'Real Obsidian reflection rejected the joined text');", + "const textFile=app.vault.getAbstractFileByPath(textPath);", + "require(textFile,'The reflected text file is missing from the Vault');", + "const vaultText=await app.vault.read(textFile);", + "require(vaultText===textContent,'The Vault text does not match the joined input');", + "const textMetaAfter=await core.localDatabase.getDBEntryMeta(textPath,undefined,true);", + "require(textMetaAfter?._rev===initialTextRevision,'Text reflection changed the Metadata revision');", + "const textProvenance=await store.get(textPath);", + "require(textProvenance?.revision===initialTextRevision,'Text provenance has the wrong revision');", + `require(textProvenance?.contentSize===${textSize},'Text provenance has the wrong UTF-8 byte size');`, + `require(textProvenance?.contentHash===${JSON.stringify(textHash)},'Text provenance has the wrong content hash');`, + "const reflectedControl=await core.serviceModules.fileHandler.dbToStorage(controlMeta,null,true);", + "require(reflectedControl,'Real Obsidian reflection rejected the complete-pair control');", + "const controlFile=app.vault.getAbstractFileByPath(controlPath);", + "require(controlFile&&await app.vault.read(controlFile)===controlContent,'The complete-pair control content changed');", + "const controlVaultText=await app.vault.read(controlFile);", + "const controlMetaAfter=await core.localDatabase.getDBEntryMeta(controlPath,undefined,true);", + "require(controlMetaAfter?._rev===initialControlRevision,'Control reflection changed the Metadata revision');", + "const controlProvenance=await store.get(controlPath);", + "require(controlProvenance?.revision===initialControlRevision,'Control provenance has the wrong revision');", + `require(controlProvenance?.contentSize===${controlSize},'Control provenance has the wrong UTF-8 byte size');`, + `require(controlProvenance?.contentHash===${JSON.stringify(controlHash)},'Control provenance has the wrong content hash');`, + "const reflectedBinary=await core.serviceModules.fileHandler.dbToStorage(binaryMeta,null,true);", + "require(reflectedBinary,'Real Obsidian reflection rejected the binary attachment');", + "const binaryFile=app.vault.getAbstractFileByPath(binaryPath);", + "require(binaryFile,'The reflected binary attachment is missing from the Vault');", + "const binaryVaultBytes=Array.from(new Uint8Array(await app.vault.readBinary(binaryFile)));", + "require(JSON.stringify(binaryVaultBytes)===JSON.stringify(binaryBytes),'The Vault binary bytes changed');", + "const binaryMetaAfter=await core.localDatabase.getDBEntryMeta(binaryPath,undefined,true);", + "require(binaryMetaAfter?._rev===binaryMeta._rev,'Binary reflection changed the Metadata revision');", + "return JSON.stringify({sizeCheckingEnabled:!core.settings.processSizeMismatchedFiles,", + " split:{reflected:reflectedText,metadataRevisionUnchanged:textMetaAfter._rev===initialTextRevision,chunkCount:textMeta.children.length,", + " provenanceRevision:textProvenance.revision,contentSize:textProvenance.contentSize,contentHash:textProvenance.contentHash},", + " control:{reflected:reflectedControl,content:controlVaultText,", + " metadataRevisionUnchanged:controlMetaAfter._rev===initialControlRevision,provenanceRevision:controlProvenance.revision,", + " contentSize:controlProvenance.contentSize,contentHash:controlProvenance.contentHash},", + " binary:{reflected:reflectedBinary,bytes:binaryVaultBytes,metadataRevisionUnchanged:binaryMetaAfter._rev===binaryMeta._rev}});", + "})()", + ].join(""), + session.cliEnv + ); + + assertEqual(reflection.sizeCheckingEnabled, true, "Size mismatch checking was disabled."); + assertEqual(reflection.split.reflected, true, "Split-pair text was not reflected."); + assertEqual( + reflection.split.metadataRevisionUnchanged, + true, + "Split-pair reflection changed its Metadata revision." + ); + assertEqual(reflection.split.provenanceRevision.length > 0, true, "Split-pair provenance is missing."); + assertEqual(reflection.split.contentSize, textSize, "Split-pair provenance has the wrong UTF-8 byte size."); + assertEqual(reflection.split.contentHash, textHash, "Split-pair provenance has the wrong content hash."); + assertEqual(reflection.control.reflected, true, "Complete-pair control was not reflected."); + assertEqual(reflection.control.content, controlContent, "Complete-pair control content changed."); + assertEqual( + reflection.control.metadataRevisionUnchanged, + true, + "Control reflection changed its Metadata revision." + ); + assertEqual(reflection.control.contentSize, controlSize, "Control provenance has the wrong UTF-8 byte size."); + assertEqual(reflection.control.contentHash, controlHash, "Control provenance has the wrong content hash."); + assertEqual(reflection.binary.reflected, true, "Binary attachment was not reflected."); + if (!Buffer.from(reflection.binary.bytes).equals(binaryBytes)) { + throw new Error("The Obsidian Vault API returned different binary attachment bytes."); + } + assertEqual( + reflection.binary.metadataRevisionUnchanged, + true, + "Binary reflection changed its Metadata revision." + ); + + const diskText = await readFile(join(vault.path, textPath)); + assertEqual(diskText.toString("utf8"), textContent, "On-disk UTF-8 text differs from the joined input."); + assertEqual(diskText.byteLength, textSize, "On-disk UTF-8 text has the wrong byte length."); + const diskControl = await readFile(join(vault.path, controlPath)); + assertEqual( + diskControl.toString("utf8"), + controlContent, + "On-disk complete-pair control differs from the input." + ); + const diskBinary = await readFile(join(vault.path, binaryPath)); + if (!diskBinary.equals(binaryBytes)) throw new Error("On-disk binary attachment bytes changed."); + + console.log( + JSON.stringify( + { + ok: true, + bundleHash, + syntheticInput: + "A local fetchEntryFromMeta result split the joined text between the emoji surrogates; this does not establish that CouchDB or any named third-party producer emits lone-surrogate chunks.", + sizeCheckingEnabled: reflection.sizeCheckingEnabled, + splitText: { + reflected: reflection.split.reflected, + metadataRevisionUnchanged: reflection.split.metadataRevisionUnchanged, + realChunkCount: reflection.split.chunkCount, + vaultTextMatches: true, + onDiskUtf8Bytes: diskText.byteLength, + provenanceRevision: reflection.split.provenanceRevision, + contentSize: reflection.split.contentSize, + contentHash: reflection.split.contentHash, + }, + completePairControl: { + reflected: reflection.control.reflected, + metadataRevisionUnchanged: reflection.control.metadataRevisionUnchanged, + vaultAndDiskContentMatch: true, + provenanceContentSize: reflection.control.contentSize, + provenanceContentHash: reflection.control.contentHash, + }, + binaryAttachment: { + reflected: reflection.binary.reflected, + exactVaultBytes: true, + exactDiskBytes: true, + metadataRevisionUnchanged: reflection.binary.metadataRevisionUnchanged, + }, + }, + null, + 2 + ) + ); + } finally { + try { + await session?.app.stop(); + } finally { + await vault.dispose(); + } + } +} + +await main(); diff --git a/test/e2e-obsidian/scripts/upgrade-livesync-batching.ts b/test/e2e-obsidian/scripts/upgrade-livesync-batching.ts new file mode 100644 index 00000000..47e02759 --- /dev/null +++ b/test/e2e-obsidian/scripts/upgrade-livesync-batching.ts @@ -0,0 +1,312 @@ +/** Keeps the 1.0.40 batching choice and Vault state when installing the current release. */ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + loadCouchDbConfig, + makeUniqueDatabaseName, + waitForCouchDbDocs, +} from "../runner/couchdb.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + assertEqual, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + pushLocalChanges, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { ensurePinnedReleaseArtifact, type PinnedPluginRelease } from "../runner/releaseArtifact.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { readRuntimeUpgradeState, waitForPersistentNodeIdentity } from "../runner/upgradeWorkflow.ts"; +import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; + +const sourceRelease: PinnedPluginRelease = { + pluginId: "obsidian-livesync", + version: "1.0.40", + files: [ + { + name: "main.js", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/1.0.40/main.js", + sha256: "bdd6b18ffe36a6a72ce774e857889b70d379559e4a5a3ccafee357a42daca6e9", + }, + { + name: "manifest.json", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/1.0.40/manifest.json", + sha256: "8f8f57fb105aba0ce7f9ea5104dbed92fea837d49fdf27b4f02b6099ddf1ba3b", + }, + { + name: "styles.css", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/1.0.40/styles.css", + sha256: "4ba00bef417ee98d9a62dd9febb792dc6555949cfc9e8e069450ad699ab2b9f0", + }, + ], +}; + +type BatchState = { + batchSave: boolean; + batchSaveInLiveSync: boolean | null; + liveSync: boolean; + periodicReplication: boolean; + shouldBatchSave: boolean; + minimum: number; + maximum: number; +}; + +const beforePath = "upgrade-batching-before.md"; +const afterPath = "upgrade-batching-after.md"; +const returnPath = "upgrade-batching-return.md"; +const beforeContent = "Saved by the published 1.0.40 plug-in.\n"; +const afterContent = "Saved after upgrading with LiveSync batching still disabled.\n"; +const returnContent = "Returned by an independent current-release Vault.\n"; + +async function artefactHashes(root: string): Promise> { + return Object.fromEntries( + await Promise.all( + sourceRelease.files.map(async ({ name }) => [ + name, + createHash("sha256") + .update(new Uint8Array(await readFile(join(root, name)))) + .digest("hex"), + ]) + ) + ); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const sourceRoot = await ensurePinnedReleaseArtifact(sourceRelease); + const targetRoot = resolve(process.env.E2E_LIVESYNC_TARGET_ARTIFACT_ROOT?.trim() || process.cwd()); + const targetManifest = JSON.parse(await readFile(join(targetRoot, "manifest.json"), "utf8")); + if (targetManifest.id !== sourceRelease.pluginId || targetManifest.version === sourceRelease.version) { + throw new Error("The target must be a newer Self-hosted LiveSync build."); + } + const hashes = await artefactHashes(targetRoot); + const config = await loadCouchDbConfig(); + if (!["127.0.0.1", "localhost"].includes(new URL(config.uri).hostname)) { + throw new Error("Use a disposable local CouchDB for the upgrade scenario."); + } + const dbName = makeUniqueDatabaseName(config.dbPrefix, "upgrade-batching"); + const vault = await createTemporaryVault("obsidian-upgrade-batching-"); + const verifier = await createTemporaryVault("obsidian-upgrade-verifier-"); + let session: ObsidianLiveSyncSession | undefined; + let ownsDatabase = false; + const pluginData = createE2eCouchDbPluginData( + { ...config, dbName }, + { + batchSave: true, + batchSaveMinimumDelay: 3, + batchSaveMaximumDelay: 7, + liveSync: false, + periodicReplication: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + doctorProcessedVersion: "1.0.0", + } + ); + + async function stop(): Promise { + if (session) await session.app.stop(); + session = undefined; + } + + async function start(currentVault: TemporaryVault, artifactRoot: string, seed = false): Promise { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault: currentVault, + artifactRoot, + pluginStartup: seed ? undefined : "natural", + pluginData: seed ? pluginData : undefined, + localStorageEntries: seed ? createE2eObsidianDeviceLocalState(currentVault.name) : undefined, + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + } + + async function evaluate(body: string): Promise { + if (!session) throw new Error("No active Obsidian session"); + return await evalObsidianJson( + cliBinary, + `(async()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + return JSON.stringify(await(async()=>{${body}})()); + })()`, + session.cliEnv + ); + } + + async function batching(): Promise { + return await evaluate(` + const s=core.services.setting.currentSettings(); + return {batchSave:s.batchSave,batchSaveInLiveSync:s.batchSaveInLiveSync??null, + liveSync:s.liveSync,periodicReplication:s.periodicReplication, + shouldBatchSave:core.serviceModules.storageAccess.vaultManager.shouldBatchSave, + minimum:s.batchSaveMinimumDelay,maximum:s.batchSaveMaximumDelay}; + `); + } + + async function createNote(path: string, content: string): Promise { + await evaluate(`await app.vault.create(${JSON.stringify(path)},${JSON.stringify(content)}); return true;`); + const entry = await waitForLocalDatabaseEntry(cliBinary, session!.cliEnv, path); + if ((await batching()).liveSync) { + await waitForCouchDbDocs(config, dbName, (docs) => + docs.some((doc) => doc._id === entry.id && doc._rev === entry.rev) + ); + } else { + await pushLocalChanges(cliBinary, session!.cliEnv); + } + } + + async function assertNote(currentVault: TemporaryVault, path: string, content: string): Promise { + const actual = await evaluate(` + const until=Date.now()+15000; + while(Date.now()setTimeout(resolve,100)); + } + throw new Error('Expected Vault contents did not arrive: '+${JSON.stringify(path)}); + `); + assertEqual(actual, content, "The Obsidian Vault did not contain the expected text."); + assertEqual(await readFile(join(currentVault.path, path), "utf8"), content, "On-disk text differed."); + } + + function assertUpgradedBatching(state: BatchState, live: boolean): void { + assertEqual(state.batchSave, true, "The legacy batching choice changed during upgrade."); + assertEqual(state.batchSaveInLiveSync, false, "Upgrade unexpectedly enabled LiveSync batching."); + assertEqual(state.liveSync, live, "The selected synchronisation mode changed."); + assertEqual(state.shouldBatchSave, !live, "The retained choice applied to the wrong mode."); + assertEqual(state.minimum, 3, "The minimum batch delay changed."); + assertEqual(state.maximum, 7, "The maximum batch delay changed."); + } + + try { + await assertCouchDbReachable(config); + await createCouchDbDatabase(config, dbName); + ownsDatabase = true; + await start(vault, sourceRoot, true); + await prepareRemote(cliBinary, session!.cliEnv); + await createNote(beforePath, beforeContent); + await evaluate(` + const s=core.services.setting.currentSettings(); + s.liveSync=true; + await core.services.setting.saveSettingData(); + await core.services.control.applySettings(); + return true; + `); + await stop(); + + // Confirm that the old release itself persisted this state before replacing any artefacts. + await start(vault, sourceRoot); + const oldBatch = await batching(); + assertEqual(oldBatch.batchSave, true, "The old release did not retain its batching choice."); + assertEqual(oldBatch.batchSaveInLiveSync, null, "The old release unexpectedly owns the new setting."); + assertEqual(oldBatch.liveSync, true, "The old release did not retain LiveSync mode."); + assertEqual(oldBatch.shouldBatchSave, false, "The old release unexpectedly batches LiveSync mode."); + await waitForPersistentNodeIdentity(cliBinary, session!.cliEnv); + const before = await readRuntimeUpgradeState(cliBinary, session!.cliEnv); + const originalEntry = await waitForLocalDatabaseEntry(cliBinary, session!.cliEnv, beforePath); + await assertNote(vault, beforePath, beforeContent); + await stop(); + const savedOldData = JSON.parse( + await readFile(join(vault.path, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8") + ); + assertEqual( + Object.prototype.hasOwnProperty.call(savedOldData, "batchSaveInLiveSync"), + false, + "The old settings fixture contains the new option." + ); + + await start(vault, targetRoot); + const upgraded = await readRuntimeUpgradeState(cliBinary, session!.cliEnv); + assertEqual(upgraded.pluginVersion, targetManifest.version, "The requested target build was not loaded."); + assertEqual(upgraded.localDatabaseName, before.localDatabaseName, "Upgrade replaced the local database."); + assertEqual(upgraded.nodeId, before.nodeId, "Upgrade replaced the device identity."); + assertEqual( + upgraded.settings.activeConfigurationId, + before.settings.activeConfigurationId, + "Upgrade changed the remote profile." + ); + assertEqual(upgraded.settings.couchDB_DBNAME, dbName, "Upgrade selected another remote database."); + assertEqual(upgraded.settings.versionUpFlash, "", "Upgrade unexpectedly paused for compatibility review."); + const firstBatch = await batching(); + assertUpgradedBatching(firstBatch, true); + assertEqual( + (await waitForLocalDatabaseEntry(cliBinary, session!.cliEnv, beforePath)).rev, + originalEntry.rev, + "Upgrade rewrote existing Metadata." + ); + await assertNote(vault, beforePath, beforeContent); + await createNote(afterPath, afterContent); + await stop(); + + await start(verifier, targetRoot, true); + await prepareRemote(cliBinary, session!.cliEnv); + await pushLocalChanges(cliBinary, session!.cliEnv); + await assertNote(verifier, beforePath, beforeContent); + await assertNote(verifier, afterPath, afterContent); + await createNote(returnPath, returnContent); + await stop(); + + await start(vault, targetRoot); + assertUpgradedBatching(await batching(), true); + await assertNote(vault, returnPath, returnContent); + await evaluate(` + const s=core.services.setting.currentSettings(); + s.liveSync=false;s.periodicReplication=true;s.periodicReplicationInterval=60; + await core.services.setting.saveSettingData(); + await core.services.control.applySettings(); + return true; + `); + assertUpgradedBatching(await batching(), false); + await stop(); + await start(vault, targetRoot); + const finalBatch = await batching(); + assertUpgradedBatching(finalBatch, false); + assertEqual(finalBatch.periodicReplication, true, "The periodic mode choice did not survive restart."); + await assertNote(vault, returnPath, returnContent); + assertEqual( + JSON.stringify(await artefactHashes(targetRoot)), + JSON.stringify(hashes), + "Target artefacts changed during the test." + ); + console.log( + JSON.stringify( + { + passed: true, + sourceVersion: sourceRelease.version, + targetVersion: targetManifest.version, + sourceHashes: Object.fromEntries(sourceRelease.files.map((file) => [file.name, file.sha256])), + targetHashes: hashes, + oldBatch, + firstBatch, + finalBatch, + retainedDatabase: true, + retainedNodeIdentity: true, + retainedMetadataRevision: true, + twoWayTransfer: true, + naturalRestarts: true, + }, + null, + 2 + ) + ); + } finally { + await stop(); + await vault.dispose(); + await verifier.dispose(); + if (ownsDatabase) await deleteCouchDbDatabase(config, dbName); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/updates.md b/updates.md index 63cc06b6..2a288762 100644 --- a/updates.md +++ b/updates.md @@ -12,40 +12,64 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased -### Customisation Sync +## 1.0.41 -#### Fixed - -- We can reload the customisation list and apply every file in a multi-file item on the first attempt. (#1271) - - Reloading or clearing the list also prevents earlier pending updates from restoring removed files and items. - -### Setup tools - -#### Fixed - -- We can generate a Setup URI with prefixed connection and encryption variables, avoiding collisions with shell or system variables such as `hostname` and `USERNAME`. (#817, #1274) - - The generator reads `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE`. The previous names remain supported when the corresponding prefixed value is unset or blank. +26th October, 2026 ### Synchronisation #### New Feature -- We can see CouchDB change-history comparison progressing before new documents arrive, with a ⏊ status icon and a comparison count before the upload count. An unchanged count disappears after three seconds. -- We can hide or style individual status items with CSS snippets using the `livesync-stat-` classes. Tabular, right-aligned numbers retain their width while visible, reducing movement as counts change. Wider gaps between editor status items make the counts easier to read. The `livesync-stat-inactive` class lets CSS snippets control how inactive items disappear. -- We can enable **Batch database updates in LiveSync mode** to combine repeated local edits before sending them to other devices. - - The choice starts disabled, including when older settings contain the previously ignored batch option. Each mode retains its own choice and uses the same minimum and maximum delays. - - Batching delays remote visibility and reduces intermediate updates. It does not establish when a template has finished or prevent genuine concurrent edits. +- Rapid edits can produce fewer intermediate versions on other devices while LiveSync mode remains active. + - Enable **Batch database updates in LiveSync mode** to group successive local edits. + - Disabled by default; each synchronisation mode keeps its own choice. + - Updates reach other devices later. Batching does not detect template completion or prevent genuine concurrent-edit conflicts. +- CouchDB synchronisation no longer looks idle while it is comparing change history. + - A ⏊ icon and comparison count show activity before documents are transferred. + - This count represents history entries, not files remaining to synchronise. +- Status counts are easier to follow, with less movement as values change. + - CSS snippets can hide unwanted items or style individual `livesync-stat-` items, including inactive ones. #### Fixed -- Local changes remain available for recovery while their database save is running, including when the application stops after a batch delay ends. - - Manual synchronisation and settings application now wait for the accepted local saves to finish and report failures. Pending events retain the existing snapshot format. - - Hidden File Sync and Customisation Sync now report completion after saving, including delayed customisation updates. Failed saves remain pending, while unchanged notifications allow later edits to proceed. -- Start-up reconciliation checks content when modification times are equal, so an edit within the same timestamp interval is not mistaken for an unchanged file. - - Existing revision provenance preserves the correct edit branch. Unreadable or continually changing content remains unresolved rather than being reported as successfully reconciled. -- We receive a clear Notice when **Automatic** requires Hidden File Sync to be enabled on this device. Our existing mode remains unchanged, and hidden-file initialisation cannot start while the feature is disabled. (#1270) -- We can initialise a whole plug-in's **MAIN** and **DATA** selections using the chosen Automatic initial action. -- We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283) +- Recent Vault edits remain recoverable after an interrupted local database save. + - This includes edits whose modification times compare equal to the database copy, and pending Hidden File Sync and Customisation Sync changes. +- Manual synchronisation no longer starts before already accepted local changes finish saving to the database. + - Settings application also waits for those saves. Failed saves stay pending and report a failure. +- Text from third-party tools is handled more robustly. + - Characters split across Chunks no longer trigger false corruption warnings. +- Start-up integrity checks use less memory for healthy Vault files. (#1283) +- Start-up checks avoid false repair warnings for files that change during inspection. (#1283) + - This applies when the final size matches the database and there are no conflicts. + +### Customisation Sync + +#### Fixed + +- Applying a multi-file customisation no longer skips files after the list is refreshed. (#1271) +- Choosing **Automatic** now explains a missing Hidden File Sync prerequisite instead of proceeding into a failed initialisation. (#1270) + - Enable and initialise Hidden File Sync on this device first. The current mode is kept when the prerequisite is missing. +- Initialising a whole plug-in in **Automatic** mode no longer omits its selected **MAIN** or **DATA** files. + +### Setup tools + +#### Fixed + +- Setup URI generation can avoid shell-variable collisions and unintended use of the operating-system username. (#817, #1274) + - Use `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE` for connection and encryption values. + - Existing scripts remain supported: the previous names are used when the corresponding prefixed value is unset or blank. + +### Miscellaneous + +- Dependencies have been updated, including security fixes. + +### Acknowledgements + +Thank you for your contributions! + +- @rwbru, for the original Setup URI environment-variable fix ([#817](https://github.com/vrtmrz/obsidian-livesync/pull/817)). +- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts ([#1274](https://github.com/vrtmrz/obsidian-livesync/pull/1274)). +- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib ([#1283](https://github.com/vrtmrz/obsidian-livesync/pull/1283), [Commonlib #184](https://github.com/vrtmrz/livesync-commonlib/pull/184)). ## 1.0.40 diff --git a/versions.json b/versions.json index 19b183c0..a68f2a1a 100644 --- a/versions.json +++ b/versions.json @@ -51,5 +51,6 @@ "1.0.37": "1.7.2", "1.0.38": "1.7.2", "1.0.39": "1.7.2", - "1.0.40": "1.7.2" + "1.0.40": "1.7.2", + "1.0.41": "1.7.2" } From e83aa892b4619ee0662e236125bf8daf5bb2615d Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 26 Oct 2026 15:46:55 +0000 Subject: [PATCH 51/68] Move Vite build-file copying into scripts Keep the Node.js build helper and its tests in the build-script directory recognised by Community review. Update the Vite import and unit CI path filters while preserving the copy behaviour and release artefacts. --- .github/workflows/unit-ci.yml | 2 ++ {utils => scripts}/copy-build-files.ts | 0 {utils => scripts}/copy-build-files.unit.spec.ts | 0 vite.config.ts | 2 +- 4 files changed, 3 insertions(+), 1 deletion(-) rename {utils => scripts}/copy-build-files.ts (100%) rename {utils => scripts}/copy-build-files.unit.spec.ts (100%) diff --git a/.github/workflows/unit-ci.yml b/.github/workflows/unit-ci.yml index 9df08f2d..80147fd4 100644 --- a/.github/workflows/unit-ci.yml +++ b/.github/workflows/unit-ci.yml @@ -13,6 +13,7 @@ on: - 'src/**' - 'test/**' - '_tools/**' + - 'scripts/**' - 'package.json' - 'package-lock.json' - 'tsconfig.json' @@ -51,6 +52,7 @@ on: - 'src/**' - 'test/**' - '_tools/**' + - 'scripts/**' - 'package.json' - 'package-lock.json' - 'tsconfig.json' diff --git a/utils/copy-build-files.ts b/scripts/copy-build-files.ts similarity index 100% rename from utils/copy-build-files.ts rename to scripts/copy-build-files.ts diff --git a/utils/copy-build-files.unit.spec.ts b/scripts/copy-build-files.unit.spec.ts similarity index 100% rename from utils/copy-build-files.unit.spec.ts rename to scripts/copy-build-files.unit.spec.ts diff --git a/vite.config.ts b/vite.config.ts index e4195f74..bb55bdd0 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -6,7 +6,7 @@ import path from "path"; import { fileURLToPath } from "node:url"; import fs from "node:fs"; import { platform } from "node:process"; -import { copyBuildFiles } from "./utils/copy-build-files"; +import { copyBuildFiles } from "./scripts/copy-build-files"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); function readJson(filePath: string) { From e46c05ddc20bf1761ae56d35ec3290e6aa1ed1b1 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 26 Oct 2026 18:15:51 +0000 Subject: [PATCH 52/68] Make strict startup content checks optional --- ...local_file_save_completion_and_batching.md | 54 +- docs/settings.md | 6 + package.json | 1 + .../SettingDialogue/AdvancedSettingSpecs.ts | 1 + .../SettingDialogue/PaneAdvanced.unit.spec.ts | 4 +- .../SettingDialogue/SettingSpec.unit.spec.ts | 14 +- .../SettingsPageCatalogue.unit.spec.ts | 1 + src/modules/main/ModuleLiveSyncMain.ts | 27 +- .../main/ModuleLiveSyncMain.unit.spec.ts | 98 +- test/e2e-obsidian/README.md | 2 + test/e2e-obsidian/scripts/run-focused.ts | 1 + .../scripts/startup-content-check.ts | 836 ++++++++++++++++++ updates.md | 7 + 13 files changed, 1029 insertions(+), 23 deletions(-) create mode 100644 test/e2e-obsidian/scripts/startup-content-check.ts diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md index 33ee9b8c..bbf67b13 100644 --- a/docs/adr/2026_10_local_file_save_completion_and_batching.md +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -59,10 +59,12 @@ that evidence. The inspected Commonlib source is commit ## Decision -Make local file processing recoverable until its result is established, use -content and provenance to reconcile ambiguous timestamps, and enable batching -in LiveSync mode only through an explicit choice after those contracts pass -verification. Apply the recovery contract with batching enabled or disabled. +Make local file processing recoverable until its result is established, retain +strict content and provenance reconciliation for manual and recovery scans, and +enable batching in LiveSync mode only through an explicit choice after those +contracts pass verification. Ordinary Obsidian start-up may skip a body read +only under the exact-match conditions below. Apply the recovery contract with +batching enabled or disabled. The content-reconciliation change targets selected ordinary files in the Offline Scanner's `NEWER_WINS` / `sync-newer` action. Keep the existing action @@ -208,12 +210,14 @@ finishes; correctness must also follow from the persistent record and start-up reconciliation. Remote availability must not determine whether local pending work survives shutdown. -### 4. Reconcile content when timestamps are inconclusive +### 4. Preserve strict content reconciliation -Retain the existing timestamp tolerance for platforms with coarse modification -times. Timestamp equality, including exactly equal raw timestamps, does not -prove content equality. The Offline Scanner must establish one of the following -outcomes through the ordinary non-forced file handler and its provenance rules: +Strict content checking retains the timestamp and provenance reconciliation +contract for platforms with coarse modification times. Timestamp equality, +including exactly equal raw timestamps, does not prove content equality. With +strict checking enabled, the Offline Scanner must establish one of the +following outcomes through the ordinary non-forced file handler and its +provenance rules: | Evidence | Required outcome | | ----------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | @@ -223,7 +227,19 @@ outcomes through the ordinary non-forced file handler and its provenance rules: | Vault content has no provable origin and matches no current live leaf | Preserve it according to the existing independent-branch rules. Do not attach it to an arbitrary database winner. | | Required content or provenance cannot be read, or state changes during comparison | Retain unresolved work and report or re-evaluate it. Do not mark it reconciled merely because the timestamp comparison is `EVEN`. | -The table describes stable-state reconciliation, not atomic database and +The `strictContentCheck` scan option is strict when omitted. Ordinary Obsidian +start-up can opt out through **Strict content check on startup**, which is +disabled by default. With the option disabled, a body read may be skipped only +when the file's raw modification time, byte size, and valid device-local +provenance exactly match a conflict-free current revision and no file +processing is pending. All other files retain strict content reconciliation. +An unrecorded edit that preserves its exact size and modification time can +therefore remain undetected by ordinary start-up while the option is disabled. +Manual scans, Fetch, Rebuild, and other explicit recovery operations remain +strict. Pending-event replay, incoming-change protection, and deletion guards +remain active in both modes. + +The table describes stable-state strict reconciliation, not atomic database and provenance persistence. Replay in the pre-provenance gap may create an identical sibling; with no remote descendant, one of the identical branches is tombstoned, leaving one live leaf with the exact saved content. If replication has @@ -249,12 +265,14 @@ Obsidian start-up can report `completed-with-file-failures` and allow unaffected files to proceed; an unresolved pair must not become `completed` to achieve that result. Keep the stricter callers' existing completion requirements. -This deliberately extends the current equal-time scanner contract. Existing -provenance tests expect no database write during that scan, including for an -edited file. Preserve their unchanged-file and later incoming-protection -scenarios; change the edited-file expectation to require preservation during -the scan itself. Test both identical timestamps and distinct raw timestamps -within one comparison bucket. +Keep the original unchanged-file, edited-file, untracked-provenance, +incoming-protection, and concurrent-recheck fixtures under strict content +checking, including identical timestamps and distinct raw timestamps within +one comparison bucket. Do not weaken those strict-mode assertions to describe +the ordinary start-up opt-out. Test the opt-out separately: only the exact +stat, provenance, conflict-free, and no-pending-processing match may skip a +body read. Pending-event replay and incoming or deletion guards remain covered +with the opt-out enabled. ### 5. Make batch timing and activation explicit @@ -369,7 +387,9 @@ matrix is: | Handler returns false, throws, or snapshot persistence fails | Flush reports failure; pending work is retained; unrelated recoverable work can continue. | | Buffered or in-progress work exists with no waiting timer | Flush still waits for actual completion. | | A flush overlaps new typing or a rename/delete barrier | The captured work completes without waiting for later work or deadlocking. | -| Equal timestamps: unchanged, edited, untracked, pruned provenance, unavailable Chunks, or concurrent recheck changes | Preserve correct ancestry and bytes, avoid duplicate revisions, and distinguish unresolved comparisons from successful reconciliation. | +| Strict check: equal timestamps, unchanged or edited files, untracked or pruned provenance, unavailable Chunks, or concurrent recheck changes | Preserve correct ancestry and bytes, avoid duplicate revisions, and distinguish unresolved comparisons from successful reconciliation. | +| Start-up check disabled: exact raw time, size, and provenance match with no conflict or pending processing | The body read may be skipped; an unrecorded edit which preserves exact size and modification time remains an accepted limitation. | +| Pending-event replay and incoming or deletion guards, with either start-up setting | Keep replay and protection active; do not skip work or remove a file based only on matching statistics. | | Settings save followed by stop; ordinary unload; interruption during replay | No operation is forgotten because a settings callback or shutdown released its timer. | | Sustained edits and more paths than available processing slots | Observe the intentional-delay deadline and progress of unrelated ready files. | | Old settings, mode changes, explicit opt-in, and settings import | Existing LiveSync mode users retain immediate saving until they choose batching. | diff --git a/docs/settings.md b/docs/settings.md index b53a7da2..135351aa 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -989,6 +989,12 @@ If enabled, chunks will be split into no more than 100 items. However, dedupe is Setting key: useSegmenter If this enabled, chunks will be split into semantically meaningful segments. Not all platforms support this feature. +#### Strict content check on startup + +Setting key: strictContentCheckOnStartup + +When enabled, ordinary start-up compares file content with the local database even when the file's exact modification time and size match. When disabled (the default), this comparison can be skipped only when the modification time, size, and recorded device-local provenance match a conflict-free current revision, and no file processing is pending. An edit made outside the normal file-event path which preserves its exact modification time and size can therefore go undetected by ordinary start-up. Manual scans and explicit Fetch or Rebuild operations keep strict content checking. + ### 3. Transfer Tweak #### Fetch chunks on demand diff --git a/package.json b/package.json index d5927927..076bed16 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "test:e2e:obsidian:p2p-connection-check": "tsx test/e2e-obsidian/scripts/p2p-connection-check.ts", "test:e2e:obsidian:p2p-connection-check:services": "npm run test:e2e:obsidian:p2p-connection-check -- --manage-p2p", "test:e2e:obsidian:partial-startup-file-failure": "tsx test/e2e-obsidian/scripts/partial-startup-file-failure.ts", + "test:e2e:obsidian:startup-content-check": "tsx test/e2e-obsidian/scripts/startup-content-check.ts", "test:e2e:obsidian:startup-scan": "tsx test/e2e-obsidian/scripts/startup-scan.ts", "test:e2e:obsidian:stale-file-restart": "tsx test/e2e-obsidian/scripts/stale-file-restart.ts", "test:e2e:obsidian:folder-batch": "tsx test/e2e-obsidian/scripts/folder-batch.ts", diff --git a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts index 1ba23bb5..59e09c17 100644 --- a/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts +++ b/src/modules/features/SettingDialogue/AdvancedSettingSpecs.ts @@ -23,6 +23,7 @@ export function createAdvancedSettingSpecGroups({ control: { type: "dropdown", options: () => ChunkAlgorithmNames }, }, { key: "customChunkSize", control: { type: "number", min: 0, allowZero: true } }, + { key: "strictContentCheckOnStartup", control: { type: "toggle" } }, { key: "pathNormalizationVersion", holdValue: true, diff --git a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts index 859727cd..85597fbf 100644 --- a/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts +++ b/src/modules/features/SettingDialogue/PaneAdvanced.unit.spec.ts @@ -43,7 +43,7 @@ describe("paneAdvanced", () => { }; paneAdvanced.call(host as never, {} as HTMLElement, { addPanel } as never); - await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(10)); + await vi.waitFor(() => expect(settingHarness.rendered).toHaveLength(11)); expect(addPanel.mock.calls.map(([, heading]) => heading)).toEqual([ "Memory cache", @@ -56,6 +56,7 @@ describe("paneAdvanced", () => { panelElements.get("Local Database Tweak"), panelElements.get("Local Database Tweak"), panelElements.get("Local Database Tweak"), + panelElements.get("Local Database Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), panelElements.get("Transfer Tweak"), @@ -67,6 +68,7 @@ describe("paneAdvanced", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "strictContentCheckOnStartup", "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", diff --git a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts index 62958bc4..7e957aeb 100644 --- a/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingSpec.unit.spec.ts @@ -17,7 +17,7 @@ const rangeMessages = { }; describe("Advanced setting specifications", () => { - it("lists only the ten controls currently exposed by the Advanced page", () => { + it("lists only the eleven controls currently exposed by the Advanced page", () => { const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => true }); expect(groups.map(({ heading }) => heading)).toEqual([ @@ -30,6 +30,7 @@ describe("Advanced setting specifications", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "strictContentCheckOnStartup", "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", @@ -40,6 +41,17 @@ describe("Advanced setting specifications", () => { ]); }); + it("exposes strict content checking as an ungated Advanced toggle", () => { + const spec = createAdvancedSettingSpecGroups({ isCouchDB: () => false }) + .find(({ heading }) => heading === "Local Database Tweak") + ?.items.find(({ key }) => key === "strictContentCheckOnStartup"); + if (!spec) throw new Error("Strict content checking is not exposed on the Advanced page"); + + expect(spec.control).toEqual({ type: "toggle" }); + expect(spec.visible).toBeUndefined(); + expect(spec.disabled).toBeUndefined(); + }); + it("keeps path identity and the four remote chunk controls CouchDB-only", () => { let couchDB = false; const groups = createAdvancedSettingSpecGroups({ isCouchDB: () => couchDB }); diff --git a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts index 32bf0e82..a2605f41 100644 --- a/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts +++ b/src/modules/features/SettingDialogue/SettingsPageCatalogue.unit.spec.ts @@ -86,6 +86,7 @@ describe("settings page catalogue", () => { "hashCacheMaxCount", "chunkSplitterVersion", "customChunkSize", + "strictContentCheckOnStartup", "pathNormalizationVersion", "readChunksOnline", "useOnlyLocalChunk", diff --git a/src/modules/main/ModuleLiveSyncMain.ts b/src/modules/main/ModuleLiveSyncMain.ts index 7cdc656a..228f75db 100644 --- a/src/modules/main/ModuleLiveSyncMain.ts +++ b/src/modules/main/ModuleLiveSyncMain.ts @@ -19,6 +19,22 @@ import { initialiseWorkerModule } from "@vrtmrz/livesync-commonlib/compat/worker import { manifestVersion, packageVersion } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvVars"; import { VaultScanResults } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; +const OBSIDIAN_PLATFORMS = new Set([ + "android-app", + "ios", + "macos", + "mobile-app", + "mobile", + "safari", + "desktop", + "desktop-app", + "unknown-obsidian", +]); + +function isObsidianPlatform(platform: string): boolean { + return OBSIDIAN_PLATFORMS.has(platform); +} + export class ModuleLiveSyncMain extends AbstractModule { async _onLiveSyncReady() { if (!(await this.core.services.appLifecycle.onLayoutReady())) return false; @@ -44,13 +60,20 @@ export class ModuleLiveSyncMain extends AbstractModule { } } // Ordinary start-up may continue when individual files could not be - // processed. Explicit Fetch and Rebuild flows retain the strict default. + // processed. Explicit Fetch and Rebuild flows remain strict. const { ignoreSuspending = false, continueOnFileFailure = true } = this.core.startupDatabaseOptions; + // Only ordinary Obsidian start-up honours the opt-out. Keep other hosts + // and explicit operation modes strict. + const strictContentCheck = + !isObsidianPlatform(this.services.API.getPlatform()) || + this.settings.strictContentCheckOnStartup === true || + this.core.startupDatabaseOptions.continueOnFileFailure === false; const initialisationResult = await this.services.databaseEvents.initialiseDatabase( false, false, ignoreSuspending, - continueOnFileFailure + continueOnFileFailure, + { strictContentCheck } ); if (initialisationResult === VaultScanResults.FAILED) { this._log($msg("Ui.Common.LocalDatabaseInitialisationFailed"), LOG_LEVEL_NOTICE); diff --git a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts index bbb744ce..be9a9014 100644 --- a/src/modules/main/ModuleLiveSyncMain.unit.spec.ts +++ b/src/modules/main/ModuleLiveSyncMain.unit.spec.ts @@ -90,9 +90,11 @@ describe("ModuleLiveSyncMain", () => { }, }, services: { + API: { getPlatform: () => "desktop" }, databaseEvents: { initialiseDatabase }, }, settings: { + strictContentCheckOnStartup: false, suspendFileWatching: false, suspendParseReplicationResult: false, }, @@ -102,7 +104,7 @@ describe("ModuleLiveSyncMain", () => { const result = await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); expect(result).toBe(false); - expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true); + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true, { strictContentCheck: false }); expect(log).toHaveBeenCalledWith("Ui.Common.LocalDatabaseInitialisationFailed", LOG_LEVEL_NOTICE); }); @@ -120,6 +122,7 @@ describe("ModuleLiveSyncMain", () => { services: { appLifecycle }, }, services: { + API: { getPlatform: () => "desktop" }, appLifecycle, control: { applySettings: vi.fn(async () => undefined) }, databaseEvents: { initialiseDatabase }, @@ -152,6 +155,7 @@ describe("ModuleLiveSyncMain", () => { }, }, services: { + API: { getPlatform: () => "desktop" }, databaseEvents: { initialiseDatabase }, }, settings: { @@ -163,6 +167,96 @@ describe("ModuleLiveSyncMain", () => { await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); - expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false); + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, true, false, { strictContentCheck: true }); + }); + + it("passes the strict content check setting for ordinary Obsidian start-up", async () => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: {}, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + API: { getPlatform: () => "desktop" }, + databaseEvents: { initialiseDatabase }, + }, + settings: { + strictContentCheckOnStartup: true, + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true, { strictContentCheck: true }); + }); + + it.each(["server", "browser"])( + "keeps the strict content check on non-Obsidian host start-up (%s)", + async (platform) => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: true }, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + API: { getPlatform: () => platform }, + databaseEvents: { initialiseDatabase }, + }, + settings: { + strictContentCheckOnStartup: false, + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, true, { strictContentCheck: true }); + } + ); + + it("keeps explicit recovery startup options strict when the setting is off", async () => { + const initialiseDatabase = vi.fn(async () => false); + const host = { + core: { + startupDatabaseOptions: { + ignoreSuspending: false, + continueOnFileFailure: false, + }, + services: { + appLifecycle: { + onLayoutReady: vi.fn(async () => true), + }, + }, + }, + services: { + API: { getPlatform: () => "desktop" }, + databaseEvents: { initialiseDatabase }, + }, + settings: { + strictContentCheckOnStartup: false, + suspendFileWatching: false, + suspendParseReplicationResult: false, + }, + _log: vi.fn(), + }; + + await ModuleLiveSyncMain.prototype._onLiveSyncReady.call(host as never); + + expect(initialiseDatabase).toHaveBeenCalledWith(false, false, false, false, { strictContentCheck: true }); }); }); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index ae4790b3..a8355261 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -100,6 +100,8 @@ The same scenario checks the five actions in Reporting Tools, the separate Check `startup-integrity` uses the registered Hatch integrity-check event with real Obsidian storage and a local database. It verifies that healthy files are inspected without reading their contents, that a controlled stale stat followed by matching real bytes does not produce a repair dialogue, and that a changed final stat leaves the check incomplete until a settled retry. The stat variations are injected; this scenario exercises the checker used during start-up rather than restarting the process. Natural configured start-up remains covered by `startup-scan`. +`startup-content-check` needs no remote service. It pre-creates a configurable set of Vault files, lets the plug-in populate the real local database and content provenance, then makes a same-size offline edit with the exact recorded raw mtime. A natural default-off cold restart must preserve the Vault edit without changing its database revision or branch; an explicit `scanVault(true)` must then detect it. After saving strict content checking as enabled, a second natural cold restart must detect another same-stat offline edit and update the current branch and provenance. The fixture reports a clear precision mismatch if the file system cannot restore the exact recorded mtime. Output includes the `main.js` SHA-256 and elapsed start-up timings; launch-to-core-ready timings include Obsidian and runner start-up, rather than measuring only the database scan. Run it with `npm run test:e2e:obsidian:focused -- startup-content-check`; `E2E_STARTUP_CHECK_FILE_COUNT` defaults to 20 and can be set to 5000 for a larger startup measurement. + `third-party-chunks` reflects synthetic externally generated text through the installed Commonlib and the real Obsidian file handler. It supplies a split surrogate pair at the file-access read boundary, with size checking enabled, then verifies exact Vault and on-disk text, unchanged Metadata, and matching persisted provenance. Complete-pair text and a binary attachment provide controls. The underlying Metadata and Chunks are real local database entries; this does not establish that CouchDB or a particular third-party application generates this chunk layout. No remote service is required. `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 8e394ccd..ffb805ed 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -38,6 +38,7 @@ const focusedScenarios = new Set([ "partial-startup-file-failure", "startup-scan", "startup-integrity", + "startup-content-check", "stale-file-restart", "folder-batch", "setup-uri-workflow", diff --git a/test/e2e-obsidian/scripts/startup-content-check.ts b/test/e2e-obsidian/scripts/startup-content-check.ts new file mode 100644 index 00000000..b648d166 --- /dev/null +++ b/test/e2e-obsidian/scripts/startup-content-check.ts @@ -0,0 +1,836 @@ +/** Checks the default and strict body checks across natural Obsidian restarts. */ +import { createHash } from "node:crypto"; +import { mkdir, readFile, stat, utimes, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { captureObsidianElement, captureObsidianPage, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +const targetPath = "startup-content-check-00000.md"; +const versions = [ + "Startup content check fixture body A\n", + "Startup content check fixture body B\n", + "Startup content check fixture body C\n", +] as const; + +type TargetState = { + rev: string; + rawMtime: number | null; + metadataMtime: number; + size: number; + documentSize: number; + dbContent: string; + vaultContent: string; + branchCount: number; + conflictCount: number; + provenanceRevision: string | null; + contentHash: string | null; + contentSize: number | null; + fileMtime: number | null; + adapterMtime: number | null; + storageMtime: number | null; + adapterSize: number | null; + storageSize: number | null; +}; + +type InitialState = { + fileCount: number; + indexedCount: number; + conflictCount: number; + pendingCount: number; + strictContentCheckOnStartup: boolean; + targetProvenanceValid: boolean; + target: TargetState; +}; + +type ManualScanState = TargetState & { + scanResult: boolean | string; + scanMs: number; + strictContentCheckOnStartup: boolean; +}; + +type StartupFailureCore = { + services?: { + database?: { isDatabaseReady?: () => boolean }; + appLifecycle?: { + isReady?: () => boolean; + isSuspended?: () => boolean; + getUnresolvedMessages?: () => Promise; + }; + }; +}; + +type StartupFailureGlobal = typeof globalThis & { + app?: { + commands?: { executeCommandById?: (commandId: string) => boolean }; + plugins?: { plugins?: Record }; + }; +}; + +const sensitiveDiagnosticLine = + /security seed|passphrase|password|credential|secret|access.?key|jwt.?key|authori[sz]ation|obsidian:\/\/setuplivesync|sls\+/iu; + +function redactDiagnosticLine(line: string): string { + if (sensitiveDiagnosticLine.test(line)) return "[REDACTED SENSITIVE LOG LINE]"; + return line.replace(/\bhttps?:\/\/[^/\s:@]+:[^@\s/]+@/giu, "https://[REDACTED]@"); +} + +function requireCondition(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function assertEqual(actual: unknown, expected: unknown, message: string): void { + if (actual !== expected) { + throw new Error(message + "\nExpected: " + String(expected) + "\nActual: " + String(actual)); + } +} + +function fileCountFromEnvironment(): number { + const raw = process.env.E2E_STARTUP_CHECK_FILE_COUNT ?? "20"; + if (!/^[1-9]\d*$/.test(raw)) { + throw new Error("E2E_STARTUP_CHECK_FILE_COUNT must be a positive integer; received " + raw + "."); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value > 5000) { + throw new Error("E2E_STARTUP_CHECK_FILE_COUNT must be between 1 and 5000; received " + raw + "."); + } + return value; +} + +function fixturePath(index: number): string { + return "startup-content-check-" + String(index).padStart(5, "0") + ".md"; +} + +function fixtureContent(index: number): string { + if (index === 0) return versions[0]; + return "Startup content check fixture " + String(index).padStart(5, "0") + "\n"; +} + +function sha256(content: string): string { + return "sha256:" + createHash("sha256").update(content).digest("hex"); +} + +function assertExactMtime(state: TargetState, label: string): void { + const expected = state.rawMtime; + if (expected === null || !Number.isFinite(expected)) { + throw new Error( + "Fixture precision mismatch during " + label + ": persisted provenance raw mtime was unavailable." + ); + } + for (const [name, actual] of [ + ["TFile", state.fileMtime], + ["Vault adapter", state.adapterMtime], + ["StorageAccess", state.storageMtime], + ] as const) { + if (actual !== expected) { + throw new Error( + "Fixture precision mismatch during " + + label + + ": " + + name + + " mtime " + + String(actual) + + " did not exactly match persisted raw mtime " + + expected + + "." + ); + } + } + assertEqual(state.metadataMtime, expected, label + ": Metadata and raw mtimes differed."); +} + +function requireRecordedMtime(state: TargetState, label: string): number { + if (state.rawMtime === null || !Number.isFinite(state.rawMtime)) { + throw new Error( + "Fixture precision mismatch before " + label + ": persisted provenance raw mtime was unavailable." + ); + } + return state.rawMtime; +} + +async function createFixtures(vaultPath: string, fileCount: number, fixtureMtime: number): Promise { + const paths = Array.from({ length: fileCount }, (_, index) => fixturePath(index)); + const exactMtime = new Date(fixtureMtime); + const batchSize = 100; + for (let offset = 0; offset < paths.length; offset += batchSize) { + await Promise.all( + paths.slice(offset, offset + batchSize).map(async (path, index) => { + const fileIndex = offset + index; + const fullPath = join(vaultPath, path); + await writeFile(fullPath, fixtureContent(fileIndex), "utf-8"); + await utimes(fullPath, exactMtime, exactMtime); + const created = await stat(fullPath); + if (created.mtimeMs !== fixtureMtime) { + throw new Error( + "Fixture precision mismatch before bootstrap: filesystem mtimeMs " + + created.mtimeMs + + " did not exactly match requested integer-millisecond mtime " + + fixtureMtime + + "." + ); + } + }) + ); + } + return paths; +} + +async function restoreOfflineEdit( + vaultPath: string, + content: string, + expectedMtime: number, + expectedSize: number, + label: string +): Promise { + const fullPath = join(vaultPath, targetPath); + if (Buffer.byteLength(content, "utf-8") !== expectedSize) { + throw new Error(label + ": the offline edit did not preserve the recorded byte size."); + } + if (!Number.isFinite(expectedMtime)) { + throw new Error(label + ": the persisted raw mtime was not a finite number."); + } + await writeFile(fullPath, content, "utf-8"); + const exactMtime = new Date(expectedMtime); + await utimes(fullPath, exactMtime, exactMtime); + const restored = await stat(fullPath); + if (restored.mtimeMs !== expectedMtime) { + throw new Error( + "Fixture precision mismatch before " + + label + + ": filesystem mtimeMs " + + restored.mtimeMs + + " did not exactly match persisted raw mtime " + + expectedMtime + + "." + ); + } + assertEqual(restored.size, expectedSize, label + ": the restored fixture byte size differed from Metadata."); +} + +async function main(): Promise { + const fileCount = fileCountFromEnvironment(); + if (fileCount >= 1000) { + process.env.E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS ??= "120000"; + process.env.E2E_OBSIDIAN_READY_TIMEOUT_MS ??= "120000"; + } + for (const content of versions) { + assertEqual( + Buffer.byteLength(content, "utf-8"), + Buffer.byteLength(versions[0], "utf-8"), + "The same-stat offline edit fixtures must have identical byte lengths." + ); + } + + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + const vault = await createTemporaryVault("obsidian-startup-content-check-"); + const fixtureMtime = Math.floor(Date.now()); + const sessionState: { current: ObsidianLiveSyncSession | undefined } = { current: undefined }; + const pluginData = createE2eCouchDbPluginData( + // Replication is disabled; the scenario uses only the persisted local database. + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName: "startup-content-check-local" }, + { + deviceAndVaultName: "startup-content-check-device", + strictContentCheckOnStartup: false, + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + } + ); + + async function launch(pluginStartup: "controlled" | "natural", label: string, bootstrap = false): Promise { + console.log("Starting " + label + " with the real Obsidian process."); + const startedAt = Date.now(); + const options = { + binary, + cliBinary, + vault, + pluginStartup, + ...(bootstrap + ? { + pluginData, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + } + : {}), + }; + const currentSession = await startObsidianLiveSyncSession(options); + sessionState.current = currentSession; + const readiness = await waitForLiveSyncCoreReady(cliBinary, currentSession.cliEnv); + requireCondition( + readiness.databaseReady && readiness.appReady, + "Self-hosted LiveSync did not reach core readiness: " + JSON.stringify(readiness) + ); + const elapsedMs = Date.now() - startedAt; + console.log("Reached LiveSync core readiness for " + label + " in " + elapsedMs + " ms."); + return elapsedMs; + } + + async function evaluate(body: string): Promise { + const currentSession = sessionState.current; + if (!currentSession) throw new Error("The Obsidian session is not running."); + const script = [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync']?.core;", + "const require=(condition,message)=>{if(!condition)throw new Error(message);};", + "const contentOf=value=>Array.isArray(value)?value.join(''):value;", + "const targetState=async path=>{", + " require(core,'Self-hosted LiveSync did not load');", + " const meta=await core.localDatabase.getDBEntryMeta(path,{conflicts:true},true);", + " require(meta&&meta._rev,'The target Metadata was missing');", + " const document=await core.localDatabase.getDBEntryFromMeta(meta);", + " require(document&&document.data!==undefined,'The target Chunks could not be loaded');", + " const dbContent=contentOf(document.data);", + " require(typeof dbContent==='string','The text fixture did not load as text');", + " const file=app.vault.getAbstractFileByPath(path);", + " require(file&&file.stat,'The target Vault file was missing');", + " const adapterStat=await app.vault.adapter.stat(path);", + " const storage=core.serviceModules.storageAccess;", + " const storageStat=await storage.statHidden(path);", + " const provenance=await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').get(path);", + " return {rev:meta._rev,rawMtime:provenance?.observedStorageMtime??null,metadataMtime:Number(meta.mtime),", + " size:Number(meta.size),documentSize:Number(document.size),dbContent,", + " vaultContent:await app.vault.adapter.read(path),", + " branchCount:1+(meta._conflicts??[]).length,conflictCount:(meta._conflicts??[]).length,", + " provenanceRevision:provenance?.revision??null,contentHash:provenance?.contentHash??null,", + " contentSize:provenance?.contentSize??null,fileMtime:file.stat.mtime??null,", + " adapterMtime:adapterStat?.mtime??null,storageMtime:storageStat?.mtime??null,", + " adapterSize:adapterStat?.size??null,storageSize:storageStat?.size??null};", + "};", + body, + "})()", + ].join("\n"); + return await evalObsidianJson(cliBinary, script, currentSession.cliEnv); + } + + async function stopSession(): Promise { + const currentSession = sessionState.current; + if (!currentSession) return; + await currentSession.app.stop(); + sessionState.current = undefined; + } + + async function captureSettingScreenshots(): Promise<{ advancedPage: string; settingRow: string }> { + const currentSession = sessionState.current; + if (!currentSession) throw new Error("The Obsidian session is not running."); + const port = currentSession.remoteDebuggingPort; + const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e"; + const advancedPagePath = join(outputDirectory, "startup-content-check-setting-location.png"); + await mkdir(outputDirectory, { recursive: true }); + + await withObsidianPage(port, async (page) => { + const navigator = await openLiveSyncSettings(page); + const advanced = await navigator.openPage("Advanced"); + const section = advanced.getByText("Local Database Tweak", { exact: true }).first(); + await section.waitFor({ state: "visible" }); + await section.scrollIntoViewIfNeeded(); + const row = advanced.locator(".setting-item").filter({ + has: navigator.page.getByText("Strict content check on startup", { exact: true }), + }); + await row.waitFor({ state: "visible" }); + await row + .getByText( + "Compare file contents even when modification time and size match. This can slow startup in large vaults.", + { exact: true } + ) + .waitFor({ state: "visible" }); + assertEqual( + await row.locator(".checkbox-container.is-enabled").count(), + 0, + "The screenshot fixture toggle was not visibly OFF." + ); + await navigator.dialogue.screenshot({ + path: advancedPagePath, + animations: "disabled", + style: ".notice-container { visibility: hidden !important; }", + }); + }); + + const settingRowPath = await captureObsidianElement( + port, + "startup-content-check-setting-row.png", + async (page) => { + const navigator = await openLiveSyncSettings(page); + const advanced = await navigator.openPage("Advanced"); + const row = advanced.locator(".setting-item").filter({ + has: navigator.page.getByText("Strict content check on startup", { exact: true }), + }); + await row.waitFor({ state: "visible" }); + await row + .getByText( + "Compare file contents even when modification time and size match. This can slow startup in large vaults.", + { exact: true } + ) + .waitFor({ state: "visible" }); + assertEqual( + await row.locator(".checkbox-container.is-enabled").count(), + 0, + "The cropped screenshot fixture toggle was not visibly OFF." + ); + return row; + } + ); + + await withObsidianPage(port, async (page) => { + const navigator = await openLiveSyncSettings(page); + await navigator.close(); + }); + return { advancedPage: advancedPagePath, settingRow: settingRowPath }; + } + + async function captureFailureEvidence(error: unknown): Promise { + const currentSession = sessionState.current; + if (!currentSession) return; + const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e"; + const visibleScreenshotPath = join(outputDirectory, "startup-content-check-failure-visible.png"); + const screenshotPath = join(outputDirectory, "startup-content-check-failure.png"); + const snapshotPath = join(outputDirectory, "startup-content-check-failure.json"); + await mkdir(outputDirectory, { recursive: true }); + + const processOutput = currentSession.app.output(); + const snapshot: { + capturedAt: string; + failure: string; + screenshotPath: string; + visibleScreenshotPath: string; + rendererState: unknown; + recentLog: string[]; + logCaptureError?: string; + processOutput: { stdout: string; stderr: string }; + } = { + capturedAt: new Date().toISOString(), + failure: redactDiagnosticLine( + (error instanceof Error ? (error.stack ?? error.message) : String(error)).slice(0, 2000) + ), + screenshotPath, + visibleScreenshotPath, + rendererState: { unavailable: true }, + recentLog: [], + processOutput: { + stdout: processOutput.stdout + .split(/\r?\n/u) + .slice(-30) + .map(redactDiagnosticLine) + .join("\n") + .slice(-3000), + stderr: processOutput.stderr + .split(/\r?\n/u) + .slice(-30) + .map(redactDiagnosticLine) + .join("\n") + .slice(-3000), + }, + }; + + const capturedScreenshot = await captureObsidianPage( + currentSession.remoteDebuggingPort, + "startup-content-check-failure.png", + async (page) => { + await page.screenshot({ path: visibleScreenshotPath, fullPage: true }); + try { + snapshot.rendererState = await page.evaluate(async () => { + const host = globalThis as StartupFailureGlobal; + const core = host.app?.plugins?.plugins?.["obsidian-livesync"]?.core; + let unresolvedMessages: string[] = []; + let unresolvedMessagesError: string | undefined; + try { + const messages = await core?.services?.appLifecycle?.getUnresolvedMessages?.(); + unresolvedMessages = (messages ?? []) + .flat() + .filter((message) => message !== undefined && message !== null) + .map((message) => String(message).slice(0, 300)) + .slice(-20); + } catch (diagnosticError) { + unresolvedMessagesError = + diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError); + } + return { + url: location.href, + title: document.title, + bodyText: (document.body?.innerText ?? "").slice(0, 1200), + notices: Array.from(document.querySelectorAll(".notice")) + .slice(0, 10) + .map((notice) => (notice.textContent ?? "").trim().slice(0, 300)), + coreAvailable: core !== undefined, + databaseReady: core?.services?.database?.isDatabaseReady?.() ?? false, + appReady: core?.services?.appLifecycle?.isReady?.() ?? false, + suspended: core?.services?.appLifecycle?.isSuspended?.() ?? false, + unresolvedMessages, + unresolvedMessagesError, + }; + }); + } catch (snapshotError) { + snapshot.rendererState = { + error: snapshotError instanceof Error ? snapshotError.message : String(snapshotError), + }; + } + + try { + const opened = await page.evaluate( + (commandId) => + (globalThis as StartupFailureGlobal).app?.commands?.executeCommandById?.(commandId) === + true, + "obsidian-livesync:view-log" + ); + if (!opened) throw new Error("The Show log command was not registered."); + const logPane = page.locator(".logpane"); + await logPane.waitFor({ state: "visible", timeout: 3000 }); + snapshot.recentLog = (await logPane.locator(".log pre").allTextContents()) + .slice(-30) + .map((line) => redactDiagnosticLine(line.slice(0, 800))); + } catch (logError) { + snapshot.logCaptureError = logError instanceof Error ? logError.message : String(logError); + } + await writeFile(snapshotPath, JSON.stringify(snapshot, null, 2) + "\n", "utf8"); + console.error("Startup content check failure diagnostics: " + snapshotPath); + } + ); + console.error("Startup content check failure screenshot: " + capturedScreenshot); + console.error("Startup content check visible-page screenshot: " + visibleScreenshotPath); + } + + const countJson = JSON.stringify(fileCount); + const targetPathJson = JSON.stringify(targetPath); + + try { + await createFixtures(vault.path, fileCount, fixtureMtime); + // The controlled bootstrap establishes the isolated profile and compatibility marker. + // The two compared starts below are natural Obsidian cold restarts. + const bootstrapMs = await launch("controlled", "controlled bootstrap", true); + const initial = await evaluate( + [ + "require(core,'Self-hosted LiveSync did not load');", + "require(core.services.setting.currentSettings().strictContentCheckOnStartup===false,", + " 'The bootstrap did not use strictContentCheckOnStartup=false');", + "require(await core.services.fileProcessing.commitPendingFileEvents(),", + " 'The bootstrap local file events did not flush');", + "const fixturePaths=new Set(Array.from({length:", + countJson, + "},(_,index)=>'startup-content-check-'+String(index).padStart(5,'0')+'.md'));", + "let indexedCount=0,conflictCount=0;", + "for await(const meta of core.localDatabase.findAllNormalDocs({conflicts:true})||[]){", + " if(!fixturePaths.has(meta.path))continue;", + " indexedCount++;", + " if((meta._conflicts??[]).length>0)conflictCount++;", + "}", + "const pending=await core.serviceModules.storageAccess.vaultManager.adapter.persistence.loadSnapshot()??[];", + "const target=await targetState(", + targetPathJson, + ");", + "return JSON.stringify({fileCount:", + countJson, + ",indexedCount,conflictCount,pendingCount:pending.length,", + "strictContentCheckOnStartup:core.services.setting.currentSettings().strictContentCheckOnStartup===true,", + "targetProvenanceValid:target.provenanceRevision===target.rev&&", + " typeof target.contentHash==='string'&&target.contentHash!==''&&target.contentSize===target.size,", + "target});", + ].join("\n") + ); + assertEqual(initial.fileCount, fileCount, "The reported initial fixture count differed."); + assertEqual(initial.indexedCount, fileCount, "The pre-created files were not all indexed on bootstrap."); + assertEqual(initial.conflictCount, 0, "The initial fixture unexpectedly contained conflict branches."); + assertEqual(initial.pendingCount, 0, "Pending storage work remained before the cold restart."); + assertEqual( + initial.strictContentCheckOnStartup, + false, + "The initial strict startup setting was unexpectedly enabled." + ); + assertEqual( + initial.targetProvenanceValid, + true, + "The target did not have current persisted content provenance." + ); + assertEqual( + initial.target.rawMtime, + fixtureMtime, + "The initial provenance did not retain the fixture's exact integer-millisecond mtime." + ); + assertEqual(initial.target.dbContent, versions[0], "The initial target body was not stored."); + assertEqual(initial.target.vaultContent, versions[0], "The initial target Vault body differed."); + assertEqual(initial.target.documentSize, initial.target.size, "The initial Chunk size differed from Metadata."); + assertEqual( + initial.target.adapterSize, + initial.target.size, + "The initial adapter byte size differed from Metadata." + ); + assertEqual( + initial.target.storageSize, + initial.target.size, + "The initial StorageAccess byte size differed from Metadata." + ); + assertEqual( + initial.target.contentHash, + sha256(versions[0]), + "The initial provenance hash did not match the body." + ); + assertEqual( + initial.target.contentSize, + Buffer.byteLength(versions[0], "utf-8"), + "The initial provenance byte size was wrong." + ); + assertEqual( + initial.target.provenanceRevision, + initial.target.rev, + "Initial provenance did not match the current branch." + ); + assertEqual(initial.target.conflictCount, 0, "The initial target had a conflict branch."); + assertEqual(initial.target.branchCount, 1, "The initial target did not have exactly one branch."); + assertExactMtime(initial.target, "bootstrap"); + + await stopSession(); + await restoreOfflineEdit( + vault.path, + versions[1], + requireRecordedMtime(initial.target, "default-off cold restart"), + initial.target.size, + "default-off cold restart" + ); + + const ordinaryStartupMs = await launch("natural", "default-off natural cold restart"); + const ordinary = await evaluate( + [ + "const state=await targetState(", + targetPathJson, + ");", + "const pending=await core.serviceModules.storageAccess.vaultManager.adapter.persistence.loadSnapshot()??[];", + "return JSON.stringify({...state,pendingCount:pending.length,strictContentCheckOnStartup:", + "core.services.setting.currentSettings().strictContentCheckOnStartup===true});", + ].join("\n") + ); + assertEqual( + ordinary.strictContentCheckOnStartup, + false, + "The natural default-off restart enabled strict body checks." + ); + assertEqual(ordinary.pendingCount, 0, "Pending storage work remained during the default-off restart."); + assertEqual(ordinary.rev, initial.target.rev, "The default-off restart changed the local Metadata revision."); + assertEqual(ordinary.dbContent, versions[0], "The default-off restart changed the local database body."); + assertEqual(ordinary.vaultContent, versions[1], "The default-off restart overwrote the offline Vault edit."); + assertEqual(ordinary.size, initial.target.size, "The default-off restart changed the Metadata byte size."); + assertEqual(ordinary.documentSize, initial.target.size, "The default-off restart changed the Chunk byte size."); + assertEqual( + ordinary.adapterSize, + initial.target.size, + "The default-off restart changed the adapter byte size." + ); + assertEqual( + ordinary.storageSize, + initial.target.size, + "The default-off restart changed the StorageAccess byte size." + ); + assertEqual( + ordinary.provenanceRevision, + initial.target.rev, + "The default-off restart changed persisted provenance." + ); + assertEqual( + ordinary.contentHash, + initial.target.contentHash, + "The default-off restart changed the recorded content hash." + ); + assertEqual( + ordinary.contentSize, + initial.target.contentSize, + "The default-off restart changed the recorded byte size." + ); + assertEqual(ordinary.conflictCount, 0, "The default-off restart created a conflict branch."); + assertEqual(ordinary.branchCount, 1, "The default-off restart changed the branch count."); + assertExactMtime(ordinary, "default-off natural restart"); + + const manual = await evaluate( + [ + "const startedAt=Date.now();", + "const scanResult=await core.services.vault.scanVault(true);", + "const scanMs=Date.now()-startedAt;", + "require(scanResult===true,'The explicit strict manual Vault scan did not complete');", + "const state=await targetState(", + targetPathJson, + ");", + "core.settings.strictContentCheckOnStartup=true;", + "await core.services.setting.saveSettingData();", + "require(core.services.setting.currentSettings().strictContentCheckOnStartup===true,", + " 'The strict startup setting did not persist in the active settings');", + "return JSON.stringify({...state,scanResult,scanMs,strictContentCheckOnStartup:", + "core.services.setting.currentSettings().strictContentCheckOnStartup===true});", + ].join("\n") + ); + assertEqual(manual.scanResult, true, "The explicit strict manual scan returned an unexpected result."); + assertEqual(manual.dbContent, versions[1], "The manual strict scan did not detect the same-stat offline edit."); + assertEqual(manual.vaultContent, versions[1], "The manual strict scan changed the Vault body."); + assertEqual(manual.size, initial.target.size, "The manual strict scan changed the Metadata byte size."); + assertEqual(manual.documentSize, initial.target.size, "The manual strict scan changed the Chunk byte size."); + assertEqual(manual.adapterSize, initial.target.size, "The manual strict scan changed the adapter byte size."); + assertEqual( + manual.storageSize, + initial.target.size, + "The manual strict scan changed the StorageAccess byte size." + ); + requireCondition( + manual.rev !== ordinary.rev, + "The manual strict scan did not create a revision for the offline edit." + ); + assertEqual( + manual.provenanceRevision, + manual.rev, + "The manual scan did not record provenance for the detected edit." + ); + assertEqual(manual.contentHash, sha256(versions[1]), "The manual scan recorded the wrong content hash."); + assertEqual(manual.contentSize, initial.target.size, "The manual scan recorded the wrong content size."); + assertEqual(manual.conflictCount, 0, "The manual strict scan created a conflict branch."); + assertEqual(manual.branchCount, 1, "The manual strict scan changed the branch count."); + assertEqual(manual.strictContentCheckOnStartup, true, "The strict startup setting was not saved."); + assertExactMtime(manual, "manual strict scan"); + + await stopSession(); + await restoreOfflineEdit( + vault.path, + versions[2], + requireRecordedMtime(manual, "strict cold restart"), + manual.size, + "strict cold restart" + ); + + const strictStartupMs = await launch("natural", "strict natural cold restart"); + const strict = await evaluate( + [ + "const state=await targetState(", + targetPathJson, + ");", + "const pending=await core.serviceModules.storageAccess.vaultManager.adapter.persistence.loadSnapshot()??[];", + "return JSON.stringify({...state,pendingCount:pending.length,strictContentCheckOnStartup:", + "core.services.setting.currentSettings().strictContentCheckOnStartup===true});", + ].join("\n") + ); + assertEqual(strict.strictContentCheckOnStartup, true, "The natural strict restart did not retain its setting."); + assertEqual(strict.pendingCount, 0, "Pending storage work remained during the strict restart."); + requireCondition( + strict.rev !== manual.rev, + "The strict natural restart did not detect the same-stat offline edit." + ); + assertEqual(strict.dbContent, versions[2], "The strict natural restart did not store the offline Vault body."); + assertEqual(strict.vaultContent, versions[2], "The strict natural restart changed the offline Vault body."); + assertEqual(strict.size, manual.size, "The strict natural restart changed the Metadata byte size."); + assertEqual(strict.documentSize, manual.size, "The strict natural restart changed the Chunk byte size."); + assertEqual(strict.adapterSize, manual.size, "The strict natural restart changed the adapter byte size."); + assertEqual(strict.storageSize, manual.size, "The strict natural restart changed the StorageAccess byte size."); + assertEqual( + strict.provenanceRevision, + strict.rev, + "The strict restart did not persist current file provenance." + ); + assertEqual(strict.contentHash, sha256(versions[2]), "The strict restart recorded the wrong content hash."); + assertEqual(strict.contentSize, manual.size, "The strict restart recorded the wrong byte size."); + assertEqual(strict.conflictCount, 0, "The strict natural restart created a conflict branch."); + assertEqual(strict.branchCount, 1, "The strict natural restart changed the branch count."); + assertExactMtime(strict, "strict natural restart"); + + const screenshotSettings = await evaluate<{ strictContentCheckOnStartup: boolean; useAdvancedMode: boolean }>( + [ + "core.settings.strictContentCheckOnStartup=false;", + "core.settings.useAdvancedMode=true;", + "await core.services.setting.saveSettingData();", + "const settings=core.services.setting.currentSettings();", + "return JSON.stringify({strictContentCheckOnStartup:settings.strictContentCheckOnStartup===true,", + "useAdvancedMode:settings.useAdvancedMode===true});", + ].join("\n") + ); + assertEqual( + screenshotSettings.strictContentCheckOnStartup, + false, + "The screenshot fixture did not reset strict content checking to OFF after the startup assertions." + ); + assertEqual( + screenshotSettings.useAdvancedMode, + true, + "The screenshot fixture did not expose the Advanced page." + ); + const settingScreenshots = await captureSettingScreenshots(); + + console.log( + JSON.stringify( + { + ok: true, + bundleSha256: bundleHash, + fixtureFileCount: fileCount, + localOnly: true, + bootstrapPluginStartup: "controlled", + measuredRestartsPluginStartup: "natural", + observations: { + bootstrap: { + indexedCount: initial.indexedCount, + targetProvenanceValid: initial.targetProvenanceValid, + conflictCount: initial.conflictCount, + pendingCount: initial.pendingCount, + }, + defaultOffRestart: { + revisionUnchanged: ordinary.rev === initial.target.rev, + databaseBodyUnchanged: ordinary.dbContent === versions[0], + offlineVaultBodyPreserved: ordinary.vaultContent === versions[1], + branchCount: ordinary.branchCount, + provenanceRevision: ordinary.provenanceRevision, + }, + manualStrictScan: { + detectedOfflineEdit: manual.dbContent === versions[1], + revisionChanged: manual.rev !== ordinary.rev, + branchCount: manual.branchCount, + provenanceRevision: manual.provenanceRevision, + scanMs: manual.scanMs, + }, + strictRestart: { + detectedOfflineEdit: strict.dbContent === versions[2], + revisionChanged: strict.rev !== manual.rev, + branchCount: strict.branchCount, + provenanceRevision: strict.provenanceRevision, + }, + }, + timing: { + scope: "Bootstrap and natural restart timings run from startObsidianLiveSyncSession() invocation through waitForLiveSyncCoreReady() completion. They include Obsidian launch, Vault open, plug-in startup, and runner readiness; they are not isolated scanner timings. manualStrictScanMs measures only the scanVault(true) call inside the running renderer.", + bootstrapMs, + ordinaryStartupMs, + manualStrictScanMs: manual.scanMs, + strictStartupMs, + }, + settingScreenshots, + }, + null, + 2 + ) + ); + } catch (error) { + try { + await captureFailureEvidence(error); + } catch (captureError) { + console.warn( + "Could not capture startup content check failure evidence: " + + (captureError instanceof Error ? captureError.message : String(captureError)) + ); + } + throw error; + } finally { + try { + await stopSession(); + } finally { + await vault.dispose(); + } + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/updates.md b/updates.md index 2a288762..55a40933 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Changed + +- Ordinary start-up can skip reading file content when the exact modification time, size, and recorded device-local provenance match a conflict-free revision, with no pending file processing. + - **Strict content check on startup** keeps full content comparisons enabled. The setting is disabled by default; an edit which preserves its exact size and modification time can go undetected by this start-up scan. Manual scans and explicit recovery operations remain strict. + ## 1.0.41 26th October, 2026 From 51a1f702a3143c4938378675681324f83a20960b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 03:33:57 +0000 Subject: [PATCH 53/68] Prepare 1.0.42 with optional start-up content checks --- devs.md | 8 +++-- docs/releases/1.0-previews.md | 63 ++++++++++++++++++++++++++++++++++- manifest.json | 2 +- package-lock.json | 18 +++++----- package.json | 4 +-- src/apps/cli/package.json | 2 +- src/apps/webapp/package.json | 2 +- src/apps/webpeer/package.json | 2 +- updates.md | 16 ++++----- versions.json | 3 +- 10 files changed, 92 insertions(+), 28 deletions(-) diff --git a/devs.md b/devs.md index 3786afa6..a9279f14 100644 --- a/devs.md +++ b/devs.md @@ -353,6 +353,7 @@ This policy governs plug-in pre-releases and their accompanying CLI images. For - Add only user-facing changes that help users understand what they gain, what has changed, or what they may need to do after updating. - Avoid listing purely internal refactors, maintenance chores, generated-file changes, and dependency updates unless they affect users; group and label them when they are included. - When preparing a release, replace `## Unreleased` with the target version heading (for example, `## 0.25.81`) and add a fresh empty `## Unreleased` section above it for the next cycle. +- For a stable release, review the complete user-facing changes since the previous stable release. Classify new features and changed behaviour relative to that stable release. Include changes from intervening pre-releases which never became stable, even when their original notes have already moved into a release history. - Review and polish the released section in the release PR before tagging, because the content is embedded into the plug-in and may be reused as the GitHub Release notes. - Keep approximately the five most recent published plug-in versions in the embedded `updates.md`. Move older published sections unchanged into the appropriate release-line archive under `docs/releases/`, and update the history references when rotating them. @@ -372,8 +373,9 @@ For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch- - Validate installation or update through BRAT, and cold-start Obsidian to confirm the plug-in loads. Verify ordinary bidirectional synchronisation and relevant regression scenarios in real Obsidian against the published artefacts, or a build with matching hashes. When automated scenarios establish that coverage, BRAT acceptance need only cover installation or update and start-up; manually check any remaining release-specific behaviour. Record the artefact hashes and validation evidence as described in the [real Obsidian test guide](test/e2e-obsidian/README.md#combined-setup-and-security-regression-checks). - After a hyphenated pre-release passes, keep its release pull request unmerged. Add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`, then close the release pull request only through a separate maintainer action. - After a stable version passes, remove the GitHub pre-release designation and make that exact release the latest stable release. Confirm that the release is no longer a pre-release, then mark its release pull request ready, merge the exact release commit into the selected base branch with a merge commit, and integrate that exact commit through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata. Create the stable CLI tag and publish its `latest` and major-minor image tags, if selected, through a separate maintainer gate. -- If BRAT validation fails, keep the release PR in draft and do not move published tags. Before preparing the next version, add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`. Keep only changes made after that tag under `## Unreleased`. Compare the historical section with `git show :updates.md`; do not merge the failed release PR or describe it as validated. The next release PR can then rotate only the correction notes while preserving the immutable release history. -- Prepare and publish the next patch or pre-release version from that reconciled base. Leave the failed release PR draft until it is deliberately closed as superseded under a separate maintainer action. +- If BRAT validation fails, keep the release PR in draft and leave its published tag and artefacts unchanged. Prepare the correction on a new branch descended from that exact release commit, retaining its existing changes and `versions.json` entry. Prepare the next patch or pre-release version on that branch, with its own version, tag, and release PR. The selected base branch remains unchanged until a successor stable release passes validation and is promoted. +- For the successor stable release, consolidate all user-facing changes since the last stable release, including changes from intervening pre-releases which never became stable. Preserve each published pre-release's exact tagged notes in the appropriate history under `docs/releases/`, and compare the archived section with `git show :updates.md`. The new release notes must describe the whole stable upgrade, including the correction. +- After the successor stable release is validated and promoted, merge its exact release commit and ancestry through the reviewed branch chain. Leave the failed release PR in draft until it is deliberately closed as superseded under a separate maintainer action. - A hyphenated version is rejected unless `prerelease=true`. A stable version staged with `prerelease=true` is rejected unless `publish_cli=false`. ### Release Cheat Sheet @@ -403,7 +405,7 @@ For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch- 9. Validate the published release through BRAT, including start-up, ordinary bidirectional synchronisation, and any release-specific regression scenario. 10. After a hyphenated pre-release passes, keep its release PR unmerged, reconcile its `versions.json` entry and exact release-note section into the selected base branch as metadata only, then close the PR through a separate maintainer action. 11. After a stable version passes, remove the pre-release designation and make the exact release the latest stable release. Confirm that promotion, then mark the release PR ready, merge the exact release commit into the selected base branch, and integrate it through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata, then publish stable CLI tags through a separate maintainer gate if selected. -12. If validation fails, leave the PR in draft and do not move the published tags. Reconcile the published version's `updates.md` section and `versions.json` entry into the base branch as metadata only, then prepare the next patch or pre-release version from the remaining `## Unreleased` entries. +12. If validation fails, leave the PR in draft and keep its published tag and artefacts unchanged. Start a new branch from that release commit, add the correction, and prepare the next patch or pre-release version. Retain the published version's `versions.json` entry and exact archived notes, and consolidate the successor stable release notes from the last stable release. Keep the base branch unchanged until the successor stable release is validated and promoted. ### CLI-only patch releases diff --git a/docs/releases/1.0-previews.md b/docs/releases/1.0-previews.md index 2ae2b583..bba8f0bb 100644 --- a/docs/releases/1.0-previews.md +++ b/docs/releases/1.0-previews.md @@ -1,9 +1,70 @@ # 1.0 preview release history -This document records the opt-in beta and release-candidate builds published before 1.0.0. Most users upgrading from 0.25.83 only need the consolidated [1.0.0 release notes](1.0.md#100). +This document records opt-in pre-release builds in the 1.0 release line. Consolidated notes for stable upgrades are available in the [current change log](../../updates.md) and the [1.0 release history](1.0.md). The prepared `1.0.0-rc.0` tag was not published as a plug-in release and is therefore omitted. +Version 1.0.41 was published for validation and remains a GitHub pre-release. Its changes and the subsequent start-up correction are consolidated in the [1.0.42 release notes](../../updates.md#1042). + +## 1.0.41 + +26th October, 2026 + +### Synchronisation + +#### New Feature + +- Rapid edits can produce fewer intermediate versions on other devices while LiveSync mode remains active. + - Enable **Batch database updates in LiveSync mode** to group successive local edits. + - Disabled by default; each synchronisation mode keeps its own choice. + - Updates reach other devices later. Batching does not detect template completion or prevent genuine concurrent-edit conflicts. +- CouchDB synchronisation no longer looks idle while it is comparing change history. + - A ⏊ icon and comparison count show activity before documents are transferred. + - This count represents history entries, not files remaining to synchronise. +- Status counts are easier to follow, with less movement as values change. + - CSS snippets can hide unwanted items or style individual `livesync-stat-` items, including inactive ones. + +#### Fixed + +- Recent Vault edits remain recoverable after an interrupted local database save. + - This includes edits whose modification times compare equal to the database copy, and pending Hidden File Sync and Customisation Sync changes. +- Manual synchronisation no longer starts before already accepted local changes finish saving to the database. + - Settings application also waits for those saves. Failed saves stay pending and report a failure. +- Text from third-party tools is handled more robustly. + - Characters split across Chunks no longer trigger false corruption warnings. +- Start-up integrity checks use less memory for healthy Vault files. (#1283) +- Start-up checks avoid false repair warnings for files that change during inspection. (#1283) + - This applies when the final size matches the database and there are no conflicts. + +### Customisation Sync + +#### Fixed + +- Applying a multi-file customisation no longer skips files after the list is refreshed. (#1271) +- Choosing **Automatic** now explains a missing Hidden File Sync prerequisite instead of proceeding into a failed initialisation. (#1270) + - Enable and initialise Hidden File Sync on this device first. The current mode is kept when the prerequisite is missing. +- Initialising a whole plug-in in **Automatic** mode no longer omits its selected **MAIN** or **DATA** files. + +### Setup tools + +#### Fixed + +- Setup URI generation can avoid shell-variable collisions and unintended use of the operating-system username. (#817, #1274) + - Use `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE` for connection and encryption values. + - Existing scripts remain supported: the previous names are used when the corresponding prefixed value is unset or blank. + +### Miscellaneous + +- Dependencies have been updated, including security fixes. + +### Acknowledgements + +Thank you for your contributions! + +- @rwbru, for the original Setup URI environment-variable fix ([#817](https://github.com/vrtmrz/obsidian-livesync/pull/817)). +- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts ([#1274](https://github.com/vrtmrz/obsidian-livesync/pull/1274)). +- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib ([#1283](https://github.com/vrtmrz/obsidian-livesync/pull/1283), [Commonlib #184](https://github.com/vrtmrz/livesync-commonlib/pull/184)). + ## 1.0.0-rc.1 27th July, 2026 diff --git a/manifest.json b/manifest.json index efe56799..e1929275 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.41", + "version": "1.0.42", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index 5393a72f..717ba6f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.41", + "version": "1.0.42", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.41", + "version": "1.0.42", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.1", + "@vrtmrz/livesync-commonlib": "0.2.2", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", @@ -4103,9 +4103,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.1.tgz", - "integrity": "sha512-fnQb2FcEG+NqIQX9KhTstea4B7C4VAL0g9ry5rp41qnJDuLaR5zfCz995L+l3Khyrp/9kcPw4VX5RUmV+8GVog==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.2.tgz", + "integrity": "sha512-+nl4ntmRtUmSUuh9mOpicE0U/wpzhSPSQQXy9eg/2xdTBL58jkrQ6bQjbVKEhqUj0bUvg0ESjZm8YX/x+4zXiA==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -11369,7 +11369,7 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.41-cli", + "version": "1.0.42-cli", "dependencies": { "chokidar": "^4.0.0", "minimatch": "^10.2.5", @@ -11394,7 +11394,7 @@ }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.41-webapp", + "version": "1.0.42-webapp", "dependencies": { "octagonal-wheels": "^0.1.55" }, @@ -11406,7 +11406,7 @@ } }, "src/apps/webpeer": { - "version": "1.0.41-webpeer", + "version": "1.0.42-webpeer", "dependencies": { "octagonal-wheels": "^0.1.55" }, diff --git a/package.json b/package.json index 076bed16..84e21ce5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.41", + "version": "1.0.42", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", @@ -204,7 +204,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.1", + "@vrtmrz/livesync-commonlib": "0.2.2", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 7f73f7f1..43e21f1c 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.41-cli", + "version": "1.0.42-cli", "main": "dist/index.cjs", "type": "module", "scripts": { diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index d0f4f5ce..ccda9b89 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.41-webapp", + "version": "1.0.42-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index c7654ed5..c9047af9 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.41-webpeer", + "version": "1.0.42-webpeer", "type": "module", "scripts": { "dev": "vite", diff --git a/updates.md b/updates.md index 55a40933..0516dada 100644 --- a/updates.md +++ b/updates.md @@ -12,16 +12,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased -### Synchronisation +## 1.0.42 -#### Changed +27th October, 2026 -- Ordinary start-up can skip reading file content when the exact modification time, size, and recorded device-local provenance match a conflict-free revision, with no pending file processing. - - **Strict content check on startup** keeps full content comparisons enabled. The setting is disabled by default; an edit which preserves its exact size and modification time can go undetected by this start-up scan. Manual scans and explicit recovery operations remain strict. - -## 1.0.41 - -26th October, 2026 +Making content checks mandatory in the 1.0.41 pre-release came with a little too much overhead, so 1.0.42 makes them optional. This has turned into a slightly larger release. ### Synchronisation @@ -37,6 +32,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi - Status counts are easier to follow, with less movement as values change. - CSS snippets can hide unwanted items or style individual `livesync-stat-` items, including inactive ones. +- We can choose stricter content checks at start-up. + - Enable **Strict content check on startup** to compare file contents even when their modification times and sizes match the recorded local database state. This can increase start-up time in large Vaults. + - The setting is disabled by default. Ordinary start-up can skip content reads when the exact modification time, file size, and recorded database revision match, with no conflicts or pending file changes. + - While the setting is disabled, an edit which preserves its exact size and modification time can go undetected during start-up. Manual scans and explicit recovery operations remain strict. + #### Fixed - Recent Vault edits remain recoverable after an interrupted local database save. diff --git a/versions.json b/versions.json index a68f2a1a..8ad7a151 100644 --- a/versions.json +++ b/versions.json @@ -52,5 +52,6 @@ "1.0.38": "1.7.2", "1.0.39": "1.7.2", "1.0.40": "1.7.2", - "1.0.41": "1.7.2" + "1.0.41": "1.7.2", + "1.0.42": "1.7.2" } From efa3e356b4f529a929df00ac24b1fb62e8ff2180 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 04:11:11 +0000 Subject: [PATCH 54/68] Use plain pull request references in 1.0.42 notes --- updates.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/updates.md b/updates.md index 0516dada..f5551d3b 100644 --- a/updates.md +++ b/updates.md @@ -74,9 +74,9 @@ Making content checks mandatory in the 1.0.41 pre-release came with a little too Thank you for your contributions! -- @rwbru, for the original Setup URI environment-variable fix ([#817](https://github.com/vrtmrz/obsidian-livesync/pull/817)). -- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts ([#1274](https://github.com/vrtmrz/obsidian-livesync/pull/1274)). -- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib ([#1283](https://github.com/vrtmrz/obsidian-livesync/pull/1283), [Commonlib #184](https://github.com/vrtmrz/livesync-commonlib/pull/184)). +- @rwbru, for the original Setup URI environment-variable fix (#817). +- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts (#1274). +- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib (#1283, Commonlib #184). ## 1.0.40 From 7d98f0c084239f909774524c9b9fe6d8ac2e9d90 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 11:11:15 +0000 Subject: [PATCH 55/68] Document release notes and recovery assessment --- AGENTS.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 6b93bc33..9e9529ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,13 @@ Always adhere to the following stylistic and spelling rules: --- +## Release Notes + +- Follow the release-note scope and history rules in [devs.md](devs.md#release-notes). Structure substantive entries as the user-visible outcome, the method that produces it, and supporting conditions, defaults, limitations, or required actions. Put the outcome first and use subordinate items when the explanation needs them. +- When a pull request explains only its implementation, consult the relevant issue or design decision to establish the problem it resolves. Verify the claimed outcome against the change and its evidence. Distinguish removing a cause from reducing occurrence, detecting a problem, or improving recovery. + +--- + ## Technical & Architecture Rules 1. **Database Structure**: @@ -56,6 +63,15 @@ Always adhere to the following stylistic and spelling rules: --- +## Failure Assessment and Recovery + +- Establish the intended behaviour from [the conflict policy](devs.md#conflict-merge-policy) and [the revision-provenance contract](docs/specs_conflict_resolution.md) before classifying a failure. A visible conflict which preserves both versions can be the correct outcome; manual selection alone does not establish a serious defect. +- Assess preservation of user edits and required revision ancestry separately from extra work, temporary synchronisation delays, or duplicate internal records. A successful automatic merge or an empty queue is not proof of correct recovery. Check the resulting contents and history, safe convergence, and behaviour after restart where applicable. +- Distinguish data still present on a device or in readable history from irretrievable loss. Verify that a recovery source contains the affected edits before treating the failure as fully recoverable. Follow [the recovery guidance](docs/recovery.md) before recommending a rebuild or reset which could discard unique work. +- Give demonstrated irreversible information loss particular weight. Describe the actual burden of safe manual conflict resolution without treating automation as a correctness requirement. If preservation, recovery, impact, or the appropriate response remains unclear, review the evidence and alternatives with the collaborating user before changing the design or accepting the risk; continue independent investigation. + +--- + ## Development & Verification Commands Before submitting code, you should run verification scripts locally to ensure correct syntax and function. From 2cfa4268cce1af9c3f2aaf51654ffe496fc80d73 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 13:14:38 +0000 Subject: [PATCH 56/68] Document published features and current technical behaviour --- docker/coturn/README.md | 2 +- .../2026_08_p2p_transport_compatibility.md | 32 ++- ...local_file_save_completion_and_batching.md | 30 ++- docs/adr/2026_10_unicode_path_identity.md | 7 +- .../design_docs/configurable_id_derivation.md | 10 +- .../document_history_revision_restoration.md | 45 ++-- .../internal_metadata_encryption.md | 15 +- .../path_component_length_compatibility.md | 8 +- .../design_docs/renewable_turn_credentials.md | 5 +- docs/design_docs/time_bound_setup_uri.md | 200 +++++++++--------- docs/design_docs/tweak_compatibility.md | 11 +- docs/design_docs/unicode_path_identity.md | 8 +- docs/design_docs_of_journalsync.md | 4 +- docs/design_docs_of_remote_configurations.md | 21 +- docs/design_docs_of_sharing_tweak_value.md | 4 +- docs/review_harness.md | 5 +- docs/specs_data_compression.md | 6 +- src/apps/cli/testdeno/test_dev_deno.md | 6 +- src/apps/webpeer/README.md | 2 + 19 files changed, 241 insertions(+), 180 deletions(-) diff --git a/docker/coturn/README.md b/docker/coturn/README.md index f62bb088..33642f03 100644 --- a/docker/coturn/README.md +++ b/docker/coturn/README.md @@ -51,7 +51,7 @@ Enter both client paths in the P2P profile's TURN server list: turn:turn.example.com:3478?transport=udp,turn:turn.example.com:3478?transport=tcp ``` -Use `TURN_USERNAME` and `TURN_PASSWORD` as the TURN username and credential. Keep the normal `Automatic` ICE policy unless a future LiveSync release offers `TURN relay only` and the direct path needs to be excluded deliberately. +Use `TURN_USERNAME` and `TURN_PASSWORD` as the TURN username and credential. Keep the normal `Automatic` connection path unless the direct path needs to be excluded deliberately. Select `TURN relay only` under `Connection compatibility` to require the configured TURN route. Both synchronising devices must be able to reach the server. Prove an explicit two-way `Replicate now` round trip on the intended networks before relying on the configuration. diff --git a/docs/adr/2026_08_p2p_transport_compatibility.md b/docs/adr/2026_08_p2p_transport_compatibility.md index 48ef540b..1045e6f4 100644 --- a/docs/adr/2026_08_p2p_transport_compatibility.md +++ b/docs/adr/2026_08_p2p_transport_compatibility.md @@ -2,7 +2,10 @@ ## Status -Accepted — the user-facing controls will be introduced in stages. This record defines their boundaries before Commonlib settings and LiveSync interfaces are changed. +Accepted. The P2P message-size presets and connection-path choice are present +in current Self-hosted LiveSync 1.0.42 with Commonlib 0.2.2. The proposed +`Test TURN server` action and concise route diagnostics in the Obsidian +interface remain future work. ## Context @@ -11,10 +14,12 @@ WebRTC connectivity depends on both devices, their browsers or embedded WebViews The current P2P transport has several relevant properties: - Trystero supplies the Nostr signalling strategy and the browser-owned WebRTC connection. -- Commonlib limits one RPC wire payload to 15,360 bytes so it remains below Trystero's own action-chunk boundary. -- Trystero supplies ordinary STUN servers and accepts an optional TURN server list with one username and credential. -- ICE chooses a direct, server-reflexive, or TURN-relayed path automatically. -- Commonlib can collect raw WebRTC statistics, but LiveSync does not yet present the selected candidate route in a concise diagnostic result. +- Commonlib defaults to a 15,360-byte RPC wire payload limit, below Trystero's own action-chunk boundary; the presets below can lower it. +- Manual TURN settings use one URL list, username, and credential. Managed preparation can supply multiple ICE server entries with their own credentials. +- ICE chooses a direct, server-reflexive, or TURN-relayed path automatically unless the profile selects `TURN relay only`. +- Commonlib can collect raw WebRTC statistics. The CLI reports the selected + candidate route, but the Obsidian interface does not yet present it in a + concise diagnostic result. Issue reports suggest that reducing the application payload may improve some mobile and constrained-network paths. A VPN such as Tailscale may also turn an unreliable route into a reliable one. These observations are consistent with NAT, path-MTU, fragmentation, or intermediary behaviour, but they do not prove one universal cause. Browser WebRTC implementations retain responsibility for SCTP, DTLS, ICE, packetisation, congestion control, and retransmission. @@ -24,7 +29,7 @@ One low-level number cannot represent all of these concerns. Users need a small ### Message-size presets -LiveSync will expose a `P2P message size` choice with four presets: +LiveSync exposes a `P2P message size` choice with four presets: | Label | Maximum RPC wire payload | Intended use | | ----------------------- | -----------------------: | ------------------------------------------------------------------------------- | @@ -37,11 +42,11 @@ This value limits Commonlib RPC wire payloads before Trystero applies its own fr The bound applies to outgoing messages. A device which only lowers its own value still receives messages produced under the sender's value. The selected preset therefore belongs to the P2P profile and is included in an encrypted Setup URI for additional devices. A device which was configured earlier must be changed separately; the interface and troubleshooting guidance must state that the same conservative preset should be selected on every participating device. An absent key preserves the current 15,360-byte default. -Automatic negotiation or fallback between presets is deferred. A failed ordered data channel may require connection replacement before a smaller retry can prove anything, and changing transport parameters during a replication session would broaden the lifecycle contract considerably. The first implementation remains explicit, stable for one room lifetime, and inspectable. +Automatic negotiation or fallback between presets is deferred. A failed ordered data channel may require connection replacement before a smaller retry can prove anything, and changing transport parameters during a replication session would broaden the lifecycle contract considerably. The implementation remains explicit, stable for one room lifetime, and inspectable. ### Connection path -LiveSync will expose a separate `Connection path` choice: +LiveSync exposes a separate `Connection path` choice: - `Automatic` retains normal ICE selection and is the default. - `TURN relay only` supplies `iceTransportPolicy: 'relay'` and prevents direct or server-reflexive candidates from being selected. @@ -81,11 +86,15 @@ Where the browser exposes the evidence, the result should report: - UDP, TCP, or TLS transport; and - a bounded failure or inconclusive result. -Ordinary P2P diagnostics should later summarise the selected candidate pair as direct, server-reflexive, or relayed, with its transport. Raw `getStats()` output remains supporting evidence rather than the primary interface. +The proposed Obsidian P2P diagnostics should summarise the selected candidate +pair as direct, server-reflexive, or relayed, with its transport. The CLI's +candidate-route output does not provide this user-facing summary. Raw +`getStats()` output remains supporting evidence rather than the primary +interface. ### Placement and defaults -These controls belong inside `P2P Configuration` under a `Connection compatibility` section. They do not require the repository-wide Advanced, Power User, or Edge Case modes. P2P itself remains a supported opt-in feature. +These controls appear inside `P2P Configuration` under a `Connection compatibility` section. They do not require the repository-wide Advanced, Power User, or Edge Case modes. P2P itself remains a supported opt-in feature. Existing profiles retain the following defaults: @@ -161,7 +170,8 @@ A layer-4 TLS router could share one public address between distinct CouchDB and ## Verification -The first implementation stage must add focused tests before production changes: +The following list records the original focused-test plan for the message-size +and connection-path controls. It remains useful as regression coverage: - settings-schema defaults for absent keys; - P2P connection-string and Setup URI round trips which retain both transport compatibility settings; diff --git a/docs/adr/2026_10_local_file_save_completion_and_batching.md b/docs/adr/2026_10_local_file_save_completion_and_batching.md index bbf67b13..f100d2a6 100644 --- a/docs/adr/2026_10_local_file_save_completion_and_batching.md +++ b/docs/adr/2026_10_local_file_save_completion_and_batching.md @@ -2,15 +2,17 @@ date: 2026-10-24 commonlib-version: "0.2.0" self-hosted-livesync-version: "1.0.40" -status: accepted-unreleased +status: accepted --- # Architectural Decision Record: Local File Save Completion and Batching ## Status -Accepted on 2026-10-25. The package versions above identify the investigated -baseline, rather than a released implementation of this decision. +Accepted on 2026-10-25. The recovery contracts and LiveSync mode batching in +implementation-order steps 1–3 are present in current Self-hosted LiveSync +1.0.42 with Commonlib 0.2.2. The version fields above retain the investigated +baseline. Further same-path coalescing in step 4 remains a follow-up. ## Context @@ -349,6 +351,9 @@ eliminating genuine concurrent-edit conflicts or as a confirmed fix for #1207. ## Ownership and implementation order +The list below records the implementation sequence. Steps 1–3 are published +in the current version line; the further coalescing in step 4 remains planned. + 1. **Commonlib recovery:** event ownership, ordered snapshot persistence, successful completion and failure propagation, finite flush boundaries, and same-time content reconciliation. Add focused failing regressions before @@ -373,6 +378,11 @@ automation transaction API require a further design decision. ## Verification required +The matrix below records the acceptance criteria for implementation steps 1–3 +and remains useful for regression work. Publication in the current version line +does not extend the historical test results above to untested platforms or +automation workflows. + Use controlled promises at the manager and persistence boundaries for focused regressions, and real PouchDB tests for content and ancestry. The acceptance matrix is: @@ -395,13 +405,13 @@ matrix is: | Old settings, mode changes, explicit opt-in, and settings import | Existing LiveSync mode users retain immediate saving until they choose batching. | | Incoming identical content, an edit, a deletion, or delete/recreate while waiting | Preserve genuine branches without extra conflicts or loss of local work. | -Keep the existing E2E comparison between batching disabled, an ignored legacy -setting, and explicitly enabled batching. Repeat both directions, manual -synchronisation, saved-snapshot restart, and controlled in-progress restart with -the fix. Record the original failure separately from repaired results; a pass -with a different timestamp bucket or a test-only bundle is not verification of -the production fix. Run platform-specific lifecycle checks before extending the -evidence to Windows or mobile. +The original E2E plan compared batching disabled, an ignored legacy setting, and +explicitly enabled batching in both directions, with manual synchronisation, +saved-snapshot restart, and controlled in-progress restart. Record the original +failure separately from repaired results; a pass with a different timestamp +bucket or a test-only bundle is not verification of the production fix. Keep +platform-specific lifecycle evidence bounded to the tested environments before +extending conclusions to Windows or mobile. ## Alternatives and consequences diff --git a/docs/adr/2026_10_unicode_path_identity.md b/docs/adr/2026_10_unicode_path_identity.md index 839ea986..2bc5a036 100644 --- a/docs/adr/2026_10_unicode_path_identity.md +++ b/docs/adr/2026_10_unicode_path_identity.md @@ -2,15 +2,16 @@ date: 2026-10-17 commonlib-version: "0.1.42" self-hosted-livesync-version: "1.0.37" -status: unreleased +status: released --- # Architectural Decision Record: Unicode Path Identity for CouchDB ## Status -Accepted on 2026-10-17. The feature remains unreleased. Local validation -evidence and platform limits are recorded in the design document. +Accepted on 2026-10-17. Unicode NFC path identity was released in Self-hosted +LiveSync 1.0.38 with Commonlib 0.1.42. Local validation evidence and platform +limits are recorded in the design document. ## Context diff --git a/docs/design_docs/configurable_id_derivation.md b/docs/design_docs/configurable_id_derivation.md index 8a88ce5f..00bc1d80 100644 --- a/docs/design_docs/configurable_id_derivation.md +++ b/docs/design_docs/configurable_id_derivation.md @@ -2,7 +2,7 @@ date: 2026-09-29 commonlib-version: "0.1.33" self-hosted-livesync-version: "1.0.32" -status: unreleased +status: released --- # Configurable ID derivation @@ -21,12 +21,12 @@ including its passphrase-persistence correction at commit the `next` tag, provides the construction described here. It replaces the independent Chunk algorithm from the `0.1.32` prerelease without a compatibility branch or a new settings version. Legacy ID generation remains -unchanged. LiveSync pins the published `0.1.33` package and its registry -integrity in the lockfile. +unchanged. At that implementation baseline, LiveSync pinned the published +`0.1.33` package and its registry integrity in the lockfile. Its [Internal Metadata encryption design](https://github.com/vrtmrz/obsidian-livesync/blob/126d6eadb858a79a08ad7f600061e54fc8d31196/docs/design_docs/internal_metadata_encryption.md) remains the basis for Properties encryption and CouchDB feature admission. -This document records an unreleased LiveSync feature. It does not select a -plug-in release version. +Independent ID derivation shipped in Self-hosted LiveSync 1.0.33. This document +retains the design and package baseline recorded above. ## Feasibility diff --git a/docs/design_docs/document_history_revision_restoration.md b/docs/design_docs/document_history_revision_restoration.md index c064daf6..714c4944 100644 --- a/docs/design_docs/document_history_revision_restoration.md +++ b/docs/design_docs/document_history_revision_restoration.md @@ -10,40 +10,45 @@ This document uses the independent revision properties defined under [Revision](../glossary.md#revision) and the general state model in [Conflict resolution and revision provenance](../specs_conflict_resolution.md). -Document History can reconstruct an available historical revision from its -Chunks and write that content to the Vault through **Back to this revision**. -The current action writes directly through the storage adapter. It does not -create a new Metadata revision, clear a logical deletion through a successor, -or record which database revision produced the restored Vault file. +Before the current restoration implementation, Document History could +reconstruct an available historical revision from its Chunks and write that +content to the Vault through **Back to this revision**. That action wrote +directly through the storage adapter. It did not create a new Metadata revision, +clear a logical deletion through a successor, or record which database revision +produced the restored Vault file. -This leaves a logically deleted Metadata document at `deleted: true` after the -file has returned to the Vault. A later ordinary Vault save may create a -non-deleted successor, but restoration must not depend on an unrelated later -file event. +That left a logically deleted Metadata document at `deleted: true` after the +file had returned to the Vault. A later ordinary Vault save could create a +non-deleted successor, but restoration needed to succeed without depending on +an unrelated later file event. -The same action does not inspect or preserve revision-tree intent explicitly -when conflicts exist. Document History currently displays the ancestry of the -PouchDB winner. It does not display the complete revision tree or the ancestry -of every conflict leaf. +Restoration does not resolve conflicts or provide a branch-aware history view. +Document History displays the ancestry of the PouchDB winner; it does not +display the complete revision tree or the ancestry of every conflict leaf. This design covers restoration of one readable historical revision of a normal Vault file. It creates a new non-deleted successor revision, reflects that exact revision to the Vault, and leaves every other conflict branch available for the existing conflict workflow. +The current **Back to this revision** implementation writes that successor +revision, reflects the exact created revision to the Vault, and records it as +device-local provenance. The storage-only behaviour described above is the +pre-fix history. + ## Evidence -A real-Obsidian exercise created a Markdown document, removed it through the -Vault API, and waited for LiveSync to store a logical-deletion successor. The -Metadata retained all referenced Chunks, and Document History could reconstruct -the deleted content. +A real-Obsidian exercise before the current restoration implementation created +a Markdown document, removed it through the Vault API, and waited for LiveSync +to store a logical-deletion successor. The Metadata retained all referenced +Chunks, and Document History could reconstruct the deleted content. Selecting **Back to this revision** restored the file and its exact bytes to the Vault. The local database nevertheless retained the same deleted current revision after file processing had settled. An additional ordinary Vault save -then created a new non-deleted successor. This demonstrates that content -reconstruction works and that the missing operation is the database-aware -restoration step. +then created a new non-deleted successor. This historical reproduction showed +that content reconstruction worked while the database-aware restoration step +was missing. ## Revision-tree decision diff --git a/docs/design_docs/internal_metadata_encryption.md b/docs/design_docs/internal_metadata_encryption.md index e7d16ec6..0392dbdf 100644 --- a/docs/design_docs/internal_metadata_encryption.md +++ b/docs/design_docs/internal_metadata_encryption.md @@ -2,14 +2,15 @@ date: 2026-09-27 commonlib-version: "0.1.30" self-hosted-livesync-version: "1.0.32" -status: unreleased +status: released --- # Internal Metadata encryption and remote feature changes This document defines the LiveSync integration of Commonlib's remote feature contract and encrypted Metadata for Hidden File Sync and Customisation Sync. -It describes unreleased behaviour being implemented in this branch. +The implementation shipped in Self-hosted LiveSync 1.0.33; this document +retains its original design and implementation context. Commonlib's companion `docs/remote-feature-compatibility.md` is the source of truth for the wire document, identifiers, validation, and shared @@ -159,11 +160,11 @@ after alignment. With the preference subsequently OFF, verify that Fast Fetch still decodes encrypted Metadata and preserves the remote feature declaration. Keep the primary-language settings and troubleshooting guides, the -database-compatibility ADR, and Unreleased notes aligned with this behaviour. -Keep the detailed shared protocol in Commonlib and link to it after publication; -do not maintain another copy of its wire schema here. Translations are a separate -change. Update tested-version evidence when the implementation and its -validation have been accepted. +database-compatibility ADR, and release notes consistent with this behaviour. +Commonlib's companion document remains the source of truth for the shared +protocol; this document does not repeat its wire schema. Translations remain a +separate change. The version fields above retain the original design baseline; +add later tested-version evidence only when validation is recorded. Related application contracts: [Replicator architecture](replicator_architecture.md), [Tweak compatibility](tweak_compatibility.md), and diff --git a/docs/design_docs/path_component_length_compatibility.md b/docs/design_docs/path_component_length_compatibility.md index 367d7d42..ed97a9c5 100644 --- a/docs/design_docs/path_component_length_compatibility.md +++ b/docs/design_docs/path_component_length_compatibility.md @@ -2,7 +2,7 @@ date: 2026-09-04 commonlib-version: "0.1.21" self-hosted-livesync-version: "1.0.24" -status: unreleased +status: released --- # Path component length compatibility @@ -11,9 +11,9 @@ status: unreleased File systems place limits on each file or folder name, rather than applying one common limit to an entire Vault-relative path. Those limits are also expressed -in different units. Self-hosted LiveSync therefore treats 255 UTF-8 bytes as a -focused Android and Linux compatibility warning, not as a universal definition -of a valid path. +in different units. The warning shipped in Self-hosted LiveSync 1.0.26. It +treats 255 UTF-8 bytes as a focused Android and Linux compatibility warning, +not as a universal definition of a valid path. ## Basis for the 255-byte warning diff --git a/docs/design_docs/renewable_turn_credentials.md b/docs/design_docs/renewable_turn_credentials.md index 0f903ba6..b5ae5788 100644 --- a/docs/design_docs/renewable_turn_credentials.md +++ b/docs/design_docs/renewable_turn_credentials.md @@ -2,7 +2,7 @@ date: 2026-09-16 commonlib-version: "0.1.25" self-hosted-livesync-version: "1.0.28" -status: unreleased +status: released --- # TURN credentials in P2P connection settings @@ -14,6 +14,9 @@ by acquiring temporary TURN credentials on the device before opening a P2P room. The [P2P transport compatibility ADR](../adr/2026_08_p2p_transport_compatibility.md) records the connection and persistence policy. +Managed Cloudflare TURN credentials shipped in Self-hosted LiveSync 1.0.29. +This document records the design and its remaining runtime verification limits. + LiveSync prepares a connection copy of `P2PSyncSetting`. Commonlib owns the room lifecycle and consumes the resulting ICE settings. Service-specific HTTP and validation remain under `src/integrations/`; Commonlib has no provider catalogue diff --git a/docs/design_docs/time_bound_setup_uri.md b/docs/design_docs/time_bound_setup_uri.md index bf5ceef6..2c2d3231 100644 --- a/docs/design_docs/time_bound_setup_uri.md +++ b/docs/design_docs/time_bound_setup_uri.md @@ -3,40 +3,42 @@ date: 2026-09-28 commonlib-version: "0.1.33" feasibility-probe-version: "0.1.27" self-hosted-livesync-version: "1.0.32" -status: implementing +status: released --- # Time-bound Setup URIs ## Purpose and status -Offer two modes when generating a [Setup URI](../glossary.md): +The published Obsidian generator offers two choices when creating a [Setup URI](../glossary.md): -- **Ephemeral**, selected by default, derives an effective passphrase from the - entered passphrase and the current fixed time window. -- **Persistent** uses the entered passphrase and the existing URI format - unchanged. It also serves as the compatibility mode for older readers. +- **Time-bound** (the `ephemeral` derivation mode), selected by default, + derives an effective passphrase from the entered passphrase and the current + fixed time window. +- **Compatible (no time limit)** (the `persistent` derivation mode) uses the + entered passphrase and the existing URI format unchanged. It also serves as + the compatibility mode for older readers. -The generation mode choice shows the exact time until which an Ephemeral URI -can be opened through the ordinary reader. Import still requires only the URI and -the entered passphrase. Neither the mode nor a timestamp is stored in the URI. +The Time-bound choice shows the exact time until which the URI can be opened +through the ordinary reader. Import still requires only the URI and the entered +passphrase. Neither the mode nor a timestamp is stored in the URI. -The design is technically feasible with the existing encryption primitives. -An executable protocol probe passes 22 cases against Commonlib 0.1.27 and -octagonal-wheels 0.1.54. The implementation was first published as Commonlib 0.1.32 -on the npm `next` tag. LiveSync now pins Commonlib 0.1.33 from that tag in its npm -dependency and independent Deno tool imports. A mobile performance bound remains -outstanding. +Both URI availability choices shipped in Self-hosted LiveSync 1.0.33. The +initial executable protocol probe passed 22 cases against Commonlib 0.1.27 and +octagonal-wheels 0.1.54. The implementation was first published as Commonlib +0.1.32 on the npm `next` tag. Current Self-hosted LiveSync 1.0.42 uses Commonlib +0.2.2. A mobile performance bound remains outstanding. -This document records the proposed key derivation and integration contract. The -companion probe demonstrates them without changing an application entry point. +This document records the protocol and the integration plan used for the +published feature. The companion probe remains a standalone design check; it +does not exercise an application entry point. ## Scope -The first version offers Ephemeral and Persistent only. Ephemeral uses one -fixed seven-day window shared by every implementation. Arbitrary start dates, -custom end dates, selectable window lengths, and rolling seven-day lifetimes -are outside this version. +The published protocol offers the Ephemeral and Persistent modes only. +Ephemeral uses one fixed seven-day window shared by every implementation. +Arbitrary start dates, custom end dates, selectable window lengths, and rolling +seven-day lifetimes are outside this version. Keep the current settings filtering, remote profiles, main and P2P selections, and short/full export variants. Time binding affects opening the exported @@ -255,15 +257,16 @@ separately shared message. ## Ownership and compatibility Commonlib owns preparation of exported settings, the time-window calculation, -passphrase derivation, format parsing, and dual-candidate decoding. Make these -changes in the Commonlib repository. LiveSync must consume a validated packed -artefact and then an exact reviewed package version, following +passphrase derivation, format parsing, and dual-candidate decoding. These +changes were made in the Commonlib repository. LiveSync consumes the reviewed +Commonlib package, following [the Commonlib dependency workflow](../../devs.md#commonlib-dependency). -Preserve the existing positional encoder API. Its third and fourth arguments -already mean properties to remove and whether to omit default values. Add a -separate encoder, provisionally `encodeTimeBoundSetupURI`, with an options -object containing the mode and the existing export options. Its result is: +The implementation preserves the existing positional encoder API. Its third +and fourth arguments retain their meanings: properties to remove and whether +to omit default values. It adds a separate `encodeTimeBoundSetupURI` encoder +with an options object containing the mode and the existing export options. +Its result is: ```typescript type TimeBoundSetupURIResult = { @@ -272,27 +275,27 @@ type TimeBoundSetupURIResult = { }; ``` -The new encoder defaults to Ephemeral. Preserve the old encoder's legacy -behaviour for callers which have not migrated. Extend -`decodeSettingsFromSetupURI` to try both candidates for `%$` while retaining its +The new encoder defaults to Ephemeral. The old encoder retains its legacy +behaviour for callers which have not migrated. The updated +`decodeSettingsFromSetupURI` tries both candidates for `%$` while retaining its settings return contract and historical format handling. Persistent generation -through the new API delegates to the old encoder. Time injection belongs in +through the new API delegates to the old encoder. Time injection remains in internal test seams, not an import option exposed to users. -| Surface | Required work | +| Surface | Published integration | | ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Obsidian generation](../../src/serviceFeatures/setupObsidian/setupUri.ts) | Reuse the passphrase and copy dialogues, add a mode choice with the exact current window end, and use the new encoder for all three copy variants and the copy event. Preserve each variant's export filters. | -| [Obsidian import](../../src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte) | Consume the updated decoder and retain one authentication failure message. The existing prefix check indicates URI shape, not successful authentication. | -| [Protocol handler](../../src/serviceFeatures/setupObsidian/setupProtocol.ts) | Keep the existing `settings` reconstruction; test encoded delimiters for both modes. | -| [CLI setup](../../src/apps/cli/commands/runCommand.ts) | Consume the updated decoder and leave settings unchanged after any rejection. | -| [WebPeer generator](../../src/apps/webpeer/src/P2PCheckSetup.ts) | Choose a mode explicitly and return availability to its result UI; keep Ephemeral as the default. | -| [Setup utility](../../utils/setup/generate_setup_uri.ts) | Add an explicit mode input, default to Ephemeral, and return/print the end time. Persistent is an explicit choice for durable provisioning output. | -| [Setup utility package facade](../../utils/setup/livesync-commonlib.ts) | Update its independent Commonlib pin and associated lockfiles; updating the root npm dependency alone is insufficient. | +| [Obsidian generation](../../src/serviceFeatures/setupObsidian/setupUri.ts) | Reuses the passphrase and copy dialogues, presents the mode choice and exact current window end, and uses the encoder for all copy variants and the copy event while preserving each variant's export filters. | +| [Obsidian import](../../src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte) | Consumes the updated decoder and retains one authentication failure message. The existing prefix check indicates URI shape, not successful authentication. | +| [Protocol handler](../../src/serviceFeatures/setupObsidian/setupProtocol.ts) | Retains the existing `settings` reconstruction; encoded delimiters remain part of protocol validation for both modes. | +| [CLI setup](../../src/apps/cli/commands/runCommand.ts) | Uses the updated decoder and leaves settings unchanged after a rejection. | +| [WebPeer generator](../../src/apps/webpeer/src/P2PCheckSetup.ts) | Explicitly selects Time-bound mode and returns availability to its result UI; this generator does not offer a Persistent choice. | +| [Setup utility](../../utils/setup/generate_setup_uri.ts) | Accepts an explicit mode, defaults to Time-bound, and returns or prints the end time. Persistent is selected for durable provisioning output. | +| [Setup utility package facade](../../utils/setup/livesync-commonlib.ts) | Retains its independent Commonlib pin and associated lockfiles, separate from the root npm dependency. | -The Fly.io wrapper delegates to the setup utility and should inherit the same -contract. All maintained generators must migrate explicitly before this -feature is described as the default across applications. Non-interactive -automation which needs a durable URI must select Persistent. +The Fly.io wrapper delegates to the setup utility and inherits the same +contract. The initial plan required each maintained generator to select a mode +explicitly. Non-interactive automation which needs a durable URI selects +Persistent. WebPeer hides an Ephemeral URI and disables its copy and additional-device actions after the window ends. Its connection monitor remains available for a @@ -309,8 +312,9 @@ The assessed, unchanged Commonlib 0.1.27 decoder opens Persistent with the entered passphrase and rejects Ephemeral with that same passphrase. Persistent compatibility means compatibility with readers which already support the ordinary generated `%$` representation; it does not add that representation -to still older readers. Historical supported versions and their settings -schemas still need compatibility fixtures before rollout. +to still older readers. This design does not establish compatibility with every +historical reader or settings schema; compatibility fixtures are needed before +making broader claims about them. Keep the original raw-passphrase behaviour of existing URIs. They are indistinguishable from newly generated Persistent URIs to the updated reader. @@ -358,68 +362,72 @@ All 22 cases passed: and raw-passphrase candidates; - the deliberate limitation that clock rollback reproduces an old key. -This demonstrates the proposed mechanism and baseline interoperability, not -completed application support, formal mode indistinguishability, or a mobile -latency bound. The present encryption API may perform two PBKDF2 derivations -on a cold import. Do not infer production performance from this small probe, -whose operations can reuse the dependency's in-memory key cache. +The probe demonstrates feasibility and baseline interoperability. Application +integration is recorded above; the probe does not establish formal mode +indistinguishability or a mobile latency bound. The encryption API may perform +two PBKDF2 derivations on a cold import. Do not infer production performance +from this small probe, whose operations can reuse the dependency's in-memory +key cache. -Implementation validation must cover Commonlib unit and packed-package tests, -plus shared fixed-input protocol vectors across consumers. Add targeted LiveSync -tests for the generation metadata, cancellation, a window change during mode -selection, encoded deep links, and rejected imports without settings writes. Extend CLI, WebPeer, -and Deno setup-tool round trips, including their Persistent selection. +The original implementation validation plan covered Commonlib unit and +packed-package tests, plus shared fixed-input protocol vectors across consumers. +It called for targeted LiveSync tests of generation metadata, cancellation, a +window change during mode selection, encoded deep links, and rejected imports +without settings writes, alongside CLI, WebPeer, and Deno setup-tool round trips +for both modes. -Then run the required LiveSync checks, unit suite, and production build, one -broad process at a time with `NODE_OPTIONS=--max-old-space-size=3072` and bounded -test workers. Use the existing focused real-Obsidian Setup URI workflow for -copy/paste, deep-link import, cancellation, and successful setup. Verify -selection-boundary handling with an injected test clock and mobile behaviour on a -supported runtime; do not change the host's system clock. No remote database -service is needed for the protocol tests themselves. +The plan also called for the LiveSync checks, unit suite, and production build, +one broad process at a time with `NODE_OPTIONS=--max-old-space-size=3072` and +bounded test workers. It used the focused real-Obsidian Setup URI workflow for +copy/paste, deep-link import, cancellation, and successful setup. Selection- +boundary handling was to use an injected test clock; mobile behaviour requires +a supported runtime and must not change the host's system clock. Protocol tests +do not require a remote database service. -There is no identified blocker in the Time-bound URI protocol. The Commonlib -candidate based on 0.1.31 passed its type, unit, boundary, release-process, -and packed-package gates before the 0.1.32 release. A clean `npm ci` against -the published 0.1.32 artefact and the frozen Deno tool lock resolve the same -registry integrity. LiveSync passes its source checks, production build, and -1,034 unit tests on a local stack with [PR #1222](https://github.com/vrtmrz/obsidian-livesync/pull/1222). -The two CLI installer tests need a runner which permits child processes; both -pass under normal process permissions. The Deno setup-tool suite, CLI setup +The original protocol assessment found no blocker in the Time-bound URI +protocol. The Commonlib candidate based on 0.1.31 passed its type, unit, +boundary, release-process, and packed-package gates before the 0.1.32 release. A +clean `npm ci` against the published 0.1.32 artefact and the frozen Deno tool +lock resolved the same registry integrity. In the original implementation +baseline, LiveSync passed its source checks, production build, and 1,034 unit +tests on a local stack with [PR #1222](https://github.com/vrtmrz/obsidian-livesync/pull/1222). +The two CLI installer tests needed a runner which permits child processes; both +passed under normal process permissions. The Deno setup-tool suite, CLI setup and file-operation contract, and WebPeer browser tests passed before stacking. -Focused real-Obsidian generation checks pass on desktop and emulated mobile, +The recorded real-Obsidian generation checks passed on desktop and emulated mobile, including the displayed Time-bound end, Compatible output, and mobile touch -targets. A two-device CouchDB workflow now passes on the local stack: the -provisioning tool creates a generation 12 database, the first device generates +targets. A two-device CouchDB workflow passed on the local stack: the +provisioning tool created a generation 12 database, the first device generated a Time-bound Setup URI after declaring its required generation 13 feature, and -the second device imports it. An ordinary note completes a round trip, and a -hidden snippet synchronises. The separate real-Obsidian tests for live remote +the second device imported it. An ordinary note completed a round trip, and a +hidden snippet synchronised. The separate real-Obsidian tests for live remote feature changes, internal Metadata migration, and a percent-prefixed E2EE -passphrase also pass with published Commonlib 0.1.32. Unit and protocol tests -cover window boundaries and preserve the existing distinction between an empty -passphrase and cancellation. The WebPeer browser test confirms that its monitor -remains available after the URI window ends for a device which has already -imported the URI. +passphrase also passed with published Commonlib 0.1.32. Unit and protocol tests +covered window boundaries and preserved the existing distinction between an +empty passphrase and cancellation. The WebPeer browser test confirmed that its +monitor remained available after the URI window ended for a device which had +already imported the URI. -The current Commonlib 0.1.33 integration includes the merged host changes from -PR #1222 and PR #1225. npm 10 and normal clean installations preserve the lockfile; -source checks, the production build, 1,064 unit tests, and ten Deno setup-tool -tests pass. Both URI modes preserve ID recovery and explicit legacy ID selection. +The Commonlib 0.1.33 integration included the merged host changes from +PR #1222 and PR #1225. At that baseline, npm 10 and normal clean installations +preserved the lockfile; source checks, the production build, 1,064 unit tests, +and ten Deno setup-tool tests passed. Both URI modes preserved ID recovery and +explicit legacy ID selection. -Real Obsidian tests cover independent ID keys through Time-bound, Compatible, -and QR setup, encrypted local persistence, natural restarts of both devices, -and bidirectional note transfers with stable document and Chunk IDs. Incorrect -passphrases and past-window URIs leave runtime and persisted settings unchanged. -The past clock belongs to an isolated fixture worker, not Obsidian or the host. -CouchDB tests also cover a custom ID source and recovery code, rejection of -incompatible document keys before remote writes, and the Doctor's decline, -reminder, dismissal, and later acceptance paths. The detailed commands and the -remaining published-artefact and physical-device checks are recorded in the +Recorded real-Obsidian tests covered independent ID keys through Time-bound, +Compatible, and QR setup, encrypted local persistence, natural restarts of both +devices, and bidirectional note transfers with stable document and Chunk IDs. +Incorrect passphrases and past-window URIs left runtime and persisted settings +unchanged. The past clock belonged to an isolated fixture worker, not Obsidian +or the host. CouchDB tests also covered a custom ID source and recovery code, +rejection of incompatible document keys before remote writes, and the Doctor's +decline, reminder, dismissal, and later acceptance paths. The detailed commands, +remaining published-artefact checks, and physical-device checks are recorded in the [real Obsidian test guide](../../test/e2e-obsidian/README.md#combined-setup-and-security-regression-checks). Compatible preserves the URI encryption format; each receiving client must still support the shared settings and the remote's declared features. The host compatibility checks from PR #1222 remain necessary. These results do not -establish a mobile performance bound. The first implementation changes only -primary-language resources; translation changes require separate scope. +establish a mobile performance bound. The initial implementation changed only +primary-language resources; translation changes remain a separate scope. diff --git a/docs/design_docs/tweak_compatibility.md b/docs/design_docs/tweak_compatibility.md index be7dda44..d9b0e971 100644 --- a/docs/design_docs/tweak_compatibility.md +++ b/docs/design_docs/tweak_compatibility.md @@ -2,15 +2,16 @@ date: 2026-10-17 commonlib-version: "0.1.42" self-hosted-livesync-version: "1.0.37" -status: unreleased +status: released --- # Tweak compatibility and recovery -This document describes the compatibility and recovery contract, including -unreleased changes after Self-hosted LiveSync 1.0.37 using Commonlib 0.1.42. -Commonlib owns the interpretation of synchronisation settings; LiveSync owns -the dialogues and operations which consume that result. +This document describes the compatibility and recovery contract introduced +after the Self-hosted LiveSync 1.0.37 and Commonlib 0.1.42 baseline recorded +above. The current assessment is included in Self-hosted LiveSync 1.0.42 with +Commonlib 0.2.2. Commonlib owns the interpretation of synchronisation settings; +LiveSync owns the dialogues and operations which consume that result. ## Shared assessment diff --git a/docs/design_docs/unicode_path_identity.md b/docs/design_docs/unicode_path_identity.md index 6827d9b0..c5808f92 100644 --- a/docs/design_docs/unicode_path_identity.md +++ b/docs/design_docs/unicode_path_identity.md @@ -2,7 +2,7 @@ date: 2026-10-19 commonlib-version: "0.1.43" self-hosted-livesync-version: "1.0.38" -status: unreleased +status: released --- # Unicode path identity @@ -17,8 +17,10 @@ Commonlib's [consumer contract](https://github.com/vrtmrz/livesync-commonlib/blo defines the shared package boundary. NFC identity shipped in Self-hosted LiveSync `1.0.38` with Commonlib `0.1.42`. -The deletion handling in reconstruction below uses Commonlib `0.1.43` and -remains unreleased in Self-hosted LiveSync. +Conflict-aware deletion handling during reconstruction was added separately +and is included in current Commonlib `0.2.2`, used by Self-hosted LiveSync +`1.0.42`. This later handling is distinct from the initial NFC identity +release. ## Scope diff --git a/docs/design_docs_of_journalsync.md b/docs/design_docs_of_journalsync.md index e2f18b0c..398c3844 100644 --- a/docs/design_docs_of_journalsync.md +++ b/docs/design_docs_of_journalsync.md @@ -1,5 +1,7 @@ ## The design document of the journal sync +Historical design: this is the first Journal Sync design. Its original transfer sequence and discussion of possible storage backends are retained as history. For the maintained pipeline and storage-adapter design, read [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) and the [Replicator architecture](design_docs/replicator_architecture.md). Use [Object Storage Setup](setup_object_storage.md) for supported client setup. A possible backend discussed here is not evidence that an adapter is available in the current release. + Original title: Synchronise without CouchDB ### Goal @@ -47,4 +49,4 @@ This implementation replaces the synchronisation performed by CouchDB. Therefore ### Consideration and Conclusion -This design offers a novel approach to journal synchronisation without relying on CouchDB. It leverages PouchDB's journaling capabilities and leverages simple server-side storage for efficient data exchange. Hence, the new design could be said to have gotten a broader outlook. \ No newline at end of file +This design offers a novel approach to journal synchronisation without relying on CouchDB. It leverages PouchDB's journaling capabilities and leverages simple server-side storage for efficient data exchange. Hence, the new design could be said to have gotten a broader outlook. diff --git a/docs/design_docs_of_remote_configurations.md b/docs/design_docs_of_remote_configurations.md index 1fdd2fb8..9ea42634 100644 --- a/docs/design_docs_of_remote_configurations.md +++ b/docs/design_docs_of_remote_configurations.md @@ -1,5 +1,9 @@ # The design document of remote configuration management +## Implementation status + +This document describes the profile catalogue and its runtime projections as used by Self-hosted LiveSync 1.0.42 with Commonlib 0.2.2. Commonlib owns profile migration, activation, and protected settings persistence. The [Replicator architecture](design_docs/replicator_architecture.md) and [P2P service lifecycle decision](adr/2026_08_replicator_capabilities_02_p2p_service_lifecycle.md) describe the separate P2P service boundary. + ## Goal - Allow us to manage multiple remote connections in a single vault. @@ -33,7 +37,9 @@ Once multiple remotes of the same type became desirable, the previous model no l The current settings now have two layers for remote configuration. 1. A stored collection of named remotes. -2. One active remote projected into the legacy flat settings fields. +2. One active main remote projected into the legacy flat settings fields. + +The P2P service also has its own selected profile, recorded by `P2P_ActiveRemoteConfigurationId`. This permits the P2P service to use a selected P2P profile alongside a CouchDB or Object Storage main remote; it does not introduce a second main remote. This means the replication and database layers can continue to read the effective remote from the existing settings fields, while the settings dialogue and migration logic can manage many stored remotes. @@ -54,6 +60,7 @@ type RemoteConfiguration = { type RemoteConfigurations = { remoteConfigurations: Record; activeConfigurationId: string; + P2P_ActiveRemoteConfigurationId: string; }; ``` @@ -116,7 +123,11 @@ Therefore: - Import trusts the supplied connection string as-is after parsing. - We should regard exported connection strings as sensitive information, much like Setup URI or a credentials-bearing configuration file. -The `isEncrypted` field is currently reserved for future expansion. At present, the connection string itself is stored plainly inside the settings data, in the same sense that the effective runtime configuration can contain usable remote credentials. +The `isEncrypted` field identifies an encrypted profile URI in persisted settings. When settings are saved with configuration encryption enabled, Commonlib encrypts the profile URIs and records `isEncrypted: true`. Loading protected settings decrypts the URIs into the runtime settings and clears that runtime flag before profile activation. The runtime settings and the persisted representation therefore have different purposes. + +Protection depends on the configured settings-passphrase policy and a usable key. A plaintext configuration is not made confidential by the presence of the field alone. Consult [Encrypting sensitive configuration items](settings.md#encrypting-sensitive-configuration-items) for that policy. + +Clipboard export serialises the usable connection string. Treat that exported value as sensitive even when its saved profile URI is encrypted at rest; storage protection does not encrypt the clipboard representation. ### Migration strategy @@ -129,7 +140,7 @@ Rules: - If no remote list exists and the legacy fields contain a Peer-to-Peer configuration, create `legacy-p2p`. - If more than one legacy remote is populated, create all possible entries and select the active one according to `remoteType`. -This migration is intentionally additive. It does not remove the flat fields because they remain necessary as the active runtime projection. +This migration is intentionally additive. It does not remove the flat fields because they remain necessary as the active runtime projection. A migrated legacy P2P profile also initialises the separate P2P selection. Loading an existing profile catalogue does not unconditionally replace an intentionally empty P2P selection. ### Normalisation and application paths @@ -190,7 +201,7 @@ The feature should be tested from four viewpoints. ## Outlook -Import/export configuration strings should also be encrypted in the future, but this is a separate feature that can be added on top of the current design. +A separately encrypted representation for clipboard import and export remains a possible extension. Encryption of profile URIs in persisted settings is already implemented as described above. ## Consideration and conclusion @@ -203,4 +214,4 @@ Its core idea is modest but effective. - Project it into the legacy settings for runtime use. - Normalise every imported settings object through the same path. -This keeps the implementation understandable and migration-friendly. It also opens the door for future work, such as encrypted per-remote storage, richer remote metadata, or remote-scoped options, without forcing another large redesign of how remotes are represented. \ No newline at end of file +This keeps the implementation understandable and migration-friendly. It also opens the door for future work, such as richer remote metadata or remote-scoped options, without forcing another large redesign of how remotes are represented. diff --git a/docs/design_docs_of_sharing_tweak_value.md b/docs/design_docs_of_sharing_tweak_value.md index 4016b7fb..3ab02008 100644 --- a/docs/design_docs_of_sharing_tweak_value.md +++ b/docs/design_docs_of_sharing_tweak_value.md @@ -1,6 +1,6 @@ # Sharing `Tweak values` -NOTE: This is the planned feature design document. This is planned, but not be implemented now (v0.23.3). This has not reached the design freeze and will be added to from time to time. +Historical design: this document records the original proposal written for v0.23.3. Tweak sharing and compatibility assessment are implemented in published releases. Use [Tweak compatibility and recovery](design_docs/tweak_compatibility.md) for the maintained contract, and [Settings](settings.md) for the available controls. The proposal and test plan below are retained as design history; they do not define the current behaviour. ## Goal @@ -52,4 +52,4 @@ B: On the setting dialogue. - However, this might be an essential feature. Further efforts are desired. ### Consideration and Conclusion -To be described after implemented, tested, and, released. \ No newline at end of file +The published implementation has evolved beyond this proposal, including classification of compatible differences and automatic alignment where configured. Its current decisions, recovery requirements, and remaining limitations are recorded in [Tweak compatibility and recovery](design_docs/tweak_compatibility.md). The original proposal does not establish that every item in its test plan was implemented unchanged. diff --git a/docs/review_harness.md b/docs/review_harness.md index f94c670f..92738b57 100644 --- a/docs/review_harness.md +++ b/docs/review_harness.md @@ -19,10 +19,11 @@ The command and view are registered only when developer tools are enabled. Disab | --- | --- | --- | --- | | Settings lifecycle | Automatic | Read-only | Exposes the seven synchronisation choices as typed observations and, for a genuinely new Vault, compares the selected recommendations with Commonlib's new-Vault contract. Existing-Vault setting preservation remains an automated migration and `settings-ui` E2E responsibility. | | Compatibility review boundary | Guided | Device-local state | Observes the current dedicated compatibility controller and opens its actual review action. The Harness does not restore Change Log acknowledgement, `lastReadUpdates`, or a separate manual Pass or Fail result. | -| P2P composition | Automatic | Read-only | Confirms that the live P2P result resolves one replicator bound to the active Obsidian services. Peer discovery, replacement, reconnection, and transfer remain unit, CLI, and Compose E2E responsibilities. | +| ID generation performance | Automatic, individual | Read-only | Measures legacy and independent IDs using fixed in-memory inputs, including key derivation and heap samples where available. Keep Obsidian in the foreground. | +| File content hash performance | Manual | Read-only | Compares SHA-256 and xxHash64 on fixed in-memory Blob inputs. Input creation is excluded from hash timings; engine initialisation and warm-up are reported separately. | | Vault fixture round trip | Automatic, explicit | Dedicated Vault fixtures | After explicit confirmation, creates, reads, modifies, renames, and removes one owned fixture tree. General Vault reflection remains covered by its separate real-Obsidian E2E workflow. | -**Automatic** runs only the two read-only local observations. **Full review** also starts the guided compatibility observation and asks before the Vault fixture scenario writes anything. A scenario can also be run individually. +**Automatic** runs only Settings lifecycle. **Full review** visits all listed scenarios, including the guided compatibility review and the two performance measurements, and asks before the Vault fixture scenario writes anything. Each scenario can also be run individually; a scenario's mode label does not by itself include it in the Automatic action. These observations deliberately do not repeat stronger automated workflows. They exist to record which contracts the current real-device composition exposes while a maintainer reviews an immutable artefact. diff --git a/docs/specs_data_compression.md b/docs/specs_data_compression.md index ac6e2845..119bc879 100644 --- a/docs/specs_data_compression.md +++ b/docs/specs_data_compression.md @@ -33,7 +33,11 @@ The current benchmark measures the Node CLI process. It does not establish Obsid ## Reproducible benchmark -The benchmark is implemented under `src/apps/cli/testdeno` and packaged by `test/bench-network`. It uses the real CLI mirror and synchronisation path, Commonlib 0.1.0-rc.4, CouchDB 3.5.0, the V3 Rabin–Karp splitter, optional Data Compression, and E2EE V2. It runs each of these conditions three times in rotating order: +The benchmark is implemented under `src/apps/cli/testdeno` and packaged by `test/bench-network`. It exercises the real CLI mirror and synchronisation path. The recorded three-repeat result below used Commonlib 0.1.0-rc.4, CouchDB 3.5.0, the V3 Rabin–Karp splitter, optional Data Compression, and E2EE V2. + +The current Compose runner installs the checked-out repository's package and lockfile with `npm ci`; Self-hosted LiveSync 1.0.42 selects Commonlib 0.2.2. Running the command below measures that checkout, rather than recreating the older package environment automatically. Keep the recorded results associated with their original measurement date and dependency version. + +With the repeat count shown below, it runs each of these conditions three times in rotating order: - E2EE off, compression off; - E2EE off, compression on; diff --git a/src/apps/cli/testdeno/test_dev_deno.md b/src/apps/cli/testdeno/test_dev_deno.md index 110f2e94..bdd67142 100644 --- a/src/apps/cli/testdeno/test_dev_deno.md +++ b/src/apps/cli/testdeno/test_dev_deno.md @@ -146,7 +146,7 @@ These flows do not require Bash on the host. The S3 matrix tasks, environment va ### `test-mirror.ts` -- Verifies six core mirror scenarios. +- Verifies seven core mirror scenarios, including an unrecognised revision origin. - Does not require an external remote. ### `test-sync-two-local-databases.ts` @@ -247,11 +247,12 @@ deno task test:daemon deno task test:push-pull deno task test:sync-locked-remote -# CouchDB-based tests +# CouchDB and Object Storage tests deno task test:sync-two-local deno task test:decoupled-vault deno task test:remote-commands deno task test:e2e-couchdb +deno task test:e2e-matrix # Explicit host-direct P2P diagnostics for cross-platform investigations. # These are not part of the default suite or release evidence. @@ -260,7 +261,6 @@ deno task test:p2p-peers deno task test:p2p-sync deno task test:p2p-three-nodes deno task test:p2p-upload-download -deno task test:e2e-matrix ``` --- diff --git a/src/apps/webpeer/README.md b/src/apps/webpeer/README.md index 8441896a..08122557 100644 --- a/src/apps/webpeer/README.md +++ b/src/apps/webpeer/README.md @@ -49,6 +49,8 @@ After the first device connects, **Try another device without resetting** keeps Serve the production build over HTTPS or from `localhost`. Downloading `check.html` alone is not supported because the page uses built module assets and origin-scoped browser storage. The generated credentials are temporary and must not replace the settings of a production Vault. +The generated Setup URI is Time-bound, and the page shows when its current fixed UTC window ends. After that time, the page hides the URI and QR code and disables the additional-device action. Start a fresh check to configure another device. A monitor for a device which already imported the settings remains available. + The detailed scope and result semantics are recorded in the [browser-assisted P2P connection-check ADR](../../../docs/adr/2026_07_p2p_connection_check.md). ## Storage and lifecycle From e17979e45700e1087ddc43aca4f4cbcc42c71a85 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 13:14:38 +0000 Subject: [PATCH 57/68] Correct setup and server operation guidance --- docker/.env.example | 8 +- docker/README.md | 141 +++++++++++++--------------------- docker/config/cloudflared.yml | 15 ++-- docker/config/livesync.ini | 9 +-- docker/docker-compose.yml | 8 +- docs/setup_flyio.md | 36 +++++---- docs/setup_object_storage.md | 10 +-- docs/setup_own_server.md | 28 +++---- docs/setup_p2p.md | 10 +-- docs/tips/fast-setup.md | 22 +++--- utils/readme.md | 10 +-- 11 files changed, 133 insertions(+), 164 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 816b1460..fd1fdc45 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -34,9 +34,9 @@ ACME_EMAIL=you@yourdomain.com # PROFILE: tailscale (--profile tailscale) # ============================================================================= -# Tailscale OAuth key (not a regular auth key — must be OAuth for persistent use) -# Generate at: https://login.tailscale.com/admin/settings/oauth -# Scopes needed: devices:write +# Tailscale authentication key for this container. +# OAuth clients require their own scopes and tag configuration; see: +# https://tailscale.com/docs/features/containers/docker/docker-params#ts_authkey TS_AUTHKEY=tskey-auth-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx # Hostname this node will have on your tailnet (becomes ..ts.net) @@ -47,6 +47,6 @@ TS_HOSTNAME=livesync # ============================================================================= # Tunnel token from Cloudflare Zero Trust dashboard -# Create at: https://one.dash.cloudflare.com/ → Networks → Tunnels → Create tunnel +# Current setup guide: https://developers.cloudflare.com/tunnel/get-started/ # Copy the token from the "Install connector" step CF_TUNNEL_TOKEN=eyJhIjoixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/docker/README.md b/docker/README.md index eaa1bcdd..0eb33e55 100644 --- a/docker/README.md +++ b/docker/README.md @@ -61,11 +61,11 @@ docker compose --profile cloudflare up -d ```bash # Should return {"status":"ok"} -curl -u admin:yourpassword http://localhost:5984/_up +curl -u ':' http://localhost:5984/_up # Check CORS headers curl -v -H "Origin: app://obsidian.md" \ - -u admin:yourpassword \ + -u ':' \ http://localhost:5984/ ``` @@ -75,7 +75,7 @@ In the Obsidian plugin settings (**Self-hosted LiveSync**): | Field | Value | |---|---| -| URI | `https://your-domain-or-ts-hostname:5984` (or `http://localhost:5984` for LAN-only) | +| URI | `https://your-domain-or-ts-hostname` for an HTTPS profile, or `http://localhost:5984` for local HTTP | | Username | value of `COUCHDB_USER` | | Password | value of `COUCHDB_PASSWORD` | | Database name | value of `COUCHDB_DATABASE` (default: `obsidiannotes`) | @@ -85,10 +85,13 @@ In the Obsidian plugin settings (**Self-hosted LiveSync**): ## Profile Details -### Default (no profile) — LAN / Localhost only + -CouchDB is exposed on `http://localhost:5984` (or LAN IP). -**Desktop Obsidian works over HTTP.** Mobile Obsidian requires HTTPS — use a tunnel profile. +### Default (no profile) — Host HTTP access + +The Compose file publishes CouchDB on host port 5984 by default, on all host interfaces. Its reachability depends on the host firewall and network; selecting an HTTPS profile does not close that direct HTTP port. Restrict the host port to the intended network, bind it to loopback, or remove the mapping when all access uses the proxy. + +Desktop Obsidian can use HTTP. Use HTTPS for mobile clients and connections outside a trusted local network. The supplied HTTPS profiles use port 443, so their client URL normally has no `:5984` suffix. ### `--profile caddy` — Public Domain + Auto TLS @@ -104,13 +107,14 @@ ACME_EMAIL=you@example.com Caddy automatically issues a Let's Encrypt certificate. No manual cert management. -### `--profile tailscale` — No Domain Required ✅ Recommended for privacy + + +### `--profile tailscale` — Tailnet or public HTTPS **Requires**: -- Free [Tailscale account](https://login.tailscale.com/) -- Install the Tailscale app on all your Obsidian devices -- Generate an **OAuth key** at: https://login.tailscale.com/admin/settings/oauth - (Scopes: `devices:write`) +- A [Tailscale account](https://login.tailscale.com/). +- The Tailscale app on devices using private tailnet access. +- An authentication key for `TS_AUTHKEY`. Follow the [Docker authentication parameters](https://tailscale.com/docs/features/containers/docker/docker-params#ts_authkey) if using an OAuth client instead. **Set in `.env`**: ``` @@ -118,34 +122,22 @@ TS_AUTHKEY=tskey-auth-... TS_HOSTNAME=livesync ``` -**Two sub-modes**: -- **VPN mode** (default): CouchDB accessible only to devices on your Tailnet at - `https://livesync..ts.net` — completely private -- **Funnel mode**: public HTTPS at `https://livesync..ts.net` — no domain purchase - Enable in your [Tailscale ACL](https://login.tailscale.com/admin/acls): - ```json - "nodeAttrs": [{"target": ["tag:container"], "attr": ["funnel"]}] - ``` +Choose the exposure before starting the profile. The bundled `config/ts-serve.json` sets `AllowFunnel` to `true`, requesting public HTTPS when the tailnet permits Funnel. It is not a private-only default. -> **Note on Windows Docker Desktop**: If `/dev/net/tun` is unavailable, add `TS_USERSPACE=true` -> to the tailscale service environment in `docker-compose.yml`. +- For private tailnet access, set that entry to `false` and verify the effective Serve configuration. +- For public Funnel access, retain `true` and configure the required tailnet permissions. See the [Tailscale Docker guide](https://tailscale.com/blog/docker-tailscale-guide) for the distinction between Serve and Funnel. + +The bundled service expects `/dev/net/tun`. On hosts without that device, follow Tailscale's userspace-networking instructions and adjust the device mount as well as `TS_USERSPACE`; changing the environment variable alone leaves the required bind mount in place. ### `--profile cloudflare` — Cloudflare Tunnel **Requires**: -- Free [Cloudflare account](https://www.cloudflare.com/) -- A domain managed by Cloudflare DNS (can transfer existing domain for free) -- Cloudflare Zero Trust account (free) +- A [Cloudflare account](https://www.cloudflare.com/). +- A domain whose DNS is managed by Cloudflare. #### Step 1: Create a Cloudflare Tunnel -1. Log in to [Cloudflare Zero Trust](https://one.dash.cloudflare.com/) -2. Navigate to **Networks → Tunnels** -3. Click **Create a tunnel** -4. Choose **Cloudflared** as tunnel type -5. Name your tunnel (e.g., `obsidian-livesync`) -6. Click **Save tunnel** -7. **Copy the tunnel token** — it looks like `eyJhIjoiZX...` (very long, ~400 characters) +Create a remotely managed tunnel using the current [Cloudflare Tunnel setup instructions](https://developers.cloudflare.com/tunnel/get-started/). Copy its connector token into `CF_TUNNEL_TOKEN` below. This Compose profile runs the connector; a separate host installation of `cloudflared` is unnecessary for this profile. #### Step 2: Configure Environment @@ -157,27 +149,14 @@ COUCHDB_DOMAIN=sync.yourdomain.com # Must be a domain managed by Cloudflare #### Step 3: Add Public Hostname Route -🚨 **CRITICAL**: Token-based tunnels ignore the local `cloudflared.yml` config file. All routing is controlled from the dashboard. +Configure the route in the Cloudflare dashboard for this remotely managed tunnel. The local `config/cloudflared.yml` is not the routing source for the token-managed profile. -Back in the Zero Trust dashboard, **in the same tunnel creation flow** (or edit your tunnel later): +1. Open the tunnel under **Networking → Tunnels**. +2. Under **Routes**, add a **Published application**. +3. Choose the public hostname, for example `sync.yourdomain.com`. +4. Set the service URL to `http://couchdb:5984` and save the route. -1. Go to the **Public Hostname** tab -2. Click **Add a public hostname** -3. Configure: - - **Subdomain**: `sync` (or your preferred subdomain) - - **Domain**: Select your Cloudflare domain from dropdown - - **Type**: `HTTP` - - **URL**: `couchdb:5984` ← **Do NOT use `localhost`!** - -**Why `couchdb:5984` not `localhost:5984`?** -- The `cloudflared` container runs inside Docker on the same network as `couchdb` -- Docker's internal DNS resolves `couchdb` to the correct container -- Using `localhost` would look inside the `cloudflared` container (nothing there) - -4. Under **Additional application settings** (expand): - - **No TLS Verify**: Leave **OFF** (CouchDB uses plain HTTP internally, that's fine) - - Leave other settings at defaults -5. Click **Save hostname** +The `cloudflared` and `couchdb` services share a Docker network. The service name reaches CouchDB; `localhost` would address the `cloudflared` container itself. The local connection is HTTP, while the public client URL is HTTPS. Keep TLS certificate verification enabled for any HTTPS origin you configure later. The [current setup guide](https://developers.cloudflare.com/tunnel/get-started/) describes the dashboard flow. #### Step 4: Start the Stack @@ -203,7 +182,7 @@ You should see: curl -I https://sync.yourdomain.com # Should return {"couchdb":"Welcome",...} -curl -u admin:yourpassword https://sync.yourdomain.com +curl -u ':' https://sync.yourdomain.com ``` If you get **404**, see Troubleshooting below. @@ -220,9 +199,7 @@ In Obsidian → Settings → **Self-hosted LiveSync**: | Database name | value of `COUCHDB_DATABASE` from `.env` (default: `obsidiannotes`) | | End-to-end passphrase | *Choose your own* — never stored server-side | -Under **Remote Database Configuration → Advanced**: -- Enable: ✅ **Use Request API to avoid inevitable CORS problem** - (See "Known Issue" below for why this is critical) +If browser requests encounter a CORS failure, consult [Use Request API](../docs/settings.md#use-request-api-to-avoid-inevitable-cors-problem). That setting selects Obsidian's HTTP request adapter. It does not select the synchronisation mode or remove Cloudflare's proxy limits. --- @@ -241,13 +218,11 @@ docker logs livesync-cloudflared --tail 20 **Fix**: The public hostname rule is missing or incorrect. -1. Go to Zero Trust → Networks → Tunnels → your tunnel → **Edit** -2. Click **Public Hostname** tab -3. Verify a hostname exists with: - - Service Type: `HTTP` - - URL: `couchdb:5984` (NOT `localhost:5984`) -4. If no hostname exists, add it (see Step 3 above) -5. Wait 30 seconds for changes to propagate, then test again +1. Open the tunnel's published application routes in the Cloudflare dashboard. +2. Verify that the requested hostname maps to `http://couchdb:5984`. +3. Add or correct the route if required, then retry after the configuration has propagated. + +A connected tunnel alone does not establish that the hostname route or origin is correct. **Problem: Connection immediately closes / 502 Bad Gateway** @@ -258,10 +233,10 @@ docker ps --filter "name=livesync-couchdb" # Status should be: Up (healthy) docker inspect livesync-couchdb -f '{{.NetworkSettings.Networks}}' -# Should show: livesync-net +# Both services should list the same Compose network. docker inspect livesync-cloudflared -f '{{.NetworkSettings.Networks}}' -# Should also show: livesync-net +# Compare with the CouchDB network above. ``` **Fix**: If CouchDB is unhealthy, check logs: @@ -271,41 +246,35 @@ docker logs livesync-couchdb --tail 50 **Problem: 524 Timeout Errors During Sync** -**Root cause**: Cloudflare's proxy has a **100-second idle timeout**. CouchDB's replication protocol uses long-polling on the `_changes` feed, which can idle for longer during quiet periods. +A `524` can indicate that the origin did not respond or accept request data within Cloudflare's proxy timeout. Inspect the failing request, CouchDB response times, and connector logs before selecting a remedy. See [Cloudflare's current 524 guidance](https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-5xx-errors/error-524/) for the applicable limits and diagnostic steps. -**Fix**: Switch to short-polling mode in the Obsidian plugin: -1. Obsidian → Settings → Self-hosted LiveSync -2. **Remote Database Configuration → Advanced** -3. Enable: ✅ **Use Request API to avoid inevitable CORS problem** -4. Save and restart sync - -This keeps all requests under 100 seconds. - -**Alternative**: Use Tailscale or Caddy profiles instead — neither has aggressive timeouts. +**Use Request API** changes the HTTP adapter; it does not turn LiveSync mode into short polling or guarantee that a request completes within a proxy timeout. Review the actual synchronisation mode and [heartbeat/timeout setting](../docs/settings.md#use-timeouts-instead-of-heartbeats), then verify transfers on the intended devices. If the required traffic does not fit the proxy limits, consider an appropriately secured direct HTTPS route through Caddy or private Tailscale access. --- ## Data & Backup -All vault data lives in the `couchdb-data` Docker named volume. +The server database files live in the `couchdb-data` Docker named volume. Keep Vault backups, encryption passphrases, and ID recovery information separately. Back up the Compose configuration and CouchDB configuration as well; the database volume alone does not contain all of them. + +The following example creates an archive while CouchDB is stopped. Verify the volume name before running it, especially if you have overridden the Compose project name: ```bash -# Backup -docker run --rm -v obsidian-livesync_couchdb-data:/data \ - -v $(pwd)/backup:/backup alpine \ - tar czf /backup/couchdb-backup-$(date +%Y%m%d).tar.gz -C /data . - -# Restore -docker run --rm -v obsidian-livesync_couchdb-data:/data \ - -v $(pwd)/backup:/backup alpine \ - tar xzf /backup/couchdb-backup-20260218.tar.gz -C /data +docker volume inspect obsidian-livesync_couchdb-data +mkdir -p backup +docker compose stop couchdb +docker run --rm --network none -v obsidian-livesync_couchdb-data:/data:ro \ + -v "$(pwd)/backup:/backup" alpine \ + tar czf "/backup/couchdb-backup-$(date +%Y%m%d-%H%M%S).tar.gz" -C /data . +docker compose start couchdb ``` +Before restoration, preserve the existing database volume. Restore and verify the archive in a separate recovery volume with a compatible CouchDB installation before replacing the active data. Do not extract an archive over a running database. See [Backing up CouchDB](https://docs.couchdb.org/en/stable/maintenance/backups.html) for database, configuration, and replication-based backup options. + --- ## Security Notes -- CouchDB requires authentication for **all** requests (configured by `livesync.ini`) +- The init service enables authenticated CouchDB access through the REST API. Verify that protected database endpoints reject unauthenticated requests before exposing the service. - Enable **End-to-End Encryption** passphrase in the Obsidian plugin — vault data is encrypted before it ever leaves your device - The init container runs once and exits — it has no persistent access @@ -344,8 +313,8 @@ open http://localhost:5984/_utils | Init container keeps restarting | CouchDB not healthy yet — wait 30s, check `docker compose logs couchdb` | | `curl: (52) Empty reply` | CouchDB not fully started — the healthcheck should gate this | | Mobile can't connect | Needs HTTPS — use tailscale or caddy profile | -| 524 errors with Cloudflare | Enable "Use Request API" toggle in Obsidian plugin | -| `Permission denied` on volumes | Run `docker compose down -v` and retry — first-run volume ownership issue | +| 524 errors with Cloudflare | Inspect the failing request and origin response time; follow the timeout guidance above. | +| `Permission denied` on volumes | Preserve the volume, inspect the CouchDB logs and mount ownership, and correct the specific permission problem. `docker compose down -v` deletes the database volume and is not a repair step. | | CORS errors in browser | Confirm CouchDB headers: `curl -v -H "Origin: app://obsidian.md" http://localhost:5984/` | | CouchDB exits immediately, zero logs (Windows) | **Do not add `:ro`** to the `livesync.ini` volume mount. CouchDB's entrypoint runs `chmod 0644` on all files in `/opt/couchdb/etc` — read-only bind mounts cause a silent EPERM crash on Docker Desktop for Windows (WSL2). The compose file is already correct; do not modify it. | | Settings in `livesync.ini` seem ignored | Settings requiring restart (e.g. bind_address) load at start. Runtime-only settings (require_valid_user, enable_cors) are set by the init container via REST API and take effect immediately without restart. | diff --git a/docker/config/cloudflared.yml b/docker/config/cloudflared.yml index 3e18bcfb..0ea3a205 100644 --- a/docker/config/cloudflared.yml +++ b/docker/config/cloudflared.yml @@ -2,17 +2,16 @@ # ============================================================================= # # Prerequisites: -# 1. Create a tunnel in Cloudflare Zero Trust → Networks → Tunnels +# 1. Create a remotely managed tunnel using the current Cloudflare dashboard guide. # 2. Copy the tunnel token to CF_TUNNEL_TOKEN in your .env -# 3. Add a public hostname in the tunnel config: +# 3. Add a published application route in the dashboard: # Hostname : couchdb.yourdomain.com (or whatever you set COUCHDB_DOMAIN to) # Service : http://couchdb:5984 # -# Known issue: Cloudflare's 100-second proxy timeout can interrupt CouchDB's -# long-polling replication change feed, causing 524 errors. -# MITIGATION: In the Obsidian plugin settings, enable: -# "Use Request API to avoid inevitable CORS problem" -# This switches from long-poll to short-poll mode. +# The token-managed Compose profile uses dashboard routing. The local ingress +# example below describes locally managed routing and is not that profile's +# source of routes. Diagnose 524 errors using current Cloudflare guidance. +# The Obsidian Request API setting changes the HTTP adapter, not the sync mode. # # ============================================================================= @@ -23,7 +22,7 @@ ingress: - hostname: ${COUCHDB_DOMAIN} service: http://couchdb:5984 originRequest: - # Increase timeouts for CouchDB replication streams + # Origin connection settings for this locally managed example connectTimeout: 30s keepAliveTimeout: 90s keepAliveConnections: 100 diff --git a/docker/config/livesync.ini b/docker/config/livesync.ini index fd1ffb00..062d8e98 100644 --- a/docker/config/livesync.ini +++ b/docker/config/livesync.ini @@ -1,10 +1,9 @@ ; CouchDB local configuration for Self-hosted LiveSync -; This file is volume-mounted into /opt/couchdb/etc/local.d/livesync.ini +; This file is volume-mounted into /opt/couchdb/etc/local.ini ; -; IMPORTANT: Do NOT set require_valid_user here. -; CouchDB needs to start without auth to complete its first-run cluster setup -; (_users, _replicator databases must be created first). -; The couchdb-init service applies auth lockdown via REST API after first-run. +; The couchdb-init service configures require_valid_user and CORS through the +; REST API during provisioning. Verify that the init service completes before +; exposing the database to clients. [couchdb] ; Max size per document (50MB). Large enough for binary attachments. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5f7350b0..b074af09 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,7 +2,7 @@ # ============================================================================= # PROFILES # -------- -# (default) CouchDB only — LAN/localhost access, no TLS +# (default) CouchDB only — host HTTP port, no TLS # Suitable for desktop-only use or testing. # # --profile caddy CouchDB + Caddy reverse proxy @@ -14,7 +14,7 @@ # # --profile cloudflare CouchDB + cloudflared tunnel daemon # Free public HTTPS via Cloudflare. Needs a CF account + tunnel token. -# NOTE: Enable "Use Request API" in the Obsidian plugin to avoid 524 timeouts. +# Review proxy timeouts separately from the Obsidian HTTP adapter setting. # # QUICK START (local test): # cp .env.example .env && edit .env @@ -147,8 +147,8 @@ services: # PROFILE: cloudflare — cloudflared tunnel daemon # Requirements: Cloudflare account, tunnel token from CF Zero Trust dashboard # Usage: docker compose --profile cloudflare up -d - # NOTE: Enable "Use Request API" toggle in the Obsidian LiveSync plugin settings - # to avoid Cloudflare's 100-second proxy timeout (524 errors). + # NOTE: The Request API setting changes the HTTP adapter, not the sync mode. + # Diagnose proxy timeouts using the Docker README and current provider limits. # --------------------------------------------------------------------------- cloudflared: image: cloudflare/cloudflared:latest diff --git a/docs/setup_flyio.md b/docs/setup_flyio.md index 56f38caa..0b926b3f 100644 --- a/docs/setup_flyio.md +++ b/docs/setup_flyio.md @@ -12,7 +12,7 @@ This is how to configure fly.io and CouchDB on it for Self-hosted LiveSync. > **What and why is the Fly.io?** > At some point, we started to experience problems related to our IBM Cloudant account. At the same time, Self-hosted LiveSync started to improve its functionality, requiring CouchDB in a more natural state to use all its features. > -> Then we found Fly.io. Fly.io is the PaaS Platform, which can be useable for a very reasonable price. It generally falls within the `Free Allowances` range in most cases. +> Fly.io charges according to current account and resource usage, including Machines, persistent volumes, and network usage. Check [Fly.io pricing](https://fly.io/pricing/) and the [pricing documentation](https://fly.io/docs/about/pricing/) before deploying. A time-limited trial does not provide ongoing free hosting. ## Required materials @@ -31,10 +31,10 @@ This is how to configure fly.io and CouchDB on it for Self-hosted LiveSync. 4. Copy the Setup-URI and Use it in the Obsidian. 5. You have been synchronised. Use the Setup-URI in subsequent devices. -Steps 4 and 5 are detailed in the [Quick Setup](./quick_setup.md#1-using-setup-uris). +For steps 4 and 5, see [Set up the first device](./quick_setup.md#set-up-the-first-device), then [Add another device](./quick_setup.md#add-another-device) for subsequent devices. > [!NOTE] -> Your automatically configured configurations will be shown on the result in the Colab note like below, and **it will not be saved**. Please make a note of it somewhere. +> Save your own generated connection details and Setup URI securely. This is a historical Colab output example from 2024; its values are illustrative and will differ for each deployment. Replace all example credentials before deployment. > ``` > -- YOUR CONFIGURATION -- > URL : https://billowing-dawn-6619.fly.dev @@ -45,16 +45,18 @@ Steps 4 and 5 are detailed in the [Quick Setup](./quick_setup.md#1-using-setup-u ### B. Scripted Setup -Please refer to the document of [deploy-server.sh](../utils/readme.md#deploy-serversh). +Please refer to the document of [deploy-server.sh](../utils/readme.md#flyio-deployment). ### C. Manual Setup | Used in the text | Meaning and where to use | Memo | | ---------------- | --------------------------- | ------------------------------------------------------------------------ | -| campanella | Username | It is less likely to fail if it consists only of letters and numbers. | -| dfusiuada9suy | Password | | +| `` | Username | Choose a unique value using only letters and numbers. | +| `` | Password | Generate a strong, unique password for this instance. | | nrt | Region to make the instance | We can use any [region](https://fly.io/docs/reference/regions/) near us. | +Choose an available, unique Fly.io application name and replace both credential placeholders before deployment. Historical output below illustrates the command responses; use your own application name, hostname, and credentials. + #### 1. Install flyctl - Mac or Linux @@ -97,8 +99,8 @@ For more information, please refer to [Sign up](https://fly.io/docs/hands-on/sig ```bash -$ fly launch --name=billowing-dawn-6619 --env="COUCHDB_USER=campanella" --copy-config=true --detach --no-deploy --region nrt --yes -$ fly secrets set COUCHDB_PASSWORD=dfusiuada9suy +$ fly launch --name='' --env="COUCHDB_USER=" --copy-config=true --detach --no-deploy --region nrt --yes +$ fly secrets set COUCHDB_PASSWORD='' ``` #### 4. Deploy @@ -177,17 +179,17 @@ This section is set up using the REST API. - Mac or Linux: ```bash -export couchHost=https://billowing-dawn-6619.fly.dev -export couchUser=campanella -export couchPwd=dfusiuada9suy +export couchHost='https://' +export couchUser='' +export couchPwd='' ``` - Windows ```powershell -set couchHost https://billowing-dawn-6619.fly.dev -set couchUser campanella -set couchPwd dfusiuada9suy +set couchHost 'https://' +set couchUser '' +set couchPwd '' $creds = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("${couchUser}:${couchPwd}")) ``` @@ -242,11 +244,11 @@ Note: Each of these should also be repeated until finished in 200. #### 6. Use it from Self-hosted LiveSync -Now the CouchDB is ready to use from Self-hosted LiveSync. We can use `https://billowing-dawn-6619.fly.dev` in URI, `campanella` in `Username` and `dfusiuada9suy` in `Password` on Self-hosted LiveSync. The `Database name` could be anything you want. -Please refer to the [Minimal Setup of the Quick Setup](./quick_setup.md#2-minimal-setup). +Now the CouchDB is ready to use from Self-hosted LiveSync. Use your Fly.io app's hostname in `URI`, and the unique credentials configured above in `Username` and `Password`. The `Database name` could be anything you want. +Please refer to [Configure CouchDB manually on the first device](./quick_setup.md#configure-couchdb-manually-on-the-first-device). ## Delete the Instance If you want to delete the CouchDB instance, you can do that in [fly.io Dashboard](https://fly.io/dashboard/personal) -If you have done with [B. Scripted Setup](#b-scripted-setup), we can use [delete-server.sh](../utils/readme.md#delete-serversh). \ No newline at end of file +If you have done with [B. Scripted Setup](#b-scripted-setup), we can use [delete-server.sh](../utils/readme.md#flyio-deployment). diff --git a/docs/setup_object_storage.md b/docs/setup_object_storage.md index c2a5237b..37b93075 100644 --- a/docs/setup_object_storage.md +++ b/docs/setup_object_storage.md @@ -18,14 +18,14 @@ The public generator applies the Object Storage preset and records the connectio ```sh export remote_type=s3 -export endpoint=https://objects.example.com -export access_key= -export secret_key= +export endpoint='https://objects.example.com' +export access_key='' +export secret_key='' export bucket=vault-data export region=auto export bucket_prefix=my-vault -export LIVESYNC_PASSPHRASE= -export uri_passphrase= +export LIVESYNC_PASSPHRASE='' +export uri_passphrase='' deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/docs/setup_own_server.md b/docs/setup_own_server.md index 6924ec55..cc1994e1 100644 --- a/docs/setup_own_server.md +++ b/docs/setup_own_server.md @@ -122,14 +122,14 @@ The `username` and `password` in this step are CouchDB administrator credentials ``` export hostname=http://localhost:5984 -export username= -export password= +export username='' +export password='' export database=obsidiannotes curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | bash ``` > [!TIP] -> In fish, `hostname` is read-only and cannot be exported. Pass the values through `env` instead: `curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | env hostname=http://localhost:5984 username=<...> password=<...> database=obsidiannotes bash`. +> In fish, `hostname` is read-only and cannot be exported. Pass the values through `env` instead: `curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | env hostname='http://localhost:5984' username='' password='' database=obsidiannotes bash`. If it results like the following: ``` @@ -140,7 +140,7 @@ The wrapper runs the exact registry-pinned Commonlib consumer. When `database` i If you are using Docker Compose and the above command does not work or displays `ERROR: Hostname missing`, you can try running the following command, replacing the placeholders with your own values: ``` -curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | hostname=http://:5984 username= password= database=obsidiannotes bash +curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | hostname='http://:5984' username='' password='' database=obsidiannotes bash ``` ## 3. Expose CouchDB to the Internet @@ -148,13 +148,13 @@ curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/co - You can skip this instruction if you using only in intranet and only with desktop devices. - For mobile devices, Obsidian requires a valid SSL certificate. Usually, it needs exposing the internet. -Whatever solutions we can use. For simplicity, the following sample uses Cloudflare Zero Trust for testing. +For temporary testing, create a Cloudflare Quick Tunnel by following the [Quick Tunnels instructions](https://developers.cloudflare.com/tunnel/get-started/quick-tunnels/). It prints a random hostname which changes each time you create a tunnel, and the URL stops working when the `cloudflared` process stops. Quick Tunnels are intended for testing and development, not production. See the [Cloudflare Tunnel guide](https://developers.cloudflare.com/tunnel/get-started/) for durable use. ``` cloudflared tunnel --url http://localhost:5984 ``` -You will then get the following output: +The following 2024 output is an example of the format. Its hostname is illustrative only; use the URL printed by your own tunnel instead. ``` 2024-02-14T10:35:25Z INF Thank you for trying Cloudflare Tunnel. Doing so, without a Cloudflare account, is a quick way to experiment and try it out. However, be aware that these account-less Tunnels have no uptime guarantee. If you intend to use Tunnels in production you should use a pre-created named tunnel by following: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps @@ -167,7 +167,7 @@ You will then get the following output: : : ``` -Now `https://tiles-photograph-routine-groundwater.trycloudflare.com` is our server. Make it into the background once, please. +Use the URL printed by your tunnel as the server address. Keep the tunnel process running while you configure and test the client. ## 4. Client Setup @@ -179,12 +179,12 @@ Now `https://tiles-photograph-routine-groundwater.trycloudflare.com` is our serv The `LIVESYNC_USER` and `LIVESYNC_PASS` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. ```bash -export LIVESYNC_HOST=https://tiles-photograph-routine-groundwater.trycloudflare.com +export LIVESYNC_HOST='' export LIVESYNC_DB=obsidiannotes -export LIVESYNC_USER= -export LIVESYNC_PASS= -export LIVESYNC_PASSPHRASE= -export uri_passphrase= # Optional +export LIVESYNC_USER='' +export LIVESYNC_PASS='' +export LIVESYNC_PASSPHRASE='' +export uri_passphrase='' # Optional deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` @@ -199,9 +199,9 @@ The generator consumes the exact registry-pinned Commonlib release used by the p By default, the URI is Ephemeral and can be opened only until the exact UTC time printed by the generator. This is the end of a fixed seven-day window, not seven days after creation. Set `uri_mode=persistent` before running the command if the URI needs no time condition or must be readable by an older client that supports the existing encrypted format. -You will then get the following output: +The output has the following form. The timestamp and values below are illustrative: -```bash +```text Generated couchdb Setup URI. Ephemeral: usable until 2026-10-01T00:00:00.000Z (UTC). Your passphrase for the Setup URI is: H7vX...a-random-32-character-value diff --git a/docs/setup_p2p.md b/docs/setup_p2p.md index c755ce0a..ea334e19 100644 --- a/docs/setup_p2p.md +++ b/docs/setup_p2p.md @@ -127,11 +127,11 @@ The public Setup URI generator is also available when configuration must be crea ```sh export remote_type=p2p -export p2p_relays=wss://relay.example.com -export p2p_room_id= # Optional; generated when omitted -export p2p_passphrase= # Optional; generated when omitted -export LIVESYNC_PASSPHRASE= -export uri_passphrase= +export p2p_relays='wss://relay.example.com' +export p2p_room_id='' # Optional; generated when omitted +export p2p_passphrase='' # Optional; generated when omitted +export LIVESYNC_PASSPHRASE='' +export uri_passphrase='' deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts ``` diff --git a/docs/tips/fast-setup.md b/docs/tips/fast-setup.md index d67c743d..e570ed5e 100644 --- a/docs/tips/fast-setup.md +++ b/docs/tips/fast-setup.md @@ -1,8 +1,8 @@ # Fast Setup (Simple Fetch) -Fast Setup is a streamlined, user-friendly data retrieval and initialisation flow designed to simplify setting up secondary devices or recovering databases. +Fast Setup (Simple Fetch) is the preferred flow for initial replication to a new or empty secondary device. It can also be used for recovery, but a retrieval choice may replace local files. Preserve valuable local and remote copies before choosing how to handle a Vault that already contains work. See [Recovery and flag files](../recovery.md#first-choose-the-authoritative-copy) for guidance on identifying the authoritative copy. -Instead of guiding the user through the detailed multi-step setup wizard dialogues, Fast Setup prompts the user with high-level sync decisions and automates database download and local storage scanning in one continuous process. +Instead of guiding the user through the detailed multi-step setup wizard dialogues, Fast Setup prompts the user with high-level synchronisation decisions and automates data retrieval and local storage scanning in one process. The retrieval path depends on the selected remote. --- @@ -16,14 +16,14 @@ On the next startup, the plug-in boots in scheduled fetch mode and opens a simpl ## Technical Characteristics -Fast Setup leverages several backend optimisations to make the retrieval fast, safe, and clean: +Fast Setup uses several retrieval and reflection behaviours: -1. **Stream-based Replication for Speed** - - It fetches all remote metadata via stream reception, which is significantly faster than traditional chunk-by-chunk retrieval. +1. **Stream-based Fast Fetch for CouchDB** + - The stream-based Fast Fetch path is CouchDB-specific. It retrieves remote Metadata incrementally; Object Storage and P2P use their own retrieval paths. 2. **Delayed File Reflection to Prevent Corrupted Warnings** - - By suspending file reflection during the download phase, it prevents the plug-in from raising temporary or false "corrupted data synchronisation" or "size mismatch" warnings that can occur during the chunk download process. -3. **Time-Based Comparison is Generally Sufficient** - - Since the vault is entering a fresh synchronisation or recovery state, comparing files based on their modification timestamps (newer-wins) is highly reliable and sufficient to reconcile files without needing complex manual conflict resolution. + - Delaying local file reflection during retrieval suppresses temporary synchronisation warnings while remote data is being fetched. +3. **Timestamp Comparison Selects a Whole File** + - **Compare time and take newer** compares modification timestamps and selects one whole-file version. It does not merge file contents. Before choosing this policy for a Vault with existing work, preserve valuable local and remote edits and review [Recovery and flag files](../recovery.md#first-choose-the-authoritative-copy). --- @@ -33,11 +33,11 @@ Fast Setup leverages several backend optimisations to make the retrieval fast, s You will be prompted to choose how the retrieved remote data will interact with your existing local files: 1. **Compare time and take newer (newer-wins)** - - Compares the modified time of files and accepts the newer version. - - **Recommended if:** You have been using Self-hosted LiveSync and have made changes on multiple devices that you want to merge. + - Compares the modified time of files and selects the newer whole-file version; it does not combine edits from both versions. + - **Recommended if:** You have preserved both versions and deliberately want to select one according to its modification time. 2. **Overwrite all with remote files (remote-wins)** - Remote data is treated as the source of truth. - - **Recommended if:** You are setting up a brand new device with an empty or clean vault. + - **Recommended if:** You are setting up a new secondary device with an empty Vault. - *Warning: This will overwrite local files with remote files. Please ensure you have a backup of your local vault before proceeding.* 3. **Use the detailed flow (legacy)** - Switches back to the detailed, traditional setup wizard dialogues. diff --git a/utils/readme.md b/utils/readme.md index 8f0dca6f..7ef8e321 100644 --- a/utils/readme.md +++ b/utils/readme.md @@ -55,13 +55,13 @@ The generator defaults to `uri_mode=ephemeral`. The URI opens only during the cu ```sh export remote_type=s3 -export endpoint=https://objects.example.com -export access_key= -export secret_key= +export endpoint='https://objects.example.com' +export access_key='' +export secret_key='' export bucket=vault-data export region=auto export bucket_prefix=team-a # Optional -export LIVESYNC_PASSPHRASE= +export LIVESYNC_PASSPHRASE='' deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` @@ -71,7 +71,7 @@ Optional Object Storage variables are `use_custom_request_handler`, `force_path_ ```sh export remote_type=p2p -export LIVESYNC_PASSPHRASE= +export LIVESYNC_PASSPHRASE='' deno run --minimum-dependency-age=0 --config=./flyio/deno.jsonc --frozen --lock=./flyio/deno.lock --allow-env ./setup/generate_setup_uri.ts ``` From 2befb2afdb186f7be6054720e36f11c84d8f3c7a Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 13:14:38 +0000 Subject: [PATCH 58/68] Index documentation and clarify verification entry points --- AGENTS.md | 6 ++++ README.md | 2 ++ devs.md | 4 ++- docs/README.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 docs/README.md diff --git a/AGENTS.md b/AGENTS.md index 9e9529ec..93172ba0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,12 @@ When working on this repository (writing code, comments, documentation, or commits), you MUST follow these guidelines to maintain consistency. +## Documentation Navigation and Maintenance + +- Use the [documentation index](docs/README.md) to select the relevant implementation reference, ADR, proposal, or historical record. The required references below still apply. +- When implementation or publication changes a recorded status, update its publication note and supporting references in the same change. Preserve the original decision date, reviewed-version baseline, and remaining follow-ups. +- Mark superseded or frozen documents at their entry point and link to the maintained reference. Keep historical evidence distinguishable from current operating instructions, and preserve existing links when reorganising documentation. + ## Required Reference Files Before making changes to documentation, user-facing text, or settings: diff --git a/README.md b/README.md index 8bdbacef..3503eb81 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Self-hosted LiveSync [Japanese docs](./README_ja.md) - [Chinese docs](./README_cn.md). +Find setup guides, troubleshooting, technical references, design decisions, and historical material in the [documentation index](docs/README.md). + Self-hosted LiveSync is a community-developed synchronisation plug-in available on all Obsidian-compatible platforms. It leverages robust server solutions such as CouchDB or object storage systems (e.g., MinIO, S3, R2, etc.) to ensure reliable data synchronisation. diff --git a/devs.md b/devs.md index a9279f14..5fdbbb2b 100644 --- a/devs.md +++ b/devs.md @@ -1,5 +1,7 @@ # Self-hosted LiveSync Development Guide +Use the [documentation index](docs/README.md) to find implementation references, architectural decisions, proposals, and historical material for the area being changed. + ## Project Overview Self-hosted LiveSync is an Obsidian plugin for synchronising vaults across devices using CouchDB, MinIO/S3, or peer-to-peer WebRTC. The codebase uses a modular architecture with TypeScript, Svelte, and PouchDB. @@ -78,7 +80,7 @@ Regression tests remain in the suite owned by the implementation under test. Plu - **CLI E2E** (`src/apps/cli/testdeno/`): Host-independent consumer workflows. The canonical Compose P2P suite covers ordinary two-peer synchronisation, replacement of the current Replicator followed by transfer with the same peer, and explicit relay disconnection followed by paused and resumed reconnection. Its lifecycle entry point is included only in the Docker test build and does not add a public CLI command. Run `npm run test:e2e:cli` for the ordinary suite or `npm run test:e2e:cli:p2p` for P2P validation. - **Self-hosted setup tools** (`utils/couchdb/`, `utils/setup/`, and `utils/flyio/`): Deno contract tests consume the exact locked Commonlib registry package, verify current CouchDB, Object Storage, and random-room P2P Setup URI defaults and remote profiles, and keep CouchDB administration separate from package-owned LiveSync database-version negotiation. `unit-ci` also provisions a real temporary CouchDB database and verifies its version document against the installed Commonlib package. Run `npm run test:setup-tools` for the local contract gate. -- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. Run focused scripts such as `npm run test:e2e:obsidian:two-vault-sync`, or use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and RustFS fixtures managed by the wrapper. +- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. After source changes, run the build-owning wrapper, for example `npm run test:e2e:obsidian:focused -- two-vault-sync`, with the required fixture already running. Direct scenario scripts reuse the existing bundle and are suitable for reruns only while that build remains unchanged. Use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and RustFS fixtures managed by the wrapper. - **Docker Services**: Service-backed tests use CouchDB and RustFS (S3). Canonical P2P validation owns its relay through the CLI Compose runner: diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..86d2afd0 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,92 @@ +# Documentation index + +Choose a document by the task and its status. User guides describe supported operations; implementation references describe contracts and their limits; ADRs record why a decision was made. Proposals and historical designs are grouped separately below. + +A document's original date and reviewed-version fields describe its baseline. Read its implementation or publication note before treating that baseline as the current release. A published feature does not by itself establish that every proposal or validation step in its design is complete. + +## Setup and everyday use + +| Task | Start here | +|---|---| +| Choose a synchronisation method | [Project overview and setup workflows](../README.md#setup-workflows) | +| Set up CouchDB clients | [CouchDB server setup](setup_own_server.md), then [Quick setup](quick_setup.md) | +| Choose a packaged CouchDB deployment | [Docker stack](../docker/README.md), [Fly.io](setup_flyio.md), or [IBM Cloudant](setup_cloudant.md) | +| Set up S3-compatible storage | [Object Storage Setup](setup_object_storage.md) | +| Set up peer-to-peer synchronisation | [P2P communication model](p2p.md), then [Peer-to-Peer Setup](setup_p2p.md) | +| Diagnose P2P connectivity | [Browser-assisted check](../src/apps/webpeer/README.md#p2p-connection-check), [network tips](tips/p2p-sync-tips.md), and [self-hosted TURN](../docker/coturn/README.md) | +| Understand a setting or command | [Settings reference](settings.md) | +| Investigate or recover from a problem | [Troubleshooting](troubleshooting.md), then the relevant [recovery procedure](recovery.md) | +| Prepare an issue report | [Issue reporting](to_issue_reporting.md) | + +The project overview links to Japanese and Chinese documentation. + +## Implementation and technical references + +Use these references for the area being changed, together with its relevant ADR. Check the document's stated scope and validation limits before relying on a guarantee. + +| Area | References | +|---|---| +| Overall synchronisation and terminology | [Technical overview](tech_info.md), [glossary](glossary.md), and [terminology conventions](terms.md) | +| Metadata, Chunks, and database representations | [Data structures](datastructure.md) | +| Conflict resolution, provenance, and history restoration | [Conflict and provenance contract](specs_conflict_resolution.md), [Document History restoration](design_docs/document_history_revision_restoration.md), and [local save completion](adr/2026_10_local_file_save_completion_and_batching.md) | +| Replicator and host boundaries | [Replicator architecture snapshot](design_docs/replicator_architecture.md) and [service feature and legacy Module boundaries](design_docs/service_feature_and_legacy_module_boundaries.md) | +| Chunk retrieval and maintenance | [Chunk retrieval and waiting](design_docs/chunk_retrieval_and_waiting.md), [Garbage Collection V3](specs_garbage_collection.md), and [Data Compression](specs_data_compression.md) | +| Document identity and validation | [Unicode path identity](design_docs/unicode_path_identity.md), [configurable ID derivation](design_docs/configurable_id_derivation.md), [Metadata ID validation and repair](design_docs/metadata_document_id_validation_and_repair.md), and [path component compatibility](design_docs/path_component_length_compatibility.md) | +| Settings compatibility and profiles | [Tweak compatibility and recovery](design_docs/tweak_compatibility.md) and [remote configuration management](design_docs_of_remote_configurations.md) | +| Encryption and sharing | [Metadata encryption](design_docs/internal_metadata_encryption.md) and [time-bound Setup URIs](design_docs/time_bound_setup_uri.md) | +| P2P connection preparation | [TURN connection settings](design_docs/renewable_turn_credentials.md) | +| Journal storage internals | [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) | + +The installed `@vrtmrz/livesync-commonlib` package is authoritative for shared runtime contracts. Use the exact dependency selected by [package.json](../package.json) and its lockfile when checking behaviour. The [Commonlib documentation](https://github.com/vrtmrz/livesync-commonlib/tree/main/docs) is maintained in that repository; this index does not replace its package contracts. + +## Architectural decisions + +Read the decisions relevant to the change, rather than treating the whole ADR directory as a prerequisite. A decision's acceptance, its implementation, its release, and its remaining verification work are separate facts. + +| Decision area | Records | +|---|---| +| Shared package boundary | [Common library package boundary](adr/2026_07_common_library_package_boundary.md) | +| Replicator capabilities and migration | [Core contract](adr/2026_08_replicator_capabilities_01_core_contract.md), [P2P service lifecycle](adr/2026_08_replicator_capabilities_02_p2p_service_lifecycle.md), and [migration plan](adr/2026_08_replicator_capabilities_03_migration_plan.md) | +| Remote resources and Fast Setup | [CouchDB connection ownership](adr/2026_08_couchdb_remote_connection_ownership.md), [Fast Fetch eligibility](adr/2026_08_fast_fetch_transport_eligibility.md), and [persistence and completion](adr/2026_08_fast_fetch_persistence_and_completion.md) | +| P2P lifecycle and setup | [Transport lifecycle](adr/2026_07_p2p_transport_lifecycle.md), [connection check](adr/2026_07_p2p_connection_check.md), [profile-aware onboarding](adr/2026_07_multiple_remote_onboarding.md), and [compatibility controls](adr/2026_08_p2p_transport_compatibility.md) | +| Settings interface integration | [Declarative settings adapter](adr/2026_08_declarative_settings_adapter.md) | +| Saving and path identity | [Local file save completion and batching](adr/2026_10_local_file_save_completion_and_batching.md) and [Unicode path identity](adr/2026_10_unicode_path_identity.md) | +| Activity and Chunk arrival | [Bounded remote activity](adr/2026_07_bounded_remote_activity.md) and [Chunk arrival quiescence](adr/2026_07_chunk_arrival_quiescence.md) | +| Compatibility and validation | [Release notes and database compatibility gates](adr/2026_07_release_notes_and_database_compatibility.md) and [real Obsidian E2E](adr/2026_06_real_obsidian_e2e.md) | + +## Proposals and incomplete explanations + +These records are useful for design work. Their proposed representation or behaviour is not a current runtime contract. + +| Document | Recorded state | +|---|---| +| [Tiered Chunk Storage with Live Compaction](design_docs/tired_chunk_pack.md) | In design | +| [Tiered Chunk Storage for Bucket Sync](design_docs/tired_chunk_pack_bucket.md) | In design | +| [Design intent for Metadata and Chunks](design_docs/intention_of_chunks.md) | Incomplete explanation; use the data-structure reference for current representations | +| [Feature maturity for 1.0](adr/2026_07_feature_maturity_for_1_0.md) | Proposed; publication of 1.0 alone does not establish formal acceptance of this record | + +## Historical and retired designs + +These documents preserve earlier motivations, approaches, or results. Follow their maintained replacement for current operations. + +| Historical material | Current starting point | +|---|---| +| [Original Tweak-sharing proposal](design_docs_of_sharing_tweak_value.md) | [Tweak compatibility and recovery](design_docs/tweak_compatibility.md) | +| [First Journal Sync design](design_docs_of_journalsync.md) | [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) | +| [Eden design and operational results](design_docs_of_keep_newborn_chunks.md#results-from-actual-operation) | The document records its deprecated, frozen status; use [Data structures](datastructure.md) for representations still accepted by current code | +| [Withdrawn Chunk Aggregation by Prefix](design_docs/chunk_aggregation_by_prefix.md) | Retained as a withdrawn proposal, not an implemented storage layout | + +Dated measurements, including the [Data Compression benchmark](specs_data_compression.md#reproducible-benchmark), remain evidence for their recorded revision and environment. A current rerun is a separate measurement. + +## Development, validation, and release history + +- [Development guide](../devs.md): build, module ownership, and release workflow. +- [Contributing](../CONTRIBUTING.md): contribution and documentation checks. +- Independent applications: [CLI](../src/apps/cli/README.md), [WebApp](../src/apps/webapp/README.md), and [WebPeer](../src/apps/webpeer/README.md). +- [Setup utilities](../utils/readme.md): provisioning and Setup URI generation. +- [Review Harness](review_harness.md): bounded real-device observations and performance measurements. +- [Test procedures](../test/README.md): choose the suite which owns the boundary being changed. +- [Change notes](../updates.md): current release and unreleased changes. +- Published release archives: [1.0](releases/1.0.md), [1.0 previews](releases/1.0-previews.md), [0.25](releases/0.25.md), and [earlier releases](releases/legacy.md). + +When a feature ships or a design is replaced, update its implementation note and this index if its role changes. Preserve the original decision and measurement history, state the maintained replacement, and keep existing links working. From 68a232c6dfd8456706ba26615f523def10fd753c Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 27 Oct 2026 17:21:44 +0000 Subject: [PATCH 59/68] Move status customisation details out of the project overview --- README.md | 88 +++++++------------------------------ docs/README.md | 1 + docs/settings.md | 2 + docs/tips/status-display.md | 54 +++++++++++++++++++++++ 4 files changed, 72 insertions(+), 73 deletions(-) create mode 100644 docs/tips/status-display.md diff --git a/README.md b/README.md index 3503eb81..279558b8 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ # Self-hosted LiveSync [Japanese docs](./README_ja.md) - [Chinese docs](./README_cn.md). -Find setup guides, troubleshooting, technical references, design decisions, and historical material in the [documentation index](docs/README.md). - - Self-hosted LiveSync is a community-developed synchronisation plug-in available on all Obsidian-compatible platforms. It leverages robust server solutions such as CouchDB or object storage systems (e.g., MinIO, S3, R2, etc.) to ensure reliable data synchronisation. Additionally, it supports peer-to-peer synchronisation using WebRTC, enabling devices to exchange notes without a central data-storage server. A signalling relay is still required for peer discovery. See [How peer-to-peer synchronisation works](./docs/p2p.md). +Find setup guides, troubleshooting, technical references, design decisions, and historical material in the [documentation index](docs/README.md). + ![obsidian_live_sync_demo](https://user-images.githubusercontent.com/45774780/137355323-f57a8b09-abf2-4501-836c-8cb7d2ff24a3.gif) >[!IMPORTANT] @@ -36,39 +35,23 @@ This plug-in may be particularly useful for researchers, engineers, and develope ## How to Use -### 3-minute setup - CouchDB on fly.io - -**Recommended for beginners** - -[![LiveSync Setup onto Fly.io SpeedRun 2024 using Google Colab](https://img.youtube.com/vi/7sa_I1832Xc/0.jpg)](https://www.youtube.com/watch?v=7sa_I1832Xc) - -1. [Set up CouchDB on fly.io](docs/setup_flyio.md) -2. Configure plug-in in [Quick Setup](docs/quick_setup.md) - ### Setup workflows Choose a synchronisation method, prepare its server where required, then follow the corresponding client setup: -1. CouchDB - 1. Prepare the server: - - [Set up your own CouchDB server](docs/setup_own_server.md). - - [Set up CouchDB on fly.io](docs/setup_flyio.md). - 2. Configure the clients by following [CouchDB Quick Setup](docs/quick_setup.md). -2. Object Storage - 1. Prepare the server. A maintained MinIO server installation guide is not currently available here, so set up an S3-compatible service or server of your choice. - 2. Configure the clients by following [Object Storage Setup](docs/setup_object_storage.md). -3. Peer-to-Peer - 1. No central data-storage server is required. The project's public signalling relay requires no server provisioning; controlled deployments can provide another compatible relay. - 2. Configure the clients by following [Peer-to-Peer Setup](docs/setup_p2p.md). +| Method | What you need | Setup guides | +| --- | --- | --- | +| CouchDB | A CouchDB server you operate or trust. | [Own server](docs/setup_own_server.md), [Docker stack](docker/README.md), [Fly.io](docs/setup_flyio.md), or [IBM Cloudant](docs/setup_cloudant.md); then [Quick Setup](docs/quick_setup.md). | +| Object Storage | An S3-compatible service or server. | [Object Storage Setup](docs/setup_object_storage.md). | +| Peer-to-Peer | A signalling relay and an online device containing the data you need. | [P2P communication model](docs/p2p.md), then [Peer-to-Peer Setup](docs/setup_p2p.md). | -Each workflow establishes ordinary note synchronisation on the first device, generates a Setup URI for each additional device from that working device, and verifies synchronisation in both directions. +The project's public P2P signalling relay requires no server provisioning; controlled deployments can provide another compatible relay. A maintained MinIO server installation guide is not currently available here, so Object Storage users need to prepare their chosen S3-compatible service or server separately. -> [!TIP] -> Fly.io is no longer free. Fortunately, we can still use IBM Cloudant despite some limitations. Refer to [Set up IBM Cloudant](docs/setup_cloudant.md). -> We can also use peer-to-peer synchronisation without a central data-storage server; a signalling relay is still used for peer discovery. Alternatively, cheap object storage like Cloudflare R2 can be used for free. -> However, most importantly, we can use a server that we trust. Therefore, please set up your own server. -> CouchDB can also be run on a Raspberry Pi (please be mindful of your server's security). +Each workflow establishes ordinary note synchronisation on the first device, generates a Setup URI for each additional device from that working device, and verifies synchronisation in both directions. The linked hosting guides explain provider-specific setup and costs. + + +For the Fly.io video walkthrough, see [automated setup](docs/setup_flyio.md#a-very-automated-setup). ### Third-party managed CouchDB hosting @@ -79,52 +62,11 @@ If you would rather not set up and maintain a server yourself, Zenith Hosting of ## Information in the Status Bar -Synchronisation status is shown in the status bar with the following icons. + -- Activity Indicator - - 📲 A finite remote operation is in progress - - 🌐N Approximate remote requests currently in progress -- Status - - ⏚ī¸ Stopped - - ⏊ Comparing CouchDB change history, including entries already held locally - - 💤 LiveSync enabled. Waiting for changes - - ⚡ī¸ Synchronisation in progress - - ⚠ An error occurred -- Statistical Indicators - - ↑ Uploaded chunks and metadata - - ↓ Downloaded chunks and metadata -- Progress Indicators - - đŸ“Ĩ Unprocessed transferred items - - 📄 Working database operation - - 💾 Working write storage processes - - âŗ Working read storage processes - - đŸ›Ģ Pending read storage processes - - đŸ“Ŧ Batched read storage processes - - ⚙ī¸ Working or pending storage processes for hidden files - - 🛄 Pending initial on-demand chunk requests - - 🔁 Pending chunk retries, including retry delays - - 🔌 Working customisation items (configuration, snippets, and plug-ins) +See the [status display guide](docs/tips/status-display.md) for icon meanings, progress counters, and [CSS customisation](docs/tips/status-display.md#customising-status-items-with-css-snippets). -To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files. - -When CouchDB change history is being compared, `(⏊ 1200) ↑ 0 ↓ 0` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It appears before the upload count, disappears after three seconds without a change, and reappears when the count changes. The ⏊ status icon follows comparison activity independently of that timer. - -### Customising status items with CSS snippets - -The status bar and editor overlay use individual spans with the common `livesync-stat-` class prefix. CSS snippets can hide or style the items we need. For example, this hides the comparison count and P2P summary: - -```css -.livesync-stat-ff, -.livesync-stat-p2p { - display: none; -} -``` - -The item classes include `livesync-stat-network`, `livesync-stat-sync`, `livesync-stat-sent`, `livesync-stat-push-remaining`, `livesync-stat-arrived`, `livesync-stat-pull-remaining`, `livesync-stat-ff`, and `livesync-stat-p2p`. File-processing items use `livesync-stat-file-processing`, `livesync-stat-file-pending`, and `livesync-stat-file-batched`; queue items use `livesync-stat-queue-` followed by `replication`, `database`, `storage`, `chunk-initial`, `chunk-retrying`, `plugin-scan`, `hidden-files`, or `conflicts`. The skipped-file indicator uses `livesync-stat-file-skip`. - -All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count during an active period, reducing movement as counts decrease. The editor overlay uses wider gaps between items, while remaining counts stay beside their transfer totals. - -Inactive items receive `livesync-stat-inactive` instead of the HTML `hidden` attribute. The default CSS hides them immediately. They retain their last text, numeric width, and position amongst the other items so that CSS snippets can customise how they disappear. The numeric width starts afresh when an item becomes active again. +Allow pending file operations to finish before closing Obsidian, especially after deleting or renaming files. ## Tips and Troubleshooting - If you want a faster and simpler initial replication when setting up subsequent devices, see the [Fast Setup Guide](docs/tips/fast-setup.md). diff --git a/docs/README.md b/docs/README.md index 86d2afd0..79d97e6d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,7 @@ A document's original date and reviewed-version fields describe its baseline. Re | Set up peer-to-peer synchronisation | [P2P communication model](p2p.md), then [Peer-to-Peer Setup](setup_p2p.md) | | Diagnose P2P connectivity | [Browser-assisted check](../src/apps/webpeer/README.md#p2p-connection-check), [network tips](tips/p2p-sync-tips.md), and [self-hosted TURN](../docker/coturn/README.md) | | Understand a setting or command | [Settings reference](settings.md) | +| Read or customise synchronisation status | [Status indicators and CSS snippets](tips/status-display.md) | | Investigate or recover from a problem | [Troubleshooting](troubleshooting.md), then the relevant [recovery procedure](recovery.md) | | Prepare an issue report | [Issue reporting](to_issue_reporting.md) | diff --git a/docs/settings.md b/docs/settings.md index 135351aa..359b109c 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -161,6 +161,8 @@ Following panes will be shown when you enable this setting. ### 1. Appearance +For icon meanings, progress counters, and CSS snippets, see the [status display guide](tips/status-display.md). + #### Display Language Setting key: displayLanguage diff --git a/docs/tips/status-display.md b/docs/tips/status-display.md new file mode 100644 index 00000000..751d143e --- /dev/null +++ b/docs/tips/status-display.md @@ -0,0 +1,54 @@ +# Synchronisation status and CSS customisation + +Use [General Settings](../settings.md#2-general-settings) to choose whether synchronisation status appears in the status bar, inside the editor, or both. The following sections explain the indicators and how to customise their appearance. + +## Status indicators + +- Activity Indicator + - 📲 A finite remote operation is in progress + - 🌐N Approximate remote requests currently in progress +- Status + - ⏚ī¸ Stopped + - ⏊ Comparing CouchDB change history, including entries already held locally + - 💤 LiveSync enabled. Waiting for changes + - ⚡ī¸ Synchronisation in progress + - ⚠ An error occurred +- Statistical Indicators + - ↑ Uploaded chunks and metadata + - ↓ Downloaded chunks and metadata +- Progress Indicators + - đŸ“Ĩ Unprocessed transferred items + - 📄 Working database operation + - 💾 Working write storage processes + - âŗ Working read storage processes + - đŸ›Ģ Pending read storage processes + - đŸ“Ŧ Batched read storage processes + - ⚙ī¸ Working or pending storage processes for hidden files + - 🛄 Pending initial on-demand chunk requests + - 🔁 Pending chunk retries, including retry delays + - 🔌 Working customisation items (configuration, snippets, and plug-ins) + +To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files. + +## CouchDB comparison count + +When CouchDB change history is being compared, `(⏊ 1200) ↑ 0 ↓ 0` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It appears before the upload count, disappears after three seconds without a change, and reappears when the count changes. The ⏊ status icon follows comparison activity independently of that timer. + +## Customising status items with CSS snippets + +The status bar and editor overlay use individual spans with the common `livesync-stat-` class prefix. CSS snippets can hide or style individual items. For example, this hides the comparison count and P2P summary: + +```css +.livesync-stat-ff, +.livesync-stat-p2p { + display: none; +} +``` + +The item classes include `livesync-stat-network`, `livesync-stat-sync`, `livesync-stat-sent`, `livesync-stat-push-remaining`, `livesync-stat-arrived`, `livesync-stat-pull-remaining`, `livesync-stat-ff`, and `livesync-stat-p2p`. File-processing items use `livesync-stat-file-processing`, `livesync-stat-file-pending`, and `livesync-stat-file-batched`; queue items use `livesync-stat-queue-` followed by `replication`, `database`, `storage`, `chunk-initial`, `chunk-retrying`, `plugin-scan`, `hidden-files`, or `conflicts`. The skipped-file indicator uses `livesync-stat-file-skip`. + +All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count during an active period, reducing movement as counts decrease. The editor overlay uses wider gaps between items, while remaining counts stay beside their transfer totals. + +Inactive items receive `livesync-stat-inactive` instead of the HTML `hidden` attribute. The default CSS hides them immediately. They retain their last text, numeric width, and position amongst the other items so that CSS snippets can customise how they disappear. The numeric width starts afresh when an item becomes active again. + +See the [documentation index](../README.md) for setup and troubleshooting guides. From bd0d81d7a7111170fa099b100cb3c7adc76bbf50 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 04:25:05 +0000 Subject: [PATCH 60/68] Restore basic status icons and clarify P2P setup requirements --- README.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 279558b8..f49adbe5 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,13 @@ This plug-in may be particularly useful for researchers, engineers, and develope Choose a synchronisation method, prepare its server where required, then follow the corresponding client setup: -| Method | What you need | Setup guides | -| --- | --- | --- | -| CouchDB | A CouchDB server you operate or trust. | [Own server](docs/setup_own_server.md), [Docker stack](docker/README.md), [Fly.io](docs/setup_flyio.md), or [IBM Cloudant](docs/setup_cloudant.md); then [Quick Setup](docs/quick_setup.md). | -| Object Storage | An S3-compatible service or server. | [Object Storage Setup](docs/setup_object_storage.md). | -| Peer-to-Peer | A signalling relay and an online device containing the data you need. | [P2P communication model](docs/p2p.md), then [Peer-to-Peer Setup](docs/setup_p2p.md). | +| Method | What you need | Nice to have | Setup guides | +| --- | --- | --- | --- | +| CouchDB | A CouchDB server you operate or trust. | — | [Own server](docs/setup_own_server.md), [Docker stack](docker/README.md), [Fly.io](docs/setup_flyio.md), or [IBM Cloudant](docs/setup_cloudant.md); then [Quick Setup](docs/quick_setup.md). | +| Object Storage | An S3-compatible service or server. | — | [Object Storage Setup](docs/setup_object_storage.md). | +| Peer-to-Peer | Nothing. | Your own signalling server. | [P2P communication model](docs/p2p.md), then [Peer-to-Peer Setup](docs/setup_p2p.md). | -The project's public P2P signalling relay requires no server provisioning; controlled deployments can provide another compatible relay. A maintained MinIO server installation guide is not currently available here, so Object Storage users need to prepare their chosen S3-compatible service or server separately. +P2P can use the project's public signalling relay, so there is no server to provision. An optional self-hosted signalling server must be Nostr-compatible. A maintained MinIO server installation guide is not currently available here, so Object Storage users need to prepare their chosen S3-compatible service or server separately. Each workflow establishes ordinary note synchronisation on the first device, generates a Setup URI for each additional device from that working device, and verifies synchronisation in both directions. The linked hosting guides explain provider-specific setup and costs. @@ -62,9 +62,22 @@ If you would rather not set up and maintain a server yourself, Zenith Hosting of ## Information in the Status Bar +Synchronisation status is shown in the status bar and inside the editor by default. The main indicators are: + +| Indicator | Meaning | +| --- | --- | +| ⏚ī¸ | Stopped. | +| 🌀 | Starting synchronisation. | +| 💤 | LiveSync mode is waiting for changes. | +| ⚡ī¸ | Synchronisation is in progress. | +| ⏊ | Comparing CouchDB change history, including entries already held locally. | +| ⚠ | An error occurred. | +| ↑ / ↓ | Uploaded / downloaded Chunks and Metadata. | +| 📲 / 🌐N | A finite remote operation is in progress / approximate remote requests in progress. | + -See the [status display guide](docs/tips/status-display.md) for icon meanings, progress counters, and [CSS customisation](docs/tips/status-display.md#customising-status-items-with-css-snippets). +See the [status display guide](docs/tips/status-display.md) for detailed progress counters and [CSS customisation](docs/tips/status-display.md#customising-status-items-with-css-snippets). Allow pending file operations to finish before closing Obsidian, especially after deleting or renaming files. From b8dc4d0c0ba51569ece786ae6f0a00f606324ff6 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 04:31:03 +0000 Subject: [PATCH 61/68] List third-party hosting providers under a shared notice --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f49adbe5..55146577 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,15 @@ For the Fly.io video walkthrough, see [automated setup](docs/setup_flyio.md#a-ve ### Third-party managed CouchDB hosting > [!NOTE] -> The following is a third-party hosting option proposed by Zenith Hosting. It is not an official Self-hosted LiveSync service, and it is neither endorsed nor recommended by this project. +> The services below are operated by third parties. They are not official Self-hosted LiveSync services, and this project neither endorses nor recommends them. -If you would rather not set up and maintain a server yourself, Zenith Hosting offers a managed CouchDB server which this plug-in can be configured to connect to: [Zenith Hosting](https://zenith.hosting/host/obsidian-livesync). As with any hosted service, your data will reside on a server operated by a third party, so please consider whether that is acceptable for your vault before using it. +If you prefer a hosted CouchDB server, the following providers offer options for this plug-in: + +| Provider | Offering | +| --- | --- | +| [Zenith Hosting](https://zenith.hosting/host/obsidian-livesync) | Managed CouchDB hosting. | + +Your synchronisation data will reside on a server operated by the provider. Consider whether that is acceptable for your Vault, and review the provider's current terms, pricing, and data-handling policies before using the service. ## Information in the Status Bar From ec12e132827c2676542b9a57727fa3bcdec26f5f Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 05:01:42 +0000 Subject: [PATCH 62/68] Correct recovery guidance and setup credential encoding Preserve Vault and remote data before deleting a Fly.io instance. Build cluster setup JSON with jq and ConvertTo-Json so credential characters remain intact. Also include the starting status icon and clarify the CouchDB hosting category. --- docs/README.md | 2 +- docs/setup_flyio.md | 35 +++++++++++++++++++++++++++++------ docs/tips/status-display.md | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/README.md b/docs/README.md index 79d97e6d..9c5499f9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ A document's original date and reviewed-version fields describe its baseline. Re |---|---| | Choose a synchronisation method | [Project overview and setup workflows](../README.md#setup-workflows) | | Set up CouchDB clients | [CouchDB server setup](setup_own_server.md), then [Quick setup](quick_setup.md) | -| Choose a packaged CouchDB deployment | [Docker stack](../docker/README.md), [Fly.io](setup_flyio.md), or [IBM Cloudant](setup_cloudant.md) | +| Choose a CouchDB hosting option | [Docker stack](../docker/README.md), [Fly.io](setup_flyio.md), or [IBM Cloudant](setup_cloudant.md) | | Set up S3-compatible storage | [Object Storage Setup](setup_object_storage.md) | | Set up peer-to-peer synchronisation | [P2P communication model](p2p.md), then [Peer-to-Peer Setup](setup_p2p.md) | | Diagnose P2P connectivity | [Browser-assisted check](../src/apps/webpeer/README.md#p2p-connection-check), [network tips](tips/p2p-sync-tips.md), and [self-hosted TURN](../docker/coturn/README.md) | diff --git a/docs/setup_flyio.md b/docs/setup_flyio.md index 0b926b3f..fcc3c1ee 100644 --- a/docs/setup_flyio.md +++ b/docs/setup_flyio.md @@ -4,9 +4,7 @@ This is how to configure fly.io and CouchDB on it for Self-hosted LiveSync. > [!WARNING] -> It is **your** instance. In Obsidian, we have files locally. Hence, do not hesitate to destroy the remote database if you feel something have got weird. We can launch and switch to the new CouchDB instance anytime[^1]. -> -[^1]: Actually, I am always building the database for reproduction of the issue like so. +> Before replacing or deleting an existing CouchDB instance, back up every available Vault and the remote database, then [choose the authoritative copy](recovery.md#first-choose-the-authoritative-copy). Stop or suspend synchronisation on other devices while reviewing recovery options. Local files alone do not establish that every needed edit or revision has been preserved. > [!NOTE] > **What and why is the Fly.io?** @@ -172,7 +170,7 @@ Visit your newly deployed app at https://billowing-dawn-6619.fly.dev/ After the initial setup, CouchDB needs some more customisations to be used from Self-hosted LiveSync. It can be configured in browsers or by HTTP-REST APIs. -This section is set up using the REST API. +This section uses the REST API. The macOS and Linux commands require [jq](https://jqlang.org/download/) to encode credentials as JSON. Windows uses PowerShell's built-in `ConvertTo-Json`. 1. Prepare environment variables. @@ -198,13 +196,36 @@ $creds = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes( - Mac or Linux ```bash -curl -X POST "${couchHost}/_cluster_setup" -H "Content-Type: application/json" -d "{\"action\":\"enable_single_node\",\"username\":\"${couchUser}\",\"password\":\"${couchPwd}\",\"bind_address\":\"0.0.0.0\",\"port\":5984,\"singlenode\":true}" --user "${couchUser}:${couchPwd}" +clusterSetupJSON=$(jq -n \ + --arg username "$couchUser" \ + --arg password "$couchPwd" \ + '{ + action: "enable_single_node", + username: $username, + password: $password, + bind_address: "0.0.0.0", + port: 5984, + singlenode: true + }') && +curl -X POST "${couchHost}/_cluster_setup" \ + -H "Content-Type: application/json" \ + --data-binary "$clusterSetupJSON" \ + --user "${couchUser}:${couchPwd}" ``` - Windows ```powershell -iwr -UseBasicParsing -Method 'POST' -ContentType 'application/json; charset=utf-8' -Headers @{ 'Authorization' = 'Basic ' + $creds } "${couchHost}/_cluster_setup" -Body "{""action"":""enable_single_node"",""username"":""${couchUser}"",""password"":""${couchPwd}"",""bind_address"":""0.0.0.0"",""port"":5984,""singlenode"":true}" +$clusterSetupJSON = @{ + action = 'enable_single_node' + username = $couchUser + password = $couchPwd + bind_address = '0.0.0.0' + port = 5984 + singlenode = $true +} | ConvertTo-Json -Compress + +iwr -UseBasicParsing -Method 'POST' -ContentType 'application/json; charset=utf-8' -Headers @{ 'Authorization' = 'Basic ' + $creds } "${couchHost}/_cluster_setup" -Body $clusterSetupJSON ``` Note: if the response code is not 200. We have to retry the request once again. @@ -249,6 +270,8 @@ Please refer to [Configure CouchDB manually on the first device](./quick_setup.m ## Delete the Instance +Before deleting the instance, preserve the data you still need and follow [Recovery and flag files](recovery.md#first-choose-the-authoritative-copy). + If you want to delete the CouchDB instance, you can do that in [fly.io Dashboard](https://fly.io/dashboard/personal) If you have done with [B. Scripted Setup](#b-scripted-setup), we can use [delete-server.sh](../utils/readme.md#flyio-deployment). diff --git a/docs/tips/status-display.md b/docs/tips/status-display.md index 751d143e..d2cdf824 100644 --- a/docs/tips/status-display.md +++ b/docs/tips/status-display.md @@ -9,6 +9,7 @@ Use [General Settings](../settings.md#2-general-settings) to choose whether sync - 🌐N Approximate remote requests currently in progress - Status - ⏚ī¸ Stopped + - 🌀 Starting synchronisation - ⏊ Comparing CouchDB change history, including entries already held locally - 💤 LiveSync enabled. Waiting for changes - ⚡ī¸ Synchronisation in progress From 22d10777fc71a94ade221a4aa061594a521ac745 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 05:57:40 +0000 Subject: [PATCH 63/68] Align Customisation Sync maturity with the supported feature policy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55146577..c75dd253 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Find setup guides, troubleshooting, technical references, design decisions, and - Use open-source solutions for the server. - Compatible solutions are supported. - Support end-to-end encryption. -- Synchronise settings, snippets, themes, and plug-ins via [Customisation Sync (Beta)](docs/settings.md#6-customisation-sync-advanced) or [Hidden File Sync](docs/tips/hidden-file-sync.md). +- Synchronise settings, snippets, themes, and plug-ins via [Customisation Sync (Advanced)](docs/settings.md#6-customisation-sync-advanced) or [Hidden File Sync](docs/tips/hidden-file-sync.md). - Enable supported, opt-in WebRTC peer-to-peer synchronisation. - No central data-storage server is required, but a signalling relay is still required for peer discovery. - At least one device containing the required data must be online while another device synchronises. From fac53afe84153a0dc12cc0e9073620fc52543ffe Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 07:41:49 +0000 Subject: [PATCH 64/68] Restore self-hosting as the primary setup path --- README.md | 10 +++++++--- docs/README.md | 5 +++-- docs/setup_flyio.md | 2 ++ docs/setup_own_server.md | 18 ++++++++++++------ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c75dd253..1fd6eef7 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,13 @@ This plug-in may be particularly useful for researchers, engineers, and develope ### Setup workflows +Self-hosting lets us choose a server we trust and manage it ourselves. For CouchDB, please set up and maintain your own instance. CouchDB can run on a Raspberry Pi, another server you operate, or infrastructure such as Fly.io. Please keep your server secure. + Choose a synchronisation method, prepare its server where required, then follow the corresponding client setup: | Method | What you need | Nice to have | Setup guides | | --- | --- | --- | --- | -| CouchDB | A CouchDB server you operate or trust. | — | [Own server](docs/setup_own_server.md), [Docker stack](docker/README.md), [Fly.io](docs/setup_flyio.md), or [IBM Cloudant](docs/setup_cloudant.md); then [Quick Setup](docs/quick_setup.md). | +| CouchDB | A CouchDB server. | — | [Set up your own server](docs/setup_own_server.md), [Docker stack](docker/README.md), or [Fly.io](docs/setup_flyio.md); then [Quick Setup](docs/quick_setup.md). | | Object Storage | An S3-compatible service or server. | — | [Object Storage Setup](docs/setup_object_storage.md). | | Peer-to-Peer | Nothing. | Your own signalling server. | [P2P communication model](docs/p2p.md), then [Peer-to-Peer Setup](docs/setup_p2p.md). | @@ -56,14 +58,16 @@ For the Fly.io video walkthrough, see [automated setup](docs/setup_flyio.md#a-ve ### Third-party managed CouchDB hosting > [!NOTE] -> The services below are operated by third parties. They are not official Self-hosted LiveSync services, and this project neither endorses nor recommends them. +> The services below are operated by third parties. They are not official Self-hosted LiveSync services, and this project neither endorses nor recommends them. The providers proposed the listings in the table. -If you prefer a hosted CouchDB server, the following providers offer options for this plug-in: +If you would rather not set up and maintain a server yourself, the following providers offer managed CouchDB servers which this plug-in can be configured to connect to: | Provider | Offering | | --- | --- | | [Zenith Hosting](https://zenith.hosting/host/obsidian-livesync) | Managed CouchDB hosting. | +[IBM Cloudant](docs/setup_cloudant.md) is another managed alternative, with some limitations. + Your synchronisation data will reside on a server operated by the provider. Consider whether that is acceptable for your Vault, and review the provider's current terms, pricing, and data-handling policies before using the service. ## Information in the Status Bar diff --git a/docs/README.md b/docs/README.md index 9c5499f9..a43a43ea 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,8 +9,9 @@ A document's original date and reviewed-version fields describe its baseline. Re | Task | Start here | |---|---| | Choose a synchronisation method | [Project overview and setup workflows](../README.md#setup-workflows) | -| Set up CouchDB clients | [CouchDB server setup](setup_own_server.md), then [Quick setup](quick_setup.md) | -| Choose a CouchDB hosting option | [Docker stack](../docker/README.md), [Fly.io](setup_flyio.md), or [IBM Cloudant](setup_cloudant.md) | +| Run your own CouchDB server | [Own server](setup_own_server.md), [Docker stack](../docker/README.md), or [Fly.io](setup_flyio.md) | +| Configure CouchDB clients | [Quick setup](quick_setup.md) | +| Consider third-party managed services | [Alternatives to maintaining your own server](../README.md#third-party-managed-couchdb-hosting) | | Set up S3-compatible storage | [Object Storage Setup](setup_object_storage.md) | | Set up peer-to-peer synchronisation | [P2P communication model](p2p.md), then [Peer-to-Peer Setup](setup_p2p.md) | | Diagnose P2P connectivity | [Browser-assisted check](../src/apps/webpeer/README.md#p2p-connection-check), [network tips](tips/p2p-sync-tips.md), and [self-hosted TURN](../docker/coturn/README.md) | diff --git a/docs/setup_flyio.md b/docs/setup_flyio.md index fcc3c1ee..475e7e54 100644 --- a/docs/setup_flyio.md +++ b/docs/setup_flyio.md @@ -3,6 +3,8 @@ This is how to configure fly.io and CouchDB on it for Self-hosted LiveSync. +The CouchDB instance is yours to manage. You can configure it yourself or set up another instance and switch to it, following the preservation and recovery guidance below. + > [!WARNING] > Before replacing or deleting an existing CouchDB instance, back up every available Vault and the remote database, then [choose the authoritative copy](recovery.md#first-choose-the-authoritative-copy). Stop or suspend synchronisation on other devices while reviewing recovery options. Local files alone do not establish that every needed edit or revision has been preserved. diff --git a/docs/setup_own_server.md b/docs/setup_own_server.md index cc1994e1..4f59c95b 100644 --- a/docs/setup_own_server.md +++ b/docs/setup_own_server.md @@ -15,7 +15,7 @@ - [4. Starting Docker Compose in background](#4-run-the-docker-compose-file-in-the-background) - [C. Install CouchDB directly](#c-install-couchdb-directly) - [2. Run couchdb-init.sh for initialise](#2-run-couchdb-initsh-for-initialise) - - [3. Expose CouchDB to the Internet](#3-expose-couchdb-to-the-internet) + - [3. Make CouchDB reachable from your devices](#3-make-couchdb-reachable-from-your-devices) - [4. Client Setup](#4-client-setup) - [1. Generate the setup URI on a desktop device or server](#1-generate-the-setup-uri-on-a-desktop-device-or-server) - [2. Setup Self-hosted LiveSync to Obsidian](#2-setup-self-hosted-livesync-to-obsidian) @@ -143,12 +143,16 @@ If you are using Docker Compose and the above command does not work or displays curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | hostname='http://:5984' username='' password='' database=obsidiannotes bash ``` -## 3. Expose CouchDB to the Internet + -- You can skip this instruction if you using only in intranet and only with desktop devices. - - For mobile devices, Obsidian requires a valid SSL certificate. Usually, it needs exposing the internet. +## 3. Make CouchDB reachable from your devices -For temporary testing, create a Cloudflare Quick Tunnel by following the [Quick Tunnels instructions](https://developers.cloudflare.com/tunnel/get-started/quick-tunnels/). It prints a random hostname which changes each time you create a tunnel, and the URL stops working when the `cloudflared` process stops. Quick Tunnels are intended for testing and development, not production. See the [Cloudflare Tunnel guide](https://developers.cloudflare.com/tunnel/get-started/) for durable use. +- You can skip this step if you use CouchDB only on a trusted local network with desktop devices. + - For mobile devices, Obsidian requires HTTPS with a valid certificate. HTTPS does not require making CouchDB publicly accessible. + +Choose how your devices reach your CouchDB server. You can use a [reverse proxy you operate](#reverse-proxies), private-network access, or a tunnel service of your choice. The [Docker profiles](../docker/README.md#profile-details) provide examples. + +Cloudflare is one optional choice. The following example uses a [Quick Tunnel](https://developers.cloudflare.com/tunnel/get-started/quick-tunnels/) for temporary testing. It prints a random hostname which changes each time you create a tunnel, and the URL stops working when the `cloudflared` process stops. Quick Tunnels are intended for testing and development, not production. For ongoing use, configure your chosen connection method; the [Cloudflare Tunnel guide](https://developers.cloudflare.com/tunnel/get-started/) applies if you choose that provider. ``` cloudflared tunnel --url http://localhost:5984 @@ -178,8 +182,10 @@ Use the URL printed by your tunnel as the server address. Keep the tunnel proces The `LIVESYNC_USER` and `LIVESYNC_PASS` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account. +Use the server URL through which your devices reach CouchDB. For the temporary Quick Tunnel example above, use the URL printed by your own running tunnel. + ```bash -export LIVESYNC_HOST='' +export LIVESYNC_HOST='' export LIVESYNC_DB=obsidiannotes export LIVESYNC_USER='' export LIVESYNC_PASS='' From d15ad5c28705614407fb8ad802aa6fa7e194130e Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 08:23:10 +0000 Subject: [PATCH 65/68] Clarify CouchDB setup options and community hosting listings --- README.md | 10 ++++++---- docs/README.md | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1fd6eef7..36f53ee3 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,12 @@ Choose a synchronisation method, prepare its server where required, then follow | Method | What you need | Nice to have | Setup guides | | --- | --- | --- | --- | -| CouchDB | A CouchDB server. | — | [Set up your own server](docs/setup_own_server.md), [Docker stack](docker/README.md), or [Fly.io](docs/setup_flyio.md); then [Quick Setup](docs/quick_setup.md). | +| CouchDB | A CouchDB server. | — | [Set up your own server](docs/setup_own_server.md), [Docker stack](docker/README.md), [Fly.io](docs/setup_flyio.md), or [IBM Cloudant (managed)](docs/setup_cloudant.md); then [Quick Setup](docs/quick_setup.md). | | Object Storage | An S3-compatible service or server. | — | [Object Storage Setup](docs/setup_object_storage.md). | | Peer-to-Peer | Nothing. | Your own signalling server. | [P2P communication model](docs/p2p.md), then [Peer-to-Peer Setup](docs/setup_p2p.md). | +With your own server, you manage both the host and CouchDB; Docker is one way to deploy it. On Fly.io, you manage your CouchDB instance on hosted infrastructure. IBM Cloudant manages the database service. + P2P can use the project's public signalling relay, so there is no server to provision. An optional self-hosted signalling server must be Nostr-compatible. A maintained MinIO server installation guide is not currently available here, so Object Storage users need to prepare their chosen S3-compatible service or server separately. Each workflow establishes ordinary note synchronisation on the first device, generates a Setup URI for each additional device from that working device, and verifies synchronisation in both directions. The linked hosting guides explain provider-specific setup and costs. @@ -55,7 +57,9 @@ Each workflow establishes ordinary note synchronisation on the first device, gen For the Fly.io video walkthrough, see [automated setup](docs/setup_flyio.md#a-very-automated-setup). -### Third-party managed CouchDB hosting + + +### Hosting services proposed by the community > [!NOTE] > The services below are operated by third parties. They are not official Self-hosted LiveSync services, and this project neither endorses nor recommends them. The providers proposed the listings in the table. @@ -66,8 +70,6 @@ If you would rather not set up and maintain a server yourself, the following pro | --- | --- | | [Zenith Hosting](https://zenith.hosting/host/obsidian-livesync) | Managed CouchDB hosting. | -[IBM Cloudant](docs/setup_cloudant.md) is another managed alternative, with some limitations. - Your synchronisation data will reside on a server operated by the provider. Consider whether that is acceptable for your Vault, and review the provider's current terms, pricing, and data-handling policies before using the service. ## Information in the Status Bar diff --git a/docs/README.md b/docs/README.md index a43a43ea..95ad98d4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,9 +9,9 @@ A document's original date and reviewed-version fields describe its baseline. Re | Task | Start here | |---|---| | Choose a synchronisation method | [Project overview and setup workflows](../README.md#setup-workflows) | -| Run your own CouchDB server | [Own server](setup_own_server.md), [Docker stack](../docker/README.md), or [Fly.io](setup_flyio.md) | +| Choose a CouchDB setup | [Own server](setup_own_server.md), [Docker stack](../docker/README.md), [Fly.io](setup_flyio.md), or [IBM Cloudant (managed)](setup_cloudant.md) | | Configure CouchDB clients | [Quick setup](quick_setup.md) | -| Consider third-party managed services | [Alternatives to maintaining your own server](../README.md#third-party-managed-couchdb-hosting) | +| Find hosting services proposed by the community | [Third-party hosting listings](../README.md#hosting-services-proposed-by-the-community) | | Set up S3-compatible storage | [Object Storage Setup](setup_object_storage.md) | | Set up peer-to-peer synchronisation | [P2P communication model](p2p.md), then [Peer-to-Peer Setup](setup_p2p.md) | | Diagnose P2P connectivity | [Browser-assisted check](../src/apps/webpeer/README.md#p2p-connection-check), [network tips](tips/p2p-sync-tips.md), and [self-hosted TURN](../docker/coturn/README.md) | From f72f1011f2c321978a7af8f56f4264e5bbf9f3b4 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 08:50:29 +0000 Subject: [PATCH 66/68] Clarify requirements and first steps in the README --- README.md | 33 ++++++++++++++++++++++----------- docs/README.md | 2 +- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 36f53ee3..2ed9b552 100644 --- a/README.md +++ b/README.md @@ -35,28 +35,39 @@ This plug-in may be particularly useful for researchers, engineers, and develope ## How to Use -### Setup workflows +Self-hosting lets us choose a server we trust and manage it ourselves. Please set up and maintain your own server, and keep it secure. -Self-hosting lets us choose a server we trust and manage it ourselves. For CouchDB, please set up and maintain your own instance. CouchDB can run on a Raspberry Pi, another server you operate, or infrastructure such as Fly.io. Please keep your server secure. + -Choose a synchronisation method, prepare its server where required, then follow the corresponding client setup: +### What you need -| Method | What you need | Nice to have | Setup guides | -| --- | --- | --- | --- | -| CouchDB | A CouchDB server. | — | [Set up your own server](docs/setup_own_server.md), [Docker stack](docker/README.md), [Fly.io](docs/setup_flyio.md), or [IBM Cloudant (managed)](docs/setup_cloudant.md); then [Quick Setup](docs/quick_setup.md). | -| Object Storage | An S3-compatible service or server. | — | [Object Storage Setup](docs/setup_object_storage.md). | -| Peer-to-Peer | Nothing. | Your own signalling server. | [P2P communication model](docs/p2p.md), then [Peer-to-Peer Setup](docs/setup_p2p.md). | +| Method | What you need | Nice to have | +| --- | --- | --- | +| CouchDB | A CouchDB-compatible server. | — | +| Object Storage | An S3-compatible service or server. | — | +| Peer-to-Peer | Nothing. | Your own signalling server. | -With your own server, you manage both the host and CouchDB; Docker is one way to deploy it. On Fly.io, you manage your CouchDB instance on hosted infrastructure. IBM Cloudant manages the database service. +For a CouchDB-compatible server, choose a setup guide for your environment: -P2P can use the project's public signalling relay, so there is no server to provision. An optional self-hosted signalling server must be Nostr-compatible. A maintained MinIO server installation guide is not currently available here, so Object Storage users need to prepare their chosen S3-compatible service or server separately. +- [Your own server](docs/setup_own_server.md): run CouchDB on a Raspberry Pi or another server you manage. +- [Docker](docker/README.md): deploy CouchDB with the supplied Docker stack. +- [Fly.io](docs/setup_flyio.md): manage your own CouchDB instance on hosted infrastructure. +- [IBM Cloudant (managed)](docs/setup_cloudant.md): use a database service managed by IBM. -Each workflow establishes ordinary note synchronisation on the first device, generates a Setup URI for each additional device from that working device, and verifies synchronisation in both directions. The linked hosting guides explain provider-specific setup and costs. +The linked guides explain the setup and any provider-specific costs. For the Fly.io video walkthrough, see [automated setup](docs/setup_flyio.md#a-very-automated-setup). +P2P can use the project's public signalling relay, so there is no server to provision. See [How peer-to-peer synchronisation works](docs/p2p.md) for more about the connection. + +### Start synchronising + +1. Install and configure Self-hosted LiveSync on your first device using [Quick Setup](docs/quick_setup.md) for CouchDB, [Object Storage Setup](docs/setup_object_storage.md), or [Peer-to-Peer Setup](docs/setup_p2p.md). +2. Generate a Setup URI from that working device and follow the same guide to add your other devices. +3. Create test notes and confirm that they synchronise in both directions. + ### Hosting services proposed by the community diff --git a/docs/README.md b/docs/README.md index 95ad98d4..2938d7c1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ A document's original date and reviewed-version fields describe its baseline. Re | Task | Start here | |---|---| -| Choose a synchronisation method | [Project overview and setup workflows](../README.md#setup-workflows) | +| Get started | [What you need](../README.md#what-you-need), then [Start synchronising](../README.md#start-synchronising) | | Choose a CouchDB setup | [Own server](setup_own_server.md), [Docker stack](../docker/README.md), [Fly.io](setup_flyio.md), or [IBM Cloudant (managed)](setup_cloudant.md) | | Configure CouchDB clients | [Quick setup](quick_setup.md) | | Find hosting services proposed by the community | [Third-party hosting listings](../README.md#hosting-services-proposed-by-the-community) | From 6a97ba689170749f4a3819a27ae8d3e87ad4934f Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 09:27:45 +0000 Subject: [PATCH 67/68] Restore practical Docker setup and recovery guidance --- docker/README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index 0eb33e55..50e4b4c6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -122,10 +122,14 @@ TS_AUTHKEY=tskey-auth-... TS_HOSTNAME=livesync ``` -Choose the exposure before starting the profile. The bundled `config/ts-serve.json` sets `AllowFunnel` to `true`, requesting public HTTPS when the tailnet permits Funnel. It is not a private-only default. +**Two sub-modes**: -- For private tailnet access, set that entry to `false` and verify the effective Serve configuration. -- For public Funnel access, retain `true` and configure the required tailnet permissions. See the [Tailscale Docker guide](https://tailscale.com/blog/docker-tailscale-guide) for the distinction between Serve and Funnel. +- **Private tailnet access (VPN / Serve)**: choose this for access from your own devices on your tailnet, subject to its access rules. Set `AllowFunnel` to `false` in `config/ts-serve.json`, and use the Tailscale app on those devices. +- **Public HTTPS (Funnel)**: choose this when devices need to connect over the Internet without joining your tailnet. Keep `AllowFunnel` set to `true`, and allow Funnel for this server in your tailnet's access controls. + +Both modes use a Tailscale HTTPS name, such as `https://livesync..ts.net`, so you do not need to buy a domain. Use the actual DNS name assigned to your server in Tailscale. See the [Tailscale Docker guide](https://tailscale.com/blog/docker-tailscale-guide) for HTTPS and Funnel prerequisites. + +Choose the mode before starting the profile. The bundled `config/ts-serve.json` sets `AllowFunnel` to `true`, requesting public HTTPS when the tailnet permits Funnel; change that entry to `false` to keep the Tailscale endpoint private. After starting or restarting the Tailscale service, check the effective configuration with `docker exec livesync-tailscale tailscale serve status`. The separate host HTTP port described above is unaffected by this choice. The bundled service expects `/dev/net/tun`. On hosts without that device, follow Tailscale's userspace-networking instructions and adjust the device mount as well as `TS_USERSPACE`; changing the environment variable alone leaves the required bind mount in place. @@ -137,7 +141,11 @@ The bundled service expects `/dev/net/tun`. On hosts without that device, follow #### Step 1: Create a Cloudflare Tunnel -Create a remotely managed tunnel using the current [Cloudflare Tunnel setup instructions](https://developers.cloudflare.com/tunnel/get-started/). Copy its connector token into `CF_TUNNEL_TOKEN` below. This Compose profile runs the connector; a separate host installation of `cloudflared` is unnecessary for this profile. +1. Open the [Cloudflare dashboard](https://dash.cloudflare.com/) and go to **Networking → Tunnels**. +2. Select **Create Tunnel**, give it a name such as `obsidian-livesync`, and create it. +3. On the connector setup screen, copy the installation command into a text editor without running it. Extract only the tunnel token, the string beginning with `eyJ`, and put it in `CF_TUNNEL_TOKEN` in the next step. Keep the token private. + +For an existing tunnel, select it and use **Add a replica** to display the installation command again. This Compose profile starts the connector in step 4. Until then, the dashboard can show the tunnel as disconnected. See the official [tunnel setup](https://developers.cloudflare.com/tunnel/get-started/) and [token instructions](https://developers.cloudflare.com/tunnel/reference/tunnel-tokens/) for the current screens. #### Step 2: Configure Environment @@ -268,7 +276,72 @@ docker run --rm --network none -v obsidian-livesync_couchdb-data:/data:ro \ docker compose start couchdb ``` -Before restoration, preserve the existing database volume. Restore and verify the archive in a separate recovery volume with a compatible CouchDB installation before replacing the active data. Do not extract an archive over a running database. See [Backing up CouchDB](https://docs.couchdb.org/en/stable/maintenance/backups.html) for database, configuration, and replication-based backup options. +### Restore a backup + +Restore into a new volume so that the current database remains available. The following Bash commands run from the `docker/` directory. Use a trusted archive from the backup above, and the same CouchDB image version used to create it. Keep the saved CouchDB configuration available too. Run each block only after the previous step succeeds. + +First, choose the archive and create a new recovery volume. Docker generates an unused volume name: + +```bash +backupArchive='' +recoveryImage='' +recoveryVolume=$(docker volume create) && +docker run --rm --network none \ + --mount "type=volume,src=$recoveryVolume,dst=/data" \ + --mount "type=bind,src=$(pwd)/backup,dst=/backup,readonly" \ + alpine:3.22 tar xzf "/backup/$backupArchive" -C /data +``` + +If extraction fails, stop here. The new recovery volume may be empty or incomplete; remove it with `docker volume rm "$recoveryVolume"` before retrying. The original database volume is unaffected. + +Start a temporary CouchDB with the restored data. It has no external network connection or published ports, so synchronising clients cannot reach it: + +```bash +recoveryUser='' +recoveryPassword='' +recoveryContainer=$(docker run -d --network none \ + -e COUCHDB_USER="$recoveryUser" \ + -e COUCHDB_PASSWORD="$recoveryPassword" \ + --mount "type=volume,src=$recoveryVolume,dst=/opt/couchdb/data" \ + "$recoveryImage") +``` + +Wait for CouchDB to start, then inspect its health, database list, and your restored database. Replace the database name with the one you backed up: + +```bash +recoveryDatabase='' +docker exec "$recoveryContainer" curl -fsS \ + -u "$recoveryUser:$recoveryPassword" http://127.0.0.1:5984/_up +docker exec "$recoveryContainer" curl -fsS \ + -u "$recoveryUser:$recoveryPassword" http://127.0.0.1:5984/_all_dbs +docker exec "$recoveryContainer" curl -fsS \ + -u "$recoveryUser:$recoveryPassword" "http://127.0.0.1:5984/$recoveryDatabase" +docker exec "$recoveryContainer" curl -fsS \ + -u "$recoveryUser:$recoveryPassword" \ + "http://127.0.0.1:5984/$recoveryDatabase/_all_docs?limit=5" +``` + +Compare the database names, document counts, and expected document IDs with your backup. These checks establish that CouchDB can read the restored files; they do not establish that every required note is present. If a check fails, inspect `docker logs "$recoveryContainer"` and keep the original volume in place. + +Once you have verified the restored database, stop and remove only the temporary container, and note the recovery volume name: + +```bash +docker stop "$recoveryContainer" && docker rm "$recoveryContainer" +docker volume inspect "$recoveryVolume" --format '{{.Name}}' +``` + +To use the restored copy, first [choose the authoritative copy](../docs/recovery.md#first-choose-the-authoritative-copy) and suspend synchronisation on your devices. Stop the stack with `docker compose down`, using the same profile options as usual and without `-v`. Save a copy of `docker-compose.yml`, then replace only its `couchdb-data` volume declaration with the following, using the printed recovery volume name. Keep the other volume declarations: + +```yaml +volumes: + couchdb-data: + external: true + name: +``` + +Use the same CouchDB image version in the `couchdb` service, restore any saved configuration changes, and start the stack with your usual `docker compose up -d` profile command. The original volume and archive remain available. Verify the required notes from a separate test Vault. When the restored remote is your chosen source of truth, follow [Reset Synchronisation on This Device](../docs/recovery.md#reset-synchronisation-on-this-device) on your existing devices before resuming them, preserving any newer local work first. + +See [Backing up CouchDB](https://docs.couchdb.org/en/stable/maintenance/backups.html) for database, configuration, and replication-based backup options. --- From d550e7a200100b24e22d0eb79d8e43ba2686bacd Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 28 Oct 2026 14:26:34 +0000 Subject: [PATCH 68/68] Move P2P setup details below the requirements table --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2ed9b552..1b720a2c 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,7 @@ Find setup guides, troubleshooting, technical references, design decisions, and - Compatible solutions are supported. - Support end-to-end encryption. - Synchronise settings, snippets, themes, and plug-ins via [Customisation Sync (Advanced)](docs/settings.md#6-customisation-sync-advanced) or [Hidden File Sync](docs/tips/hidden-file-sync.md). -- Enable supported, opt-in WebRTC peer-to-peer synchronisation. - - No central data-storage server is required, but a signalling relay is still required for peer discovery. - - At least one device containing the required data must be online while another device synchronises. - - Follow the [Peer-to-Peer Setup](docs/setup_p2p.md) after reviewing the [P2P communication model](docs/p2p.md). - - If you would like to see how P2P setup and connectivity behave on your devices, try the guided connection check in the experimental [WebPeer browser utility](https://vrtmrz.github.io/obsidian-livesync/webpeer/). It uses an empty Vault and does not test note synchronisation. See the [WebPeer documentation](src/apps/webpeer/README.md) for its scope and limitations. +- Synchronise peer-to-peer using WebRTC. This plug-in may be particularly useful for researchers, engineers, and developers who need to keep their notes fully self-hosted for security reasons. It is also suitable for anyone seeking the peace of mind that comes with knowing their notes remain entirely private. @@ -60,7 +56,9 @@ The linked guides explain the setup and any provider-specific costs. For the Fly.io video walkthrough, see [automated setup](docs/setup_flyio.md#a-very-automated-setup). -P2P can use the project's public signalling relay, so there is no server to provision. See [How peer-to-peer synchronisation works](docs/p2p.md) for more about the connection. +P2P does not require a central data-storage server, but a signalling relay is still required for peer discovery. You can use the project's public signalling relay, so there is no server to provision. At least one device containing the required data must be online while another device synchronises. Follow the [Peer-to-Peer Setup](docs/setup_p2p.md) after reviewing the [P2P communication model](docs/p2p.md). + +If you would like to see how P2P setup and connectivity behave on your devices, try the guided connection check in the experimental [WebPeer browser utility](https://vrtmrz.github.io/obsidian-livesync/webpeer/). It uses an empty Vault and does not test note synchronisation. See the [WebPeer documentation](src/apps/webpeer/README.md) for its scope and limitations. ### Start synchronising