Cover stale-file recovery and bulk folder operations

Document exact file revision provenance and add real Obsidian scenarios for stale files after restart, parent-folder renames, and parent-folder deletion. Characterise replication queue delays during repeated writes to one document.

Companion to Commonlib c0a84a4. Validation used its packed artefact; the local file dependency remains uncommitted until a published Commonlib version is available.

test/e2e-obsidian/README.md

@@ -75,11 +75,17 @@ After changing plug-in source, use the focused wrapper rather than invoking a sc
 ```bash
 npm run test:e2e:obsidian:focused -- settings-ui
 npm run test:e2e:obsidian:focused -- two-vault-sync
+npm run test:e2e:obsidian:focused -- stale-file-restart
+npm run test:e2e:obsidian:focused -- folder-batch
 npm run test:e2e:obsidian:focused -- security-seed-reconnect
 ```
 
 The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite.
 
+`folder-batch` needs no remote service. It creates 24 notes in nested folders, renames and deletes the parent through the Obsidian Vault API, and checks descendant events, content, Chunks, deletion markers, and provenance. A note outside the parent must remain writable.
+
+`stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file with exact provenance receives the newer database content without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. The database advance and pending snapshot are controlled fixtures; startup processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application.
+
 The principal entry points are:
 
 ```bash

test/e2e-obsidian/scripts/folder-batch.ts

@@ -0,0 +1,167 @@
+import { evalObsidianJson } from "../runner/cli.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import { createE2eObsidianDeviceLocalState, waitForLiveSyncCoreReady } from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000";
+const originalRoot = "batch/original";
+const renamedRoot = "batch/renamed";
+const outsidePath = "batch/outside.md";
+const folders = ["alpha", "alpha/deep", "beta"];
+const notes = Array.from({ length: 24 }, (_, index) => ({
+    relativePath: `${folders[index % folders.length]}/note-${index}.md`,
+    body: `# Descendant ${index}\n\nThis body must survive a parent folder rename.\n`,
+}));
+
+async function main(): Promise<void> {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked: ${cli.checked.join(", ")}`);
+    const cliBinary = cli.binary;
+    const vault = await createTemporaryVault("obsidian-livesync-folder-batch-");
+    let session: ObsidianLiveSyncSession | undefined;
+    try {
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary,
+            vault,
+            pluginData: {
+                doctorProcessedVersion: "1.0.0",
+                isConfigured: true,
+                liveSync: false,
+                remoteType: "",
+                couchDB_URI: "http://127.0.0.1:5984",
+                couchDB_DBNAME: "folder-batch",
+                notifyThresholdOfRemoteStorageSize: -1,
+                periodicReplication: false,
+                syncOnStart: false,
+                syncOnSave: false,
+                syncOnFileOpen: false,
+                syncOnEditorSave: false,
+                syncAfterMerge: false,
+                useEden: false,
+            },
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+        });
+        await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
+        const result = await evalObsidianJson<{ descendants: number; renamed: number; deleted: number }>(
+            cliBinary,
+            `(async()=>{
+                const core=app.plugins.plugins['obsidian-livesync'].core;
+                const provenance=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1');
+                const notes=${JSON.stringify(notes)};
+                const originalRoot=${JSON.stringify(originalRoot)};
+                const renamedRoot=${JSON.stringify(renamedRoot)};
+                const outsidePath=${JSON.stringify(outsidePath)};
+                const renamed=new Set(), deleted=new Set();
+                const refs=[
+                    app.vault.on('rename',(file,oldPath)=>{
+                        if(file.stat) renamed.add(oldPath+' -> '+file.path);
+                    }),
+                    app.vault.on('delete',(file)=>{if(file.stat) deleted.add(file.path);}),
+                ];
+                const meta=(path)=>core.localDatabase.getDBEntryMeta(path,{conflicts:true},true);
+                const isDeleted=(entry)=>entry && (entry.deleted || entry._deleted);
+                const getContent=(entry)=>Array.isArray(entry.data)?entry.data.join(''):entry.data;
+
+                async function liveErrors(path,body){
+                    const errors=[];
+                    const file=app.vault.getAbstractFileByPath(path);
+                    const entry=await meta(path);
+                    if(!file?.stat || file.path!==path || await app.vault.read(file)!==body)
+                        errors.push('Vault content: '+path);
+                    if(!entry || isDeleted(entry) || entry.path!==path || !entry.children.length){
+                        errors.push('DB metadata: '+path);
+                    }else{
+                        const loaded=await core.localDatabase.getDBEntry(path,{rev:entry._rev},false,true,true);
+                        if(!loaded || getContent(loaded)!==body) errors.push('DB content: '+path);
+                        if(entry._conflicts?.length) errors.push('Unexpected conflict: '+path);
+                        if((await provenance.get(path))?.revision!==entry._rev)
+                            errors.push('Provenance: '+path);
+                    }
+                    return errors;
+                }
+                async function deletedErrors(path){
+                    const errors=[];
+                    const entry=await meta(path);
+                    if(app.vault.getAbstractFileByPath(path)) errors.push('File remains: '+path);
+                    if(!isDeleted(entry)) errors.push('Missing tombstone: '+path);
+                    if(entry?._conflicts?.length) errors.push('Deletion conflict: '+path);
+                    if(await provenance.get(path)) errors.push('Old provenance remains: '+path);
+                    return errors;
+                }
+                async function waitFor(phase,check){
+                    const deadline=Date.now()+20000;
+                    let errors=[];
+                    do{
+                        await core.services.fileProcessing.commitPendingFileEvents();
+                        errors=await check();
+                        if(!errors.length) return;
+                        await new Promise(resolve=>setTimeout(resolve,50));
+                    }while(Date.now()<deadline);
+                    throw new Error(phase+': '+errors.slice(0,8).join('; '));
+                }
+                const liveBatch=(root)=>Promise.all(notes.map(note=>
+                    liveErrors(root+'/'+note.relativePath,note.body))).then(results=>results.flat());
+                const deletedBatch=(root)=>Promise.all(notes.map(note=>
+                    deletedErrors(root+'/'+note.relativePath))).then(results=>results.flat());
+
+                try{
+                    await app.vault.createFolder('batch');
+                    await app.vault.createFolder(originalRoot);
+                    for(const folder of ${JSON.stringify(folders)})
+                        await app.vault.createFolder(originalRoot+'/'+folder);
+                    await Promise.all(notes.map(note=>app.vault.create(originalRoot+'/'+note.relativePath,note.body)));
+                    await app.vault.create(outsidePath,'Outside note');
+                    await waitFor('Initial batch',async()=>[
+                        ...await liveBatch(originalRoot), ...await liveErrors(outsidePath,'Outside note'),
+                    ]);
+                    const originalIds=await Promise.all(notes.map(async note=>(await meta(originalRoot+'/'+note.relativePath))._id));
+
+                    // Rename the parent once: Obsidian must emit every descendant event.
+                    await app.vault.rename(app.vault.getAbstractFileByPath(originalRoot),renamedRoot);
+                    await waitFor('Renamed batch',async()=>[
+                        ...await liveBatch(renamedRoot), ...await deletedBatch(originalRoot),
+                        ...await liveErrors(outsidePath,'Outside note'),
+                    ]);
+                    for(const [index,note] of notes.entries()){
+                        const from=originalRoot+'/'+note.relativePath, to=renamedRoot+'/'+note.relativePath;
+                        if(!renamed.has(from+' -> '+to)) throw new Error('Missing descendant rename: '+from);
+                        if((await meta(to))._id===originalIds[index]) throw new Error('Rename reused the source ID: '+to);
+                    }
+
+                    // Delete the parent once, without synthesising individual file events.
+                    await app.vault.delete(app.vault.getAbstractFileByPath(renamedRoot),true);
+                    await waitFor('Deleted batch',async()=>[
+                        ...await deletedBatch(renamedRoot), ...await deletedBatch(originalRoot),
+                        ...await liveErrors(outsidePath,'Outside note'),
+                    ]);
+                    for(const note of notes){
+                        const path=renamedRoot+'/'+note.relativePath;
+                        if(!deleted.has(path)) throw new Error('Missing descendant deletion: '+path);
+                    }
+                    if(app.vault.getAbstractFileByPath(renamedRoot)) throw new Error('Deleted folder remains');
+                    await app.vault.modify(app.vault.getAbstractFileByPath(outsidePath),'Outside note updated');
+                    await waitFor('Outside update',()=>liveErrors(outsidePath,'Outside note updated'));
+                    return JSON.stringify({descendants:notes.length,renamed:renamed.size,deleted:deleted.size});
+                }finally{
+                    for(const ref of refs) app.vault.offref(ref);
+                }
+            })()`,
+            session.cliEnv
+        );
+        console.log(
+            `Folder batch: ${result.descendants} descendants persisted, renamed, and deleted; ` +
+                `${result.renamed} rename and ${result.deleted} delete events observed; outside note remained writable.`
+        );
+    } finally {
+        if (session) await session.app.stop();
+        await vault.dispose();
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exitCode = 1;
+});

test/e2e-obsidian/scripts/run-focused.ts

@@ -25,6 +25,8 @@ const focusedScenarios = new Set([
     "p2p-setup-uri-workflow",
     "partial-startup-file-failure",
     "startup-scan",
+    "stale-file-restart",
+    "folder-batch",
     "setup-uri-workflow",
     "two-vault-sync",
     "security-seed-reconnect",

test/e2e-obsidian/scripts/stale-file-restart.ts

@@ -0,0 +1,188 @@
+import { evalObsidianJson } from "../runner/cli.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertEqual,
+    createE2eObsidianDeviceLocalState,
+    waitForLiveSyncCoreReady,
+    waitForLocalDatabaseEntry,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+const paths = ["stale-known.md", "stale-unknown.md"];
+const oldContent = "# Note\nKeep\n\nTail\n\nFooter\n";
+const newContent = oldContent.replace(
+    "Footer\n",
+    Array.from({ length: 50 }, (_, index) => `Remote addition ${index}\n`).join("") + "Footer\n"
+);
+
+type Branch = { rev: string; content: string; history: string[] };
+type FileState = { path: string; content: string; rev: string; branches: Branch[]; provenance: string | null };
+
+async function readState(cliBinary: string, env: NodeJS.ProcessEnv): Promise<FileState[]> {
+    return await evalObsidianJson<FileState[]>(
+        cliBinary,
+        `(async()=>{
+            const core=app.plugins.plugins['obsidian-livesync'].core;
+            const store=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1');
+            const states=[];
+            for(const path of ${JSON.stringify(paths)}){
+                const meta=await core.localDatabase.getDBEntryMeta(path,{conflicts:true},true);
+                const branches=[];
+                for(const rev of [meta._rev,...(meta._conflicts??[])]){
+                    const entry=await core.localDatabase.getDBEntry(path,{rev,revs:true},false,true,true);
+                    const raw=await core.localDatabase.getRaw(meta._id,{rev,revs:true});
+                    branches.push({rev,content:Array.isArray(entry.data)?entry.data.join(''):entry.data,
+                        history:raw._revisions.ids.map((id,i)=>(raw._revisions.start-i)+'-'+id)});
+                }
+                const file=app.vault.getAbstractFileByPath(path);
+                states.push({path,content:await app.vault.read(file),rev:meta._rev,branches,
+                    provenance:(await store.get(path))?.revision??null});
+            }
+            return JSON.stringify(states);
+        })()`,
+        env
+    );
+}
+
+async function main(): Promise<void> {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked: ${cli.checked.join(", ")}`);
+    const cliBinary = cli.binary;
+    const vault = await createTemporaryVault("obsidian-livesync-stale-file-");
+    let session: ObsidianLiveSyncSession | undefined;
+    try {
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary,
+            vault,
+            pluginData: {
+                doctorProcessedVersion: "1.0.0",
+                isConfigured: true,
+                liveSync: false,
+                remoteType: "",
+                couchDB_URI: "http://127.0.0.1:5984",
+                couchDB_DBNAME: "stale-file-restart",
+                notifyThresholdOfRemoteStorageSize: -1,
+                periodicReplication: false,
+                syncAfterMerge: false,
+                syncOnEditorSave: false,
+                syncOnFileOpen: false,
+                syncOnSave: false,
+                syncOnStart: false,
+                disableMarkdownAutoMerge: false,
+                resolveConflictsByNewerFile: false,
+                checkConflictOnlyOnOpen: true,
+                showMergeDialogOnlyOnActive: true,
+            },
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+        });
+        await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
+        await evalObsidianJson(
+            cliBinary,
+            `(async()=>{
+            for(const path of ${JSON.stringify(paths)}) await app.vault.create(path,${JSON.stringify(oldContent)});
+            return JSON.stringify(true);
+        })()`,
+            session.cliEnv
+        );
+        for (const path of paths) await waitForLocalDatabaseEntry(cliBinary, session.cliEnv, path);
+
+        // Drain real Vault events before creating a persisted pending-event fixture.
+        // The DB advances without reflecting it in the Vault, as on an offline device.
+        const fixture = await evalObsidianJson<{ current: string[]; original: string[] }>(
+            cliBinary,
+            `(async()=>{
+            const core=app.plugins.plugins['obsidian-livesync'].core;
+            const store=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1');
+            await core.services.fileProcessing.commitPendingFileEvents();
+            const snapshot=[], current=[], original=[];
+            for(const [index,path] of ${JSON.stringify(paths)}.entries()){
+                const meta=await core.localDatabase.getDBEntryMeta(path,{},true);
+                const file=await core.storageAccess.getFileStub(path);
+                const data=new Blob([${JSON.stringify(newContent)}],{type:'text/plain'});
+                const result=await core.localDatabase.putDBEntry({...meta,data,mtime:file.stat.mtime+60000,
+                    size:data.size,children:[]},false,meta._rev);
+                if(!result?.ok) throw new Error('Could not advance '+path);
+                current.push(result.rev); original.push(meta._rev);
+                if(index===0) await store.set(path,{revision:meta._rev,observedStorageMtime:file.stat.mtime});
+                else await store.delete(path);
+                snapshot.push({type:'CHANGED',key:'CHANGED-'+path,args:{file}});
+            }
+            await core.kvDB.set('storage-event-manager-snapshot',snapshot);
+            return JSON.stringify({current,original});
+        })()`,
+            session.cliEnv
+        );
+        await session.app.stop();
+        session = undefined;
+
+        session = await startObsidianLiveSyncSession({ binary, cliBinary, vault });
+        await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
+        const [known, unknown] = await readState(cliBinary, session.cliEnv);
+        assertEqual(known.rev, fixture.current[0], "An unchanged stale file created a revision during restart.");
+        assertEqual(known.branches.length, 1, "An unchanged stale file created a conflict.");
+        assertEqual(known.content, newContent, "The newer DB content was not reflected after suppressing the save.");
+        assertEqual(known.provenance, fixture.current[0], "The reflected revision was not recorded.");
+        assertEqual(unknown.branches.length, 2, "Unknown local content was not preserved as a conflict.");
+        assertEqual(unknown.content, oldContent, "Unknown local content was overwritten.");
+        const independent = unknown.branches.find((branch) => branch.content === oldContent);
+        if (!independent) throw new Error("The old local content is missing from the current branches.");
+        assertEqual(independent.history.length, 1, "Unknown content was attached to an inferred ancestor.");
+        if (independent.rev === fixture.original[1]) throw new Error("The historical root was reused.");
+        if (!unknown.branches.some((branch) => branch.content === newContent)) {
+            throw new Error("The remote additions were lost.");
+        }
+
+        await evalObsidianJson(
+            cliBinary,
+            `(async()=>{
+            const core=app.plugins.plugins['obsidian-livesync'].core;
+            const path=${JSON.stringify(paths[1])};
+            await core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1').delete(path);
+            if(!await core.fileHandler.storeFileToDB(path)) throw new Error('Repeated save failed');
+            await app.workspace.getLeaf(false).openFile(app.vault.getAbstractFileByPath(${JSON.stringify(paths[0])}));
+            await core.services.conflict.resolve(path);
+            return JSON.stringify(true);
+        })()`,
+            session.cliEnv
+        );
+        const [, repeated] = await readState(cliBinary, session.cliEnv);
+        assertEqual(
+            repeated.branches
+                .map((branch) => branch.rev)
+                .sort()
+                .join(","),
+            unknown.branches
+                .map((branch) => branch.rev)
+                .sort()
+                .join(","),
+            "Losing provenance and reprocessing added or auto-merged a branch."
+        );
+        await evalObsidianJson(
+            cliBinary,
+            `(async()=>{
+                const core=app.plugins.plugins['obsidian-livesync'].core;
+                core.settings.resolveConflictsByNewerFile=true;
+                await core.services.conflict.resolve(${JSON.stringify(paths[1])});
+                return JSON.stringify(true);
+            })()`,
+            session.cliEnv
+        );
+        const [, resolved] = await readState(cliBinary, session.cliEnv);
+        assertEqual(resolved.branches.length, 1, "The explicit newer-file option did not resolve the conflict.");
+        assertEqual(resolved.content, newContent, "The newer-file option did not reflect the newer DB version.");
+        console.log(
+            "Stale-file restart: known content reflected; unknown content preserved without duplicate branches; explicit newer-file resolution retained."
+        );
+    } finally {
+        if (session) await session.app.stop();
+        await vault.dispose();
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exitCode = 1;
+});

test/e2e-obsidian/scripts/two-vault-sync.ts

@@ -997,14 +997,26 @@ async function runMarkdownAutoMerge(
 
     session = await startConfiguredSession(context, vaultA, conflictOverrides);
     const baseOnA = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, conflictPath);
-    await storeFileRevision(context.cliBinary, session.cliEnv, conflictPath, left, baseOnA.rev);
-    await writeVaultFile(vaultA.path, conflictPath, left);
+    await writeNoteViaObsidian(context.cliBinary, session.cliEnv, conflictPath, left);
+    const storedLeft = await waitForConflictBranch(
+        context.cliBinary,
+        session.cliEnv,
+        conflictPath,
+        (branch) => branch.content === left
+    );
+    assertEqual(storedLeft.parentRev, baseOnA.rev, "Vault A's edit did not extend its displayed base.");
     await pushLocalChanges(context.cliBinary, session.cliEnv);
     await stopTrackedSession(context, session);
 
     session = await startConfiguredSession(context, vaultB, conflictOverrides);
-    await storeFileRevision(context.cliBinary, session.cliEnv, conflictPath, right, baseOnB.rev);
-    await writeVaultFile(vaultB.path, conflictPath, right);
+    await writeNoteViaObsidian(context.cliBinary, session.cliEnv, conflictPath, right);
+    const storedRight = await waitForConflictBranch(
+        context.cliBinary,
+        session.cliEnv,
+        conflictPath,
+        (branch) => branch.content === right
+    );
+    assertEqual(storedRight.parentRev, baseOnB.rev, "Vault B's edit did not extend its displayed base.");
     await pushLocalChanges(context.cliBinary, session.cliEnv);
     const conflict = await waitForFileConflict(context.cliBinary, session.cliEnv, conflictPath);
     const leftBranch = conflict.branches.find((branch) => branch.content === left);
@@ -1028,8 +1040,18 @@ async function runMarkdownAutoMerge(
     );
 
     const afterResolution = `${merged.trimEnd()}\n\nPost-resolution edit on B.\n`;
-    await storeFileRevision(context.cliBinary, session.cliEnv, conflictPath, afterResolution, mergedRev);
-    await writeVaultFile(vaultB.path, conflictPath, afterResolution);
+    await writeNoteViaObsidian(context.cliBinary, session.cliEnv, conflictPath, afterResolution);
+    const storedAfterResolution = await waitForConflictBranch(
+        context.cliBinary,
+        session.cliEnv,
+        conflictPath,
+        (branch) => branch.content === afterResolution
+    );
+    assertEqual(
+        storedAfterResolution.parentRev,
+        mergedRev,
+        "The post-resolution edit did not extend the merged revision."
+    );
     await pushLocalChanges(context.cliBinary, session.cliEnv);
     await stopTrackedSession(context, session);