test: cover recovery of failed local file operations

Record the accepted retry decision, its alternatives, and preservation
boundaries. Add real Obsidian coverage for temporary-read failure followed
by rename, a later edit, and recovery after restart.

The scenario uses an exact Commonlib candidate until its release is adopted.

Refs #1331

test/e2e-obsidian/README.md

@@ -82,6 +82,7 @@ npm run test:e2e:obsidian:focused -- configuration-encryption
 npm run test:e2e:obsidian:focused -- configuration-encryption-sync
 npm run test:e2e:obsidian:focused -- two-vault-sync
 npm run test:e2e:obsidian:focused -- stale-file-restart
+npm run test:e2e:obsidian:focused -- local-file-retry
 npm run test:e2e:obsidian:focused -- folder-batch
 npm run test:e2e:obsidian:focused -- security-seed-reconnect
 npm run test:e2e:obsidian:focused -- couchdb-http-cache
@@ -111,6 +112,8 @@ The same scenario checks the five actions in Reporting Tools, the separate Check
 
 `third-party-chunks` reflects synthetic externally generated text through the installed Commonlib and the real Obsidian file handler. It supplies a split surrogate pair at the file-access read boundary, with size checking enabled, then verifies exact Vault and on-disk text, unchanged Metadata, and matching persisted provenance. Complete-pair text and a binary attachment provide controls. The underlying Metadata and Chunks are real local database entries; this does not establish that CouchDB or a particular third-party application generates this chunk layout. No remote service is required.
 
+`local-file-retry` verifies the [local retry contract](../../docs/adr/2026_11_local_file_operation_retry.md). Until the corresponding Commonlib package is published and adopted, it requires the packed candidate; the reviewed Commonlib 0.4.0 package is its failing baseline. It needs no remote service. It injects a targeted `ENOENT` from `app.vault.adapter.read`, then uses real Vault create, rename, and modify events to check local retry after a rename, a later same-path edit, and a natural restart. Assertions inspect the current local database body, pending events, and revision ancestry. The same-path edit is observed without a manual flush; the restart reuses the same isolated Vault and profile.
+
 `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application.
 
 The principal entry points are:

test/e2e-obsidian/scripts/local-file-retry.ts

@@ -0,0 +1,401 @@
+import { evalObsidianJson } from "../runner/cli.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertEqual,
+    createE2eObsidianDeviceLocalState,
+    waitForLiveSyncCoreReady,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000";
+
+const paths = {
+    temporary: "retry-rename-temporary.md",
+    renamed: "retry-rename-final.md",
+    edited: "retry-same-path.md",
+    restarted: "retry-after-restart.md",
+};
+const content = {
+    renamed: "Rename recovery content\n",
+    initial: "Initial failed save\n",
+    firstEdit: "First later edit\n",
+    latestEdit: "Latest later edit\n",
+    restart: "Restart recovery content\n",
+};
+
+/** Shared in-app probes and controlled read-failure injection; each evaluation resolves the current plug-in instance. */
+const evaluationHelpers = `
+    const core=app.plugins.plugins['obsidian-livesync'].core;
+    const manager=core.serviceModules.storageAccess.vaultManager;
+    const sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms));
+    const require=(condition,message)=>{if(!condition)throw new Error(message);};
+    const waitFor=async(predicate,message)=>{
+        const until=Date.now()+15000;
+        while(Date.now()<until){if(await predicate())return;await sleep(25);}
+        throw new Error(message);
+    };
+    const pendingEvents=async()=>await manager.adapter.persistence.loadSnapshot()??[];
+    const eventPaths=event=>{
+        const args=event.args??{};
+        return [args.file?.path,args.oldFile?.path,args.oldPath,args.renameTarget,args.targetPath,args.path]
+            .filter(path=>typeof path==='string');
+    };
+    const isPending=async(...paths)=>{
+        const snapshot=await pendingEvents();
+        return snapshot.some(event=>eventPaths(event).some(path=>paths.includes(path)));
+    };
+    const databaseState=async path=>{
+        let meta;
+        try{meta=await core.localDatabase.getDBEntryMeta(path,{conflicts:true},true);}
+        catch(error){if(error?.status===404||error?.name==='not_found')return null;throw error;}
+        if(!meta)return null;
+        const revisions=[...new Set([meta._rev,...(meta._conflicts??[])].filter(Boolean))];
+        const branches=[];
+        for(const rev of revisions){
+            const doc=await core.localDatabase.getDBEntry(path,{rev,revs:true},false,true,true).catch(()=>null);
+            const raw=await core.localDatabase.getRaw(meta._id,{rev,revs:true}).catch(()=>null);
+            const history=raw?._revisions?.ids?.map((id,index)=>(raw._revisions.start-index)+'-'+id)??[];
+            const value=Array.isArray(doc?.data)?doc.data.join(''):typeof doc?.data==='string'?doc.data:null;
+            branches.push({rev,body:value,deleted:!!doc?._deleted,history});
+        }
+        const current=branches.find(branch=>branch.rev===meta._rev);
+        return {path:meta.path,deleted:!!meta.deleted,present:!!current&&!meta.deleted&&!current.deleted,rev:meta._rev,
+            currentBody:current?.body??null,branches};
+    };
+    const bodyMatches=(actual,expected)=>actual===expected||actual===btoa(expected);
+    const installReadFailure=paths=>{
+        const key='__livesyncLocalFileRetry';
+        globalThis[key]?.restore?.();
+        const adapter=app.vault.adapter;
+        const original=adapter.read;
+        const state={blocked:new Set(paths),failures:Object.create(null),holdPath:null,
+            holdNextSuccessfulRead:false,heldReadStarted:false,heldReadBody:null,readGate:null,read:null,restore:null};
+        state.restore=()=>{
+            if(adapter.read===state.read)adapter.read=original;
+            if(globalThis[key]===state)delete globalThis[key];
+        };
+        state.read=async function(path){
+            if(state.blocked.has(path)){
+                state.failures[path]=(state.failures[path]??0)+1;
+                const error=new Error('ENOENT: injected read failure for '+path);
+                error.code='ENOENT';
+                throw error;
+            }
+            const value=await original.call(this,path);
+            if(state.holdNextSuccessfulRead&&path===state.holdPath){
+                state.holdNextSuccessfulRead=false;
+                state.heldReadStarted=true;
+                state.heldReadBody=value;
+                await state.readGate;
+            }
+            return value;
+        };
+        globalThis[key]=state;
+        adapter.read=state.read;
+        return state;
+    };
+`;
+
+/**
+ * Evaluate one scenario step inside the running isolated Obsidian session.
+ *
+ * @typeParam T - The JSON result expected by the calling assertion.
+ * @param cliBinary - The discovered Obsidian CLI executable.
+ * @param session - The session whose environment identifies the isolated Vault.
+ * @param body - The scenario expression using the shared in-app probes.
+ * @returns The decoded scenario result; evaluation and transport failures reject the promise.
+ */
+async function evaluate<T>(
+    cliBinary: string,
+    session: ObsidianLiveSyncSession,
+    body: string
+): Promise<T> {
+    return await evalObsidianJson<T>(
+        cliBinary,
+        `(async()=>{${evaluationHelpers}\n${body}\n})()`,
+        session.cliEnv
+    );
+}
+
+/**
+ * Exercise rename, later-edit, and restart recovery through real Vault events.
+ *
+ * @returns Completion after the isolated session is closed; assertion or setup failures reject.
+ * @remarks
+ * Read failures are injected only into the temporary Vault. Content, pending
+ * ownership, and revision history are checked independently of remote replication.
+ */
+async function main(): Promise<void> {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked: ${cli.checked.join(", ")}`);
+    const cliBinary = cli.binary;
+    const vault = await createTemporaryVault("obsidian-local-file-retry-");
+    let session: ObsidianLiveSyncSession | undefined;
+
+    /**
+     * Start or reopen the same isolated Vault and profile.
+     *
+     * @param initial - Whether to seed the first run with local-only plug-in settings.
+     * @returns Completion when the session and LiveSync core are ready for the next scenario.
+     */
+    const start = async (initial = false): Promise<void> => {
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary,
+            vault,
+            pluginStartup: initial ? undefined : "natural",
+            pluginData: initial
+                ? {
+                      doctorProcessedVersion: "1.0.0",
+                      isConfigured: true,
+                      liveSync: false,
+                      remoteType: "",
+                      couchDB_URI: "http://127.0.0.1:5984",
+                      couchDB_DBNAME: "local-file-retry",
+                      notifyThresholdOfRemoteStorageSize: -1,
+                      periodicReplication: false,
+                      syncAfterMerge: false,
+                      syncOnEditorSave: false,
+                      syncOnFileOpen: false,
+                      syncOnSave: false,
+                      syncOnStart: false,
+                      disableMarkdownAutoMerge: false,
+                      resolveConflictsByNewerFile: false,
+                      checkConflictOnlyOnOpen: true,
+                      showMergeDialogOnlyOnActive: true,
+                  }
+                : undefined,
+            localStorageEntries: initial ? createE2eObsidianDeviceLocalState(vault.name) : undefined,
+        });
+        const readiness = await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
+        assertEqual(readiness.configured, true, "Self-hosted LiveSync did not start configured.");
+    };
+
+    try {
+        await start(true);
+
+        const renameObservation = await evaluate<{
+            firstFailureReads: number;
+            renameFailureReads: number;
+            finalRevision: string;
+            finalHistory: string[];
+            obsoleteTemporaryPathAbsent: boolean;
+        }>(
+            cliBinary,
+            session!,
+            `
+            const hook=installReadFailure([${JSON.stringify(paths.temporary)}]);
+            try{
+                await app.vault.create(${JSON.stringify(paths.temporary)},${JSON.stringify(content.renamed)});
+                await waitFor(async()=>hook.failures[${JSON.stringify(paths.temporary)}]>0&&
+                    await isPending(${JSON.stringify(paths.temporary)}),
+                    'The temporary file read failure was not retained as a pending Vault event.');
+                const firstFailureReads=hook.failures[${JSON.stringify(paths.temporary)}];
+                require(!(await databaseState(${JSON.stringify(paths.temporary)}))?.branches?.length,
+                    'The failed temporary file save unexpectedly reached the local database.');
+
+                hook.blocked.add(${JSON.stringify(paths.renamed)});
+                let renameEvents=0;
+                const renameRef=app.vault.on('rename',(file,oldPath)=>{
+                    if(file.path===${JSON.stringify(paths.renamed)}&&oldPath===${JSON.stringify(paths.temporary)})renameEvents++;
+                });
+                try{
+                    const file=app.vault.getAbstractFileByPath(${JSON.stringify(paths.temporary)});
+                    require(!!file,'The temporary file disappeared before the real Vault rename.');
+                    await app.vault.rename(file,${JSON.stringify(paths.renamed)});
+                    require(renameEvents===1,'The real Vault rename event was not observed exactly once.');
+                }finally{app.vault.offref(renameRef);}
+
+                await waitFor(async()=>hook.failures[${JSON.stringify(paths.renamed)}]>0&&
+                    await isPending(${JSON.stringify(paths.renamed)}),
+                    'The final-path read failure and rename event were not retained.');
+                const renameFailureReads=hook.failures[${JSON.stringify(paths.renamed)}];
+                const finalFile=app.vault.getAbstractFileByPath(${JSON.stringify(paths.renamed)});
+                require(!!finalFile,'The real Vault rename did not create the final path.');
+                hook.restore();
+                require(await app.vault.read(finalFile)===${JSON.stringify(content.renamed)},
+                    'The real Vault rename did not preserve the final file bytes.');
+
+                require(await core.services.fileProcessing.commitPendingFileEvents(),
+                    'Pending rename recovery did not complete after restoring reads.');
+                await waitFor(async()=>!(await isPending(${JSON.stringify(paths.temporary)},${JSON.stringify(paths.renamed)})),
+                    'The temporary or final path remained pending after rename recovery.');
+                const finalState=await databaseState(${JSON.stringify(paths.renamed)});
+                const oldState=await databaseState(${JSON.stringify(paths.temporary)});
+                require(finalState&&finalState.path===${JSON.stringify(paths.renamed)}&&!finalState.deleted&&
+                    bodyMatches(finalState.currentBody,${JSON.stringify(content.renamed)}),
+                    'The final current database document does not contain the renamed file bytes at the final path.');
+                require(finalState.branches.length===1&&finalState.branches[0].history.length===1,
+                    'Rename recovery created an unexpected branch or revision.');
+                require(!oldState||oldState.deleted,
+                    'The obsolete temporary path remains a current database file.');
+                return JSON.stringify({firstFailureReads,renameFailureReads,finalRevision:finalState.rev,
+                    finalHistory:finalState.branches[0].history,obsoleteTemporaryPathAbsent:!oldState||oldState.deleted});
+            }finally{hook.restore();}
+            `
+        );
+
+        const samePathObservation = await evaluate<{
+            initialFailureReads: number;
+            earlierReadBody: string;
+            laterEditPendingWhileEarlierReadWasHeld: boolean;
+            newestEditObservedWhileReadWasHeld: boolean;
+            latestBody: string;
+            finalRevision: string;
+            finalHistory: string[];
+        }>(
+            cliBinary,
+            session!,
+            `
+            const path=${JSON.stringify(paths.edited)};
+            const hook=installReadFailure([path]);
+            let releaseRead;
+            const readGate=new Promise(resolve=>{releaseRead=resolve;});
+            let readReleased=false;
+            try{
+                await app.vault.create(path,${JSON.stringify(content.initial)});
+                await waitFor(async()=>hook.failures[path]>0&&await isPending(path),
+                    'The initial same-path read failure was not retained.');
+                const initialFailureReads=hook.failures[path];
+
+                hook.holdPath=path;
+                hook.readGate=readGate;
+                hook.holdNextSuccessfulRead=true;
+                hook.blocked.delete(path);
+                const file=app.vault.getAbstractFileByPath(path);
+                require(!!file,'The same-path fixture file disappeared after its initial failure.');
+                const initialEventKeys=new Set((await pendingEvents()).filter(event=>eventPaths(event).includes(path)).map(event=>event.key));
+                await app.vault.modify(file,${JSON.stringify(content.firstEdit)});
+                await waitFor(()=>hook.heldReadStarted,
+                    'A real same-path edit did not retry the failed local save.');
+                await waitFor(async()=>(await pendingEvents()).some(event=>eventPaths(event).includes(path)&&!initialEventKeys.has(event.key)),
+                    'The edit which triggered recovery was not retained while the earlier read was held.');
+                const earlierReadBody=hook.heldReadBody;
+                require(bodyMatches(earlierReadBody,${JSON.stringify(content.initial)})||
+                    bodyMatches(earlierReadBody,${JSON.stringify(content.firstEdit)}),
+                    'The held retry read did not capture the pre-latest file body.');
+                require(!bodyMatches(earlierReadBody,${JSON.stringify(content.latestEdit)}),
+                    'The held read already contained the later edit.');
+                require(await isPending(path),
+                    'The failed save stopped being pending while its retry read was held.');
+
+                let newestEditEvents=0;
+                const modifyRef=app.vault.on('modify',changed=>{if(changed.path===path)newestEditEvents++;});
+                try{
+                    await app.vault.modify(file,${JSON.stringify(content.latestEdit)});
+                    require(newestEditEvents>0,'The newest real Vault modification was not observed.');
+                }finally{app.vault.offref(modifyRef);}
+                // The existing read guard can suppress this newest notification.
+                // The already admitted successor must still save the latest bytes.
+                require(await app.vault.read(file)===${JSON.stringify(content.latestEdit)},
+                    'The second later edit did not reach the real Vault file.');
+                require(await isPending(path),
+                    'The admitted successor disappeared while the earlier read was held.');
+                readReleased=true;
+                releaseRead();
+
+                await waitFor(async()=>{
+                    const state=await databaseState(path);
+                    return !!state&&state.present&&bodyMatches(state.currentBody,${JSON.stringify(content.latestEdit)})&&
+                        !(await isPending(path));
+                },'The latest same-path body was not committed before its pending event resolved.');
+                const finalState=await databaseState(path);
+                require(bodyMatches(finalState.currentBody,${JSON.stringify(content.latestEdit)}),
+                    'An earlier completion consumed or replaced the latest same-path body.');
+                require(finalState.branches.some(branch=>bodyMatches(branch.body,${JSON.stringify(content.latestEdit)})),
+                    'The latest same-path body is absent from the current revision leaves.');
+                require(await app.vault.read(file)===${JSON.stringify(content.latestEdit)},
+                    'The earlier completion replaced the latest real Vault body.');
+                return JSON.stringify({initialFailureReads,earlierReadBody,
+                    laterEditPendingWhileEarlierReadWasHeld:true,newestEditObservedWhileReadWasHeld:true,latestBody:finalState.currentBody,
+                    finalRevision:finalState.rev,
+                    finalHistory:finalState.branches.find(branch=>branch.rev===finalState.rev)?.history??[]});
+            }finally{
+                if(!readReleased)releaseRead();
+                hook.restore();
+            }
+            `
+        );
+
+        const restartFailureObservation = await evaluate<{
+            failureReadsBeforeShutdown: number;
+            pendingBeforeShutdown: boolean;
+            databaseAbsentBeforeShutdown: boolean;
+        }>(
+            cliBinary,
+            session!,
+            `
+            const hook=installReadFailure([${JSON.stringify(paths.restarted)}]);
+            await app.vault.create(${JSON.stringify(paths.restarted)},${JSON.stringify(content.restart)});
+            await waitFor(async()=>hook.failures[${JSON.stringify(paths.restarted)}]>0&&
+                await isPending(${JSON.stringify(paths.restarted)}),
+                'The restart fixture read failure was not retained.');
+            const state=await databaseState(${JSON.stringify(paths.restarted)});
+            require(!state||!state.present,
+                'The restart fixture unexpectedly saved before shutdown.');
+            return JSON.stringify({failureReadsBeforeShutdown:hook.failures[${JSON.stringify(paths.restarted)}],
+                pendingBeforeShutdown:await isPending(${JSON.stringify(paths.restarted)}),databaseAbsentBeforeShutdown:!state||state.deleted});
+            `
+        );
+
+        await session!.app.stop();
+        session = undefined;
+        await start();
+
+        const restartObservation = await evaluate<{
+            currentBody: string;
+            revision: string;
+            history: string[];
+            pendingResolved: boolean;
+        }>(
+            cliBinary,
+            session!,
+            `
+            await waitFor(async()=>{
+                const state=await databaseState(${JSON.stringify(paths.restarted)});
+                return !!state&&!state.deleted&&bodyMatches(state.currentBody,${JSON.stringify(content.restart)})&&
+                    !(await isPending(${JSON.stringify(paths.restarted)}));
+            },'Natural restart did not replay the pending file event with restored reads.');
+            const state=await databaseState(${JSON.stringify(paths.restarted)});
+            const file=app.vault.getAbstractFileByPath(${JSON.stringify(paths.restarted)});
+            require(!!file&&await app.vault.read(file)===${JSON.stringify(content.restart)},
+                'The restart fixture Vault file changed during local replay.');
+            require(state.branches.length===1&&state.branches[0].history.length===1,
+                'Restart replay created an unexpected branch or revision.');
+            return JSON.stringify({currentBody:state.currentBody,revision:state.rev,
+                history:state.branches[0].history,pendingResolved:!(await isPending(${JSON.stringify(paths.restarted)}))});
+            `
+        );
+
+        console.log(
+            JSON.stringify(
+                {
+                    ok: true,
+                    rename: renameObservation,
+                    samePathEdit: samePathObservation,
+                    restartBeforeShutdown: restartFailureObservation,
+                    restartRecovery: restartObservation,
+                },
+                null,
+                2
+            )
+        );
+    } finally {
+        if (session) {
+            await evalObsidianJson(
+                cliBinary,
+                `(()=>{const state=globalThis['__livesyncLocalFileRetry'];state?.restore?.();return JSON.stringify(true);})()`,
+                session.cliEnv
+            ).catch(() => undefined);
+            await session.app.stop();
+        }
+        await vault.dispose();
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exit(1);
+});

test/e2e-obsidian/scripts/run-focused.ts

@@ -41,6 +41,7 @@ const focusedScenarios = new Set([
     "startup-integrity",
     "startup-content-check",
     "stale-file-restart",
+    "local-file-retry",
     "folder-batch",
     "setup-uri-workflow",
     "two-vault-sync",