mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-11-15 12:40:12 +00:00
test: cover recovery of failed local file operations
Record the accepted retry decision, its alternatives, and preservation boundaries. Add real Obsidian coverage for temporary-read failure followed by rename, a later edit, and recovery after restart. The scenario uses an exact Commonlib candidate until its release is adopted. Refs #1331
This commit is contained in:
vorotamoroz committed
1 parent
c2e4e9198b
commit
8b77127af5
7 files changed
+734
-1
No files matched your search
@@ -0,0 +1,401 @@
|
||||
import { evalObsidianJson } from "../runner/cli.ts";
|
||||
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
|
||||
import {
|
||||
assertEqual,
|
||||
createE2eObsidianDeviceLocalState,
|
||||
waitForLiveSyncCoreReady,
|
||||
} from "../runner/liveSyncWorkflow.ts";
|
||||
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
|
||||
import { createTemporaryVault } from "../runner/vault.ts";
|
||||
|
||||
process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000";
|
||||
|
||||
const paths = {
|
||||
temporary: "retry-rename-temporary.md",
|
||||
renamed: "retry-rename-final.md",
|
||||
edited: "retry-same-path.md",
|
||||
restarted: "retry-after-restart.md",
|
||||
};
|
||||
const content = {
|
||||
renamed: "Rename recovery content\n",
|
||||
initial: "Initial failed save\n",
|
||||
firstEdit: "First later edit\n",
|
||||
latestEdit: "Latest later edit\n",
|
||||
restart: "Restart recovery content\n",
|
||||
};
|
||||
|
||||
/** Shared in-app probes and controlled read-failure injection; each evaluation resolves the current plug-in instance. */
|
||||
const evaluationHelpers = `
|
||||
const core=app.plugins.plugins['obsidian-livesync'].core;
|
||||
const manager=core.serviceModules.storageAccess.vaultManager;
|
||||
const sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms));
|
||||
const require=(condition,message)=>{if(!condition)throw new Error(message);};
|
||||
const waitFor=async(predicate,message)=>{
|
||||
const until=Date.now()+15000;
|
||||
while(Date.now()<until){if(await predicate())return;await sleep(25);}
|
||||
throw new Error(message);
|
||||
};
|
||||
const pendingEvents=async()=>await manager.adapter.persistence.loadSnapshot()??[];
|
||||
const eventPaths=event=>{
|
||||
const args=event.args??{};
|
||||
return [args.file?.path,args.oldFile?.path,args.oldPath,args.renameTarget,args.targetPath,args.path]
|
||||
.filter(path=>typeof path==='string');
|
||||
};
|
||||
const isPending=async(...paths)=>{
|
||||
const snapshot=await pendingEvents();
|
||||
return snapshot.some(event=>eventPaths(event).some(path=>paths.includes(path)));
|
||||
};
|
||||
const databaseState=async path=>{
|
||||
let meta;
|
||||
try{meta=await core.localDatabase.getDBEntryMeta(path,{conflicts:true},true);}
|
||||
catch(error){if(error?.status===404||error?.name==='not_found')return null;throw error;}
|
||||
if(!meta)return null;
|
||||
const revisions=[...new Set([meta._rev,...(meta._conflicts??[])].filter(Boolean))];
|
||||
const branches=[];
|
||||
for(const rev of revisions){
|
||||
const doc=await core.localDatabase.getDBEntry(path,{rev,revs:true},false,true,true).catch(()=>null);
|
||||
const raw=await core.localDatabase.getRaw(meta._id,{rev,revs:true}).catch(()=>null);
|
||||
const history=raw?._revisions?.ids?.map((id,index)=>(raw._revisions.start-index)+'-'+id)??[];
|
||||
const value=Array.isArray(doc?.data)?doc.data.join(''):typeof doc?.data==='string'?doc.data:null;
|
||||
branches.push({rev,body:value,deleted:!!doc?._deleted,history});
|
||||
}
|
||||
const current=branches.find(branch=>branch.rev===meta._rev);
|
||||
return {path:meta.path,deleted:!!meta.deleted,present:!!current&&!meta.deleted&&!current.deleted,rev:meta._rev,
|
||||
currentBody:current?.body??null,branches};
|
||||
};
|
||||
const bodyMatches=(actual,expected)=>actual===expected||actual===btoa(expected);
|
||||
const installReadFailure=paths=>{
|
||||
const key='__livesyncLocalFileRetry';
|
||||
globalThis[key]?.restore?.();
|
||||
const adapter=app.vault.adapter;
|
||||
const original=adapter.read;
|
||||
const state={blocked:new Set(paths),failures:Object.create(null),holdPath:null,
|
||||
holdNextSuccessfulRead:false,heldReadStarted:false,heldReadBody:null,readGate:null,read:null,restore:null};
|
||||
state.restore=()=>{
|
||||
if(adapter.read===state.read)adapter.read=original;
|
||||
if(globalThis[key]===state)delete globalThis[key];
|
||||
};
|
||||
state.read=async function(path){
|
||||
if(state.blocked.has(path)){
|
||||
state.failures[path]=(state.failures[path]??0)+1;
|
||||
const error=new Error('ENOENT: injected read failure for '+path);
|
||||
error.code='ENOENT';
|
||||
throw error;
|
||||
}
|
||||
const value=await original.call(this,path);
|
||||
if(state.holdNextSuccessfulRead&&path===state.holdPath){
|
||||
state.holdNextSuccessfulRead=false;
|
||||
state.heldReadStarted=true;
|
||||
state.heldReadBody=value;
|
||||
await state.readGate;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
globalThis[key]=state;
|
||||
adapter.read=state.read;
|
||||
return state;
|
||||
};
|
||||
`;
|
||||
|
||||
/**
|
||||
* Evaluate one scenario step inside the running isolated Obsidian session.
|
||||
*
|
||||
* @typeParam T - The JSON result expected by the calling assertion.
|
||||
* @param cliBinary - The discovered Obsidian CLI executable.
|
||||
* @param session - The session whose environment identifies the isolated Vault.
|
||||
* @param body - The scenario expression using the shared in-app probes.
|
||||
* @returns The decoded scenario result; evaluation and transport failures reject the promise.
|
||||
*/
|
||||
async function evaluate<T>(
|
||||
cliBinary: string,
|
||||
session: ObsidianLiveSyncSession,
|
||||
body: string
|
||||
): Promise<T> {
|
||||
return await evalObsidianJson<T>(
|
||||
cliBinary,
|
||||
`(async()=>{${evaluationHelpers}\n${body}\n})()`,
|
||||
session.cliEnv
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Exercise rename, later-edit, and restart recovery through real Vault events.
|
||||
*
|
||||
* @returns Completion after the isolated session is closed; assertion or setup failures reject.
|
||||
* @remarks
|
||||
* Read failures are injected only into the temporary Vault. Content, pending
|
||||
* ownership, and revision history are checked independently of remote replication.
|
||||
*/
|
||||
async function main(): Promise<void> {
|
||||
const binary = requireObsidianBinary();
|
||||
const cli = discoverObsidianCli();
|
||||
if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked: ${cli.checked.join(", ")}`);
|
||||
const cliBinary = cli.binary;
|
||||
const vault = await createTemporaryVault("obsidian-local-file-retry-");
|
||||
let session: ObsidianLiveSyncSession | undefined;
|
||||
|
||||
/**
|
||||
* Start or reopen the same isolated Vault and profile.
|
||||
*
|
||||
* @param initial - Whether to seed the first run with local-only plug-in settings.
|
||||
* @returns Completion when the session and LiveSync core are ready for the next scenario.
|
||||
*/
|
||||
const start = async (initial = false): Promise<void> => {
|
||||
session = await startObsidianLiveSyncSession({
|
||||
binary,
|
||||
cliBinary,
|
||||
vault,
|
||||
pluginStartup: initial ? undefined : "natural",
|
||||
pluginData: initial
|
||||
? {
|
||||
doctorProcessedVersion: "1.0.0",
|
||||
isConfigured: true,
|
||||
liveSync: false,
|
||||
remoteType: "",
|
||||
couchDB_URI: "http://127.0.0.1:5984",
|
||||
couchDB_DBNAME: "local-file-retry",
|
||||
notifyThresholdOfRemoteStorageSize: -1,
|
||||
periodicReplication: false,
|
||||
syncAfterMerge: false,
|
||||
syncOnEditorSave: false,
|
||||
syncOnFileOpen: false,
|
||||
syncOnSave: false,
|
||||
syncOnStart: false,
|
||||
disableMarkdownAutoMerge: false,
|
||||
resolveConflictsByNewerFile: false,
|
||||
checkConflictOnlyOnOpen: true,
|
||||
showMergeDialogOnlyOnActive: true,
|
||||
}
|
||||
: undefined,
|
||||
localStorageEntries: initial ? createE2eObsidianDeviceLocalState(vault.name) : undefined,
|
||||
});
|
||||
const readiness = await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
|
||||
assertEqual(readiness.configured, true, "Self-hosted LiveSync did not start configured.");
|
||||
};
|
||||
|
||||
try {
|
||||
await start(true);
|
||||
|
||||
const renameObservation = await evaluate<{
|
||||
firstFailureReads: number;
|
||||
renameFailureReads: number;
|
||||
finalRevision: string;
|
||||
finalHistory: string[];
|
||||
obsoleteTemporaryPathAbsent: boolean;
|
||||
}>(
|
||||
cliBinary,
|
||||
session!,
|
||||
`
|
||||
const hook=installReadFailure([${JSON.stringify(paths.temporary)}]);
|
||||
try{
|
||||
await app.vault.create(${JSON.stringify(paths.temporary)},${JSON.stringify(content.renamed)});
|
||||
await waitFor(async()=>hook.failures[${JSON.stringify(paths.temporary)}]>0&&
|
||||
await isPending(${JSON.stringify(paths.temporary)}),
|
||||
'The temporary file read failure was not retained as a pending Vault event.');
|
||||
const firstFailureReads=hook.failures[${JSON.stringify(paths.temporary)}];
|
||||
require(!(await databaseState(${JSON.stringify(paths.temporary)}))?.branches?.length,
|
||||
'The failed temporary file save unexpectedly reached the local database.');
|
||||
|
||||
hook.blocked.add(${JSON.stringify(paths.renamed)});
|
||||
let renameEvents=0;
|
||||
const renameRef=app.vault.on('rename',(file,oldPath)=>{
|
||||
if(file.path===${JSON.stringify(paths.renamed)}&&oldPath===${JSON.stringify(paths.temporary)})renameEvents++;
|
||||
});
|
||||
try{
|
||||
const file=app.vault.getAbstractFileByPath(${JSON.stringify(paths.temporary)});
|
||||
require(!!file,'The temporary file disappeared before the real Vault rename.');
|
||||
await app.vault.rename(file,${JSON.stringify(paths.renamed)});
|
||||
require(renameEvents===1,'The real Vault rename event was not observed exactly once.');
|
||||
}finally{app.vault.offref(renameRef);}
|
||||
|
||||
await waitFor(async()=>hook.failures[${JSON.stringify(paths.renamed)}]>0&&
|
||||
await isPending(${JSON.stringify(paths.renamed)}),
|
||||
'The final-path read failure and rename event were not retained.');
|
||||
const renameFailureReads=hook.failures[${JSON.stringify(paths.renamed)}];
|
||||
const finalFile=app.vault.getAbstractFileByPath(${JSON.stringify(paths.renamed)});
|
||||
require(!!finalFile,'The real Vault rename did not create the final path.');
|
||||
hook.restore();
|
||||
require(await app.vault.read(finalFile)===${JSON.stringify(content.renamed)},
|
||||
'The real Vault rename did not preserve the final file bytes.');
|
||||
|
||||
require(await core.services.fileProcessing.commitPendingFileEvents(),
|
||||
'Pending rename recovery did not complete after restoring reads.');
|
||||
await waitFor(async()=>!(await isPending(${JSON.stringify(paths.temporary)},${JSON.stringify(paths.renamed)})),
|
||||
'The temporary or final path remained pending after rename recovery.');
|
||||
const finalState=await databaseState(${JSON.stringify(paths.renamed)});
|
||||
const oldState=await databaseState(${JSON.stringify(paths.temporary)});
|
||||
require(finalState&&finalState.path===${JSON.stringify(paths.renamed)}&&!finalState.deleted&&
|
||||
bodyMatches(finalState.currentBody,${JSON.stringify(content.renamed)}),
|
||||
'The final current database document does not contain the renamed file bytes at the final path.');
|
||||
require(finalState.branches.length===1&&finalState.branches[0].history.length===1,
|
||||
'Rename recovery created an unexpected branch or revision.');
|
||||
require(!oldState||oldState.deleted,
|
||||
'The obsolete temporary path remains a current database file.');
|
||||
return JSON.stringify({firstFailureReads,renameFailureReads,finalRevision:finalState.rev,
|
||||
finalHistory:finalState.branches[0].history,obsoleteTemporaryPathAbsent:!oldState||oldState.deleted});
|
||||
}finally{hook.restore();}
|
||||
`
|
||||
);
|
||||
|
||||
const samePathObservation = await evaluate<{
|
||||
initialFailureReads: number;
|
||||
earlierReadBody: string;
|
||||
laterEditPendingWhileEarlierReadWasHeld: boolean;
|
||||
newestEditObservedWhileReadWasHeld: boolean;
|
||||
latestBody: string;
|
||||
finalRevision: string;
|
||||
finalHistory: string[];
|
||||
}>(
|
||||
cliBinary,
|
||||
session!,
|
||||
`
|
||||
const path=${JSON.stringify(paths.edited)};
|
||||
const hook=installReadFailure([path]);
|
||||
let releaseRead;
|
||||
const readGate=new Promise(resolve=>{releaseRead=resolve;});
|
||||
let readReleased=false;
|
||||
try{
|
||||
await app.vault.create(path,${JSON.stringify(content.initial)});
|
||||
await waitFor(async()=>hook.failures[path]>0&&await isPending(path),
|
||||
'The initial same-path read failure was not retained.');
|
||||
const initialFailureReads=hook.failures[path];
|
||||
|
||||
hook.holdPath=path;
|
||||
hook.readGate=readGate;
|
||||
hook.holdNextSuccessfulRead=true;
|
||||
hook.blocked.delete(path);
|
||||
const file=app.vault.getAbstractFileByPath(path);
|
||||
require(!!file,'The same-path fixture file disappeared after its initial failure.');
|
||||
const initialEventKeys=new Set((await pendingEvents()).filter(event=>eventPaths(event).includes(path)).map(event=>event.key));
|
||||
await app.vault.modify(file,${JSON.stringify(content.firstEdit)});
|
||||
await waitFor(()=>hook.heldReadStarted,
|
||||
'A real same-path edit did not retry the failed local save.');
|
||||
await waitFor(async()=>(await pendingEvents()).some(event=>eventPaths(event).includes(path)&&!initialEventKeys.has(event.key)),
|
||||
'The edit which triggered recovery was not retained while the earlier read was held.');
|
||||
const earlierReadBody=hook.heldReadBody;
|
||||
require(bodyMatches(earlierReadBody,${JSON.stringify(content.initial)})||
|
||||
bodyMatches(earlierReadBody,${JSON.stringify(content.firstEdit)}),
|
||||
'The held retry read did not capture the pre-latest file body.');
|
||||
require(!bodyMatches(earlierReadBody,${JSON.stringify(content.latestEdit)}),
|
||||
'The held read already contained the later edit.');
|
||||
require(await isPending(path),
|
||||
'The failed save stopped being pending while its retry read was held.');
|
||||
|
||||
let newestEditEvents=0;
|
||||
const modifyRef=app.vault.on('modify',changed=>{if(changed.path===path)newestEditEvents++;});
|
||||
try{
|
||||
await app.vault.modify(file,${JSON.stringify(content.latestEdit)});
|
||||
require(newestEditEvents>0,'The newest real Vault modification was not observed.');
|
||||
}finally{app.vault.offref(modifyRef);}
|
||||
// The existing read guard can suppress this newest notification.
|
||||
// The already admitted successor must still save the latest bytes.
|
||||
require(await app.vault.read(file)===${JSON.stringify(content.latestEdit)},
|
||||
'The second later edit did not reach the real Vault file.');
|
||||
require(await isPending(path),
|
||||
'The admitted successor disappeared while the earlier read was held.');
|
||||
readReleased=true;
|
||||
releaseRead();
|
||||
|
||||
await waitFor(async()=>{
|
||||
const state=await databaseState(path);
|
||||
return !!state&&state.present&&bodyMatches(state.currentBody,${JSON.stringify(content.latestEdit)})&&
|
||||
!(await isPending(path));
|
||||
},'The latest same-path body was not committed before its pending event resolved.');
|
||||
const finalState=await databaseState(path);
|
||||
require(bodyMatches(finalState.currentBody,${JSON.stringify(content.latestEdit)}),
|
||||
'An earlier completion consumed or replaced the latest same-path body.');
|
||||
require(finalState.branches.some(branch=>bodyMatches(branch.body,${JSON.stringify(content.latestEdit)})),
|
||||
'The latest same-path body is absent from the current revision leaves.');
|
||||
require(await app.vault.read(file)===${JSON.stringify(content.latestEdit)},
|
||||
'The earlier completion replaced the latest real Vault body.');
|
||||
return JSON.stringify({initialFailureReads,earlierReadBody,
|
||||
laterEditPendingWhileEarlierReadWasHeld:true,newestEditObservedWhileReadWasHeld:true,latestBody:finalState.currentBody,
|
||||
finalRevision:finalState.rev,
|
||||
finalHistory:finalState.branches.find(branch=>branch.rev===finalState.rev)?.history??[]});
|
||||
}finally{
|
||||
if(!readReleased)releaseRead();
|
||||
hook.restore();
|
||||
}
|
||||
`
|
||||
);
|
||||
|
||||
const restartFailureObservation = await evaluate<{
|
||||
failureReadsBeforeShutdown: number;
|
||||
pendingBeforeShutdown: boolean;
|
||||
databaseAbsentBeforeShutdown: boolean;
|
||||
}>(
|
||||
cliBinary,
|
||||
session!,
|
||||
`
|
||||
const hook=installReadFailure([${JSON.stringify(paths.restarted)}]);
|
||||
await app.vault.create(${JSON.stringify(paths.restarted)},${JSON.stringify(content.restart)});
|
||||
await waitFor(async()=>hook.failures[${JSON.stringify(paths.restarted)}]>0&&
|
||||
await isPending(${JSON.stringify(paths.restarted)}),
|
||||
'The restart fixture read failure was not retained.');
|
||||
const state=await databaseState(${JSON.stringify(paths.restarted)});
|
||||
require(!state||!state.present,
|
||||
'The restart fixture unexpectedly saved before shutdown.');
|
||||
return JSON.stringify({failureReadsBeforeShutdown:hook.failures[${JSON.stringify(paths.restarted)}],
|
||||
pendingBeforeShutdown:await isPending(${JSON.stringify(paths.restarted)}),databaseAbsentBeforeShutdown:!state||state.deleted});
|
||||
`
|
||||
);
|
||||
|
||||
await session!.app.stop();
|
||||
session = undefined;
|
||||
await start();
|
||||
|
||||
const restartObservation = await evaluate<{
|
||||
currentBody: string;
|
||||
revision: string;
|
||||
history: string[];
|
||||
pendingResolved: boolean;
|
||||
}>(
|
||||
cliBinary,
|
||||
session!,
|
||||
`
|
||||
await waitFor(async()=>{
|
||||
const state=await databaseState(${JSON.stringify(paths.restarted)});
|
||||
return !!state&&!state.deleted&&bodyMatches(state.currentBody,${JSON.stringify(content.restart)})&&
|
||||
!(await isPending(${JSON.stringify(paths.restarted)}));
|
||||
},'Natural restart did not replay the pending file event with restored reads.');
|
||||
const state=await databaseState(${JSON.stringify(paths.restarted)});
|
||||
const file=app.vault.getAbstractFileByPath(${JSON.stringify(paths.restarted)});
|
||||
require(!!file&&await app.vault.read(file)===${JSON.stringify(content.restart)},
|
||||
'The restart fixture Vault file changed during local replay.');
|
||||
require(state.branches.length===1&&state.branches[0].history.length===1,
|
||||
'Restart replay created an unexpected branch or revision.');
|
||||
return JSON.stringify({currentBody:state.currentBody,revision:state.rev,
|
||||
history:state.branches[0].history,pendingResolved:!(await isPending(${JSON.stringify(paths.restarted)}))});
|
||||
`
|
||||
);
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: true,
|
||||
rename: renameObservation,
|
||||
samePathEdit: samePathObservation,
|
||||
restartBeforeShutdown: restartFailureObservation,
|
||||
restartRecovery: restartObservation,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
} finally {
|
||||
if (session) {
|
||||
await evalObsidianJson(
|
||||
cliBinary,
|
||||
`(()=>{const state=globalThis['__livesyncLocalFileRetry'];state?.restore?.();return JSON.stringify(true);})()`,
|
||||
session.cliEnv
|
||||
).catch(() => undefined);
|
||||
await session.app.stop();
|
||||
}
|
||||
await vault.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.stack : error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -41,6 +41,7 @@ const focusedScenarios = new Set([
|
||||
"startup-integrity",
|
||||
"startup-content-check",
|
||||
"stale-file-restart",
|
||||
"local-file-retry",
|
||||
"folder-batch",
|
||||
"setup-uri-workflow",
|
||||
"two-vault-sync",
|
||||
|
||||
Reference in new issue
Block a user