Merge current main for Setup URI dependency alignment

test/e2e-obsidian/scripts/chunk-fetch-retry.ts

@@ -0,0 +1,365 @@
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import type { Page } from "playwright";
+import {
+    assertCouchDbReachable,
+    createCouchDbDatabase,
+    deleteCouchDbDatabase,
+    loadCouchDbConfig,
+    makeUniqueDatabaseName,
+    putCouchDbDocument,
+    type CouchDbConfig,
+} from "../runner/couchdb.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertEqual,
+    createE2eCouchDbPluginData,
+    createE2eObsidianDeviceLocalState,
+    prepareRemote,
+    waitForLiveSyncCoreReady,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { withObsidianPage } from "../runner/ui.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+const observationKey = "__livesyncChunkFetchRetryE2E";
+const observationSource = `globalThis[${JSON.stringify(observationKey)}]`;
+const retryDelayMs = 2_000;
+
+type FetchAttempt = {
+    startedAt: number;
+    finiteTransitionIndex: number;
+    completedAt?: number;
+    requestedIds: string[];
+    returnedIds?: string[];
+    unavailable?: boolean;
+    error?: string;
+};
+
+type Snapshot = {
+    attempts: FetchAttempt[];
+    missingEvents: number[];
+    replicationDone: boolean;
+    replicationSucceeded?: boolean;
+    replicationError?: string;
+    metadataPresent: boolean;
+    chunkPresent: boolean;
+    claimActive: boolean;
+    currentProcessing: number;
+    queued: number;
+    boundedActivity: number;
+    finiteActivity: number;
+    finiteTransitions: { at: number; count: number }[];
+    followupTransitionIndex?: number;
+    replicationResults: number;
+    databaseQueue: number;
+    storageApplying: number;
+    pendingChunkCount: number;
+    initialChunkCount: number;
+    retryChunkCount: number;
+    statusText: string;
+    content: string | null;
+};
+
+async function snapshot(page: Page): Promise<Snapshot> {
+    return await page.evaluate(`(async()=>{
+        const state=${observationSource};
+        const core=app.plugins.plugins['obsidian-livesync'].core;
+        const db=core.localDatabase;
+        const rows=await db.allDocsRaw({keys:[state.metadataId,state.chunkId],include_docs:true});
+        const present=(id)=>rows.rows.some((row)=>row.id===id&&row.doc&&!row.value?.deleted);
+        const file=app.vault.getAbstractFileByPath(state.path);
+        const statusText=document.querySelector('.syncstatusbar')?.textContent??'';
+        const initialChunkCount=Number(statusText.match(/🛄\\s*(\\d+)/u)?.[1]??0);
+        const retryChunkCount=Number(statusText.match(/🔁\\s*(\\d+)/u)?.[1]??0);
+        return {
+            attempts:state.attempts,missingEvents:state.missingEvents,
+            replicationDone:state.replicationDone,replicationSucceeded:state.replicationSucceeded,
+            replicationError:state.replicationError,metadataPresent:present(state.metadataId),
+            chunkPresent:present(state.chunkId),
+            claimActive:db.managers.chunkManager.deliveryCoordinator.isClaimActiveFor(state.chunkId),
+            currentProcessing:db.managers.chunkFetcher.currentProcessing,
+            queued:db.managers.chunkFetcher.queue.length,
+            boundedActivity:core.services.replicator.boundedRemoteActivityCount.value,
+            finiteActivity:core.services.replicator.finiteReplicationActivityCount.value,
+            finiteTransitions:state.finiteTransitions,followupTransitionIndex:state.followupTransitionIndex,
+            replicationResults:core.services.replication.replicationResultCount.value,
+            databaseQueue:core.services.replication.databaseQueueCount.value,
+            storageApplying:core.services.replication.storageApplyingCount.value,
+            initialChunkCount,retryChunkCount,pendingChunkCount:initialChunkCount+retryChunkCount,statusText,
+            content:file?await app.vault.read(file):null,
+        };
+    })()`);
+}
+
+async function waitForSnapshot(
+    page: Page,
+    predicate: (state: Snapshot) => boolean,
+    stage: string,
+    timeoutMs = 20_000
+): Promise<Snapshot> {
+    const deadline = Date.now() + timeoutMs;
+    let state: Snapshot;
+    do {
+        state = await snapshot(page);
+        if (predicate(state)) return state;
+        if (state.replicationError || state.attempts.some((attempt) => attempt.error || attempt.unavailable)) {
+            throw new Error(`The real CouchDB operation failed during ${stage}: ${JSON.stringify(state)}`);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 50));
+    } while (Date.now() < deadline);
+    throw new Error(`Timed out during ${stage}: ${JSON.stringify(state)}`);
+}
+
+function isIdle(state: Snapshot): boolean {
+    return (
+        state.replicationDone &&
+        !state.claimActive &&
+        state.currentProcessing === 0 &&
+        state.queued === 0 &&
+        state.boundedActivity === 0 &&
+        state.finiteActivity === 0 &&
+        state.replicationResults === 0 &&
+        state.databaseQueue === 0 &&
+        state.storageApplying === 0 &&
+        state.pendingChunkCount === 0
+    );
+}
+
+async function runScenario(
+    page: Page,
+    couchDb: CouchDbConfig,
+    dbName: string,
+    label: "delayed-arrival" | "permanently-missing" | "finite-completion"
+): Promise<void> {
+    const delayedArrival = label === "delayed-arrival";
+    const expediteFinalProbe = label === "finite-completion";
+    const path = `chunk-fetch-${label}.md`;
+    const chunkId = `h:e2e-chunk-fetch-${label}`;
+    const content = `# Chunk fetch retry\n${label}\n`;
+    const metadataId = await page.evaluate<string>(
+        `app.plugins.plugins['obsidian-livesync'].core.services.path.path2id(${JSON.stringify(path)})`
+    );
+    const now = Date.now();
+    // Only Metadata is initially present. The Chunk is a separate real CouchDB document.
+    await putCouchDbDocument(couchDb, dbName, {
+        _id: metadataId,
+        path,
+        type: "plain",
+        ctime: now,
+        mtime: now,
+        size: Buffer.byteLength(content),
+        children: [chunkId],
+        eden: {},
+    });
+
+    await page.evaluate(`(()=>{
+        if(${observationSource}) throw new Error('A Chunk fetch observer is already installed.');
+        const core=app.plugins.plugins['obsidian-livesync'].core;
+        const settings=core.services.setting.currentSettings();
+        if(!settings.readChunksOnline||settings.useOnlyLocalChunk||settings.liveSync||settings.periodicReplication) {
+            throw new Error('The fixture requires one-shot replication with on-demand Chunk reads.');
+        }
+        const replicator=core.services.replicator.getActiveReplicator();
+        const original=replicator.fetchRemoteChunks;
+        if(typeof original!=='function') throw new Error('The real replicator has no Chunk fetch method.');
+        const manager=core.localDatabase.managers.chunkManager;
+        const observer=new AbortController();
+        const finiteCount=core.services.replicator.finiteReplicationActivityCount;
+        const state={path:${JSON.stringify(path)},metadataId:${JSON.stringify(metadataId)},
+            chunkId:${JSON.stringify(chunkId)},attempts:[],missingEvents:[],replicationDone:false,
+            finiteTransitions:[{at:Date.now(),count:finiteCount.value}]};
+        const observeFinite=()=>state.finiteTransitions.push({at:Date.now(),count:finiteCount.value});
+        finiteCount.onChanged(observeFinite);
+        ${observationSource}=state;
+        state.restore=()=>{replicator.fetchRemoteChunks=original;observer.abort();finiteCount.offChanged(observeFinite);};
+        manager.addListener('missingChunkRemote',(id)=>{
+            if(id===state.chunkId) state.missingEvents.push(Date.now());
+        },{signal:observer.signal});
+        // Observe the real HTTP-backed method without changing its result or adding a wait.
+        replicator.fetchRemoteChunks=async function(...args){
+            if(!args[0].includes(state.chunkId)) return await original.apply(this,args);
+            const attempt={startedAt:Date.now(),finiteTransitionIndex:state.finiteTransitions.length,requestedIds:[...args[0]]};
+            state.attempts.push(attempt);
+            try{
+                const result=await original.apply(this,args);
+                attempt.unavailable=result===false;
+                attempt.returnedIds=Array.isArray(result)?result.map((chunk)=>chunk._id):[];
+                return result;
+            }catch(error){
+                attempt.error=String(error);
+                throw error;
+            }finally{
+                attempt.completedAt=Date.now();
+            }
+        };
+        state.startReplication=async()=>{
+            state.replicationDone=false;
+            try{state.replicationSucceeded=!!(await core.services.replication.replicate(true));}
+            catch(error){state.replicationError=String(error);}
+            finally{state.replicationDone=true;}
+        };
+        state.replication=state.startReplication();
+    })()`);
+
+    try {
+        const first = await waitForSnapshot(
+            page,
+            (state) => !!state.attempts[0]?.completedAt,
+            "initial missing response"
+        );
+        assertEqual(first.metadataPresent, true, "The Metadata did not arrive through real replication.");
+        assertEqual(first.chunkPresent, false, "The Chunk was already available locally before its delayed arrival.");
+        assertEqual(
+            first.attempts[0].unavailable,
+            false,
+            "The first fetch failed instead of returning a missing Chunk."
+        );
+        assertEqual(first.attempts[0].returnedIds?.length, 0, "The first fetch did not reproduce a missing Chunk.");
+        if (delayedArrival) {
+            await putCouchDbDocument(couchDb, dbName, { _id: chunkId, type: "leaf", data: content });
+        }
+        console.log(`${label}: first real response ${JSON.stringify(first)}`);
+        assertEqual(
+            first.missingEvents.length,
+            0,
+            "The first missing response ended delivery before the delayed retry."
+        );
+        assertEqual(first.claimActive, true, "The first missing response released the delivery claim.");
+        assertEqual(first.content, null, "The file was materialised before its missing Chunk arrived.");
+        const retryWaiting = await waitForSnapshot(
+            page,
+            (state) => state.initialChunkCount === 0 && state.retryChunkCount === 1,
+            "separate retry status during the retry delay",
+            1_000
+        );
+        assertEqual(retryWaiting.attempts.length, 1, "The pending count appeared only after the retry started.");
+        const completedDuringInitialLookup = retryWaiting.finiteTransitions
+            .slice(first.attempts[0].finiteTransitionIndex)
+            .some((transition) => transition.count === 0);
+        if (!completedDuringInitialLookup) {
+            assertEqual(retryWaiting.currentProcessing, 0, "Backoff retained a physical request slot.");
+        }
+        console.log(`${label}: retry waiting ${JSON.stringify(retryWaiting)}`);
+
+        if (expediteFinalProbe) {
+            await waitForSnapshot(
+                page,
+                (state) => state.replicationDone && state.finiteActivity === 0 && state.attempts.length === 1,
+                "first finite replication completion before the scheduled retry",
+                1_000
+            );
+            // A second genuine one-shot replication ends during backoff; counts are observed, never synthesised.
+            await page.evaluate(`(()=>{
+                const state=${observationSource};
+                state.followupTransitionIndex=state.finiteTransitions.length;
+                state.replication=state.startReplication();
+            })()`);
+        }
+
+        const completed = await waitForSnapshot(page, isIdle, "delivery and reflection quiescence");
+        assertEqual(completed.replicationSucceeded, true, "One-shot replication did not complete successfully.");
+        assertEqual(completed.attempts.length, 2, "The missing Chunk must be fetched exactly twice.");
+        const [initial, retry] = completed.attempts;
+        const retryDelay = retry.startedAt - initial.completedAt!;
+        if (expediteFinalProbe) {
+            const transitions = completed.finiteTransitions.slice(completed.followupTransitionIndex);
+            const ended = transitions.find(
+                (transition, index) => index > 0 && transition.count === 0 && transitions[index - 1].count > 0
+            );
+            if (!ended)
+                throw new Error(`The second real finite replication was not observed: ${JSON.stringify(transitions)}`);
+            if (retry.startedAt < ended.at) throw new Error("The final probe began before finite replication ended.");
+            if (retryDelay >= retryDelayMs)
+                throw new Error(`Finite completion did not interrupt backoff: ${retryDelay} ms.`);
+        } else {
+            const completion = completed.finiteTransitions
+                .slice(initial.finiteTransitionIndex)
+                .find((transition) => transition.count === 0);
+            if (completion) {
+                if (retry.startedAt < completion.at) throw new Error("The final probe preceded finite completion.");
+                if (retryDelay >= retryDelayMs) {
+                    throw new Error(`Finite completion did not expedite the initial missing result: ${retryDelay} ms.`);
+                }
+            } else if (retryDelay < retryDelayMs) {
+                throw new Error(`The retry started too early: ${retryDelay} ms.`);
+            }
+        }
+        assertEqual(retry.requestedIds.join(","), chunkId, "The retry requested an unexpected Chunk.");
+        assertEqual(retry.unavailable, false, "The retry failed to contact the real remote.");
+        assertEqual(retry.returnedIds?.join(","), delayedArrival ? chunkId : "", "The retry returned unexpected data.");
+        assertEqual(
+            completed.missingEvents.length,
+            delayedArrival ? 0 : 1,
+            "Unexpected terminal missing notifications."
+        );
+        assertEqual(completed.chunkPresent, delayedArrival, "The local Chunk persistence result was unexpected.");
+        assertEqual(completed.content, delayedArrival ? content : null, "The Vault file content was unexpected.");
+        if (!delayedArrival) {
+            // Observe one more retry interval after quiescence to reject an unbounded retry loop.
+            await new Promise((resolve) => setTimeout(resolve, retryDelayMs + 100));
+            const settled = await snapshot(page);
+            assertEqual(isIdle(settled), true, "The permanently missing delivery became active again.");
+            assertEqual(settled.attempts.length, 2, "The permanently missing Chunk was retried again.");
+        }
+        console.log(`${label}: passed; retry after ${retryDelay} ms; ${JSON.stringify(completed)}`);
+    } catch (error) {
+        console.error(`${label}: ${JSON.stringify(await snapshot(page))}`);
+        const diagnostics = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e";
+        await mkdir(diagnostics, { recursive: true });
+        await page.screenshot({ path: join(diagnostics, `chunk-fetch-${label}.failure.png`), fullPage: true });
+        throw error;
+    } finally {
+        await page.evaluate(`(()=>{${observationSource}?.restore();delete ${observationSource};})()`);
+    }
+}
+
+async function main(): Promise<void> {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked: ${cli.checked.join(", ")}`);
+    const couchDb = await loadCouchDbConfig();
+    await assertCouchDbReachable(couchDb);
+    const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "chunk-fetch-retry");
+    const vault = await createTemporaryVault("obsidian-livesync-chunk-fetch-");
+    let session: ObsidianLiveSyncSession | undefined;
+    try {
+        await createCouchDbDatabase(couchDb, dbName);
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: cli.binary,
+            vault,
+            pluginData: createE2eCouchDbPluginData(
+                { ...couchDb, dbName },
+                {
+                    encrypt: false,
+                    usePathObfuscation: false,
+                    showStatusOnStatusbar: true,
+                    // Keep the real retry status observable even when finite completion expedites the final probe.
+                    minimumIntervalOfReadChunksOnline: 500,
+                    periodicReplication: false,
+                    syncOnFileOpen: false,
+                    syncOnEditorSave: false,
+                    syncAfterMerge: false,
+                }
+            ),
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+        });
+        await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
+        await prepareRemote(cli.binary, session.cliEnv);
+        await withObsidianPage(session.remoteDebuggingPort, async (page) => {
+            await runScenario(page, couchDb, dbName, "delayed-arrival");
+            await runScenario(page, couchDb, dbName, "permanently-missing");
+            await runScenario(page, couchDb, dbName, "finite-completion");
+        });
+    } finally {
+        if (session) await session.app.stop();
+        await vault.dispose();
+        await deleteCouchDbDatabase(couchDb, dbName);
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exitCode = 1;
+});

test/e2e-obsidian/scripts/received-change-readiness.ts

@@ -0,0 +1,420 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { EVENT_APPLICATION_READY } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents";
+import { evalObsidianJson } from "../runner/cli.ts";
+import {
+    assertCouchDbReachable,
+    createCouchDbDatabase,
+    deleteCouchDbDatabase,
+    fetchCouchDbDocument,
+    loadCouchDbConfig,
+    makeUniqueDatabaseName,
+    putCouchDbDocument,
+    waitForCouchDbDocs,
+    type CouchDbConfig,
+    type CouchDbDocument,
+} from "../runner/couchdb.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertEqual,
+    createE2eCouchDbPluginData,
+    createE2eObsidianDeviceLocalState,
+    prepareRemote,
+    pushLocalChanges,
+    waitForLiveSyncCoreReady,
+    waitForLocalDatabaseEntry,
+    type LocalDatabaseEntry,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts";
+
+process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "20000";
+
+const observerKey = "__livesyncE2eReceivedChangeReadiness";
+const readyPath = "E2E/received-change-readiness/ready.md";
+const suspendedPath = "E2E/received-change-readiness/suspended.md";
+const readyContent = [
+    "# Received before readiness",
+    "",
+    "This note is replicated into the target database before the application readiness event.",
+    "Its content is longer than one configured chunk so the test uses the ordinary Chunk path.",
+    "",
+].join("\n");
+const suspendedContent = [
+    "# Received during explicit suspension",
+    "",
+    "This note remains queued while database reflecting is explicitly suspended.",
+    "Resuming result application must reflect the received metadata and its stored Chunks.",
+    "",
+].join("\n");
+
+type ReadinessObservation = {
+    readinessEvents: number;
+    content: string | null;
+};
+
+type CapturedNote = {
+    entry: LocalDatabaseEntry;
+    documents: CouchDbDocument[];
+};
+
+async function waitForVaultContent(
+    vault: TemporaryVault,
+    path: string,
+    expected: string,
+    timeoutMs = Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ?? 10000)
+): Promise<void> {
+    const fullPath = join(vault.path, path);
+    const deadline = Date.now() + timeoutMs;
+    let lastContent: string | null = null;
+    while (Date.now() < deadline) {
+        try {
+            lastContent = await readFile(fullPath, "utf8");
+            if (lastContent === expected) return;
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+        }
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+    throw new Error(`Timed out waiting for ${path} in the target Vault. Last content: ${String(lastContent)}`);
+}
+
+async function assertVaultPathStaysMissing(vault: TemporaryVault, path: string, durationMs: number): Promise<void> {
+    const fullPath = join(vault.path, path);
+    const deadline = Date.now() + durationMs;
+    while (Date.now() < deadline) {
+        try {
+            const content = await readFile(fullPath, "utf8");
+            throw new Error(`The suspended or unready document was reflected early at ${path}: ${content}`);
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+        }
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+}
+
+async function createAndUploadNote(
+    cliBinary: string,
+    env: NodeJS.ProcessEnv,
+    couchDb: CouchDbConfig,
+    dbName: string,
+    path: string,
+    content: string
+): Promise<CapturedNote> {
+    const created = await evalObsidianJson<{ path: string; content: string }>(
+        cliBinary,
+        `(async()=>{const path=${JSON.stringify(path)};const folders=path.split('/');for(let i=1;i<folders.length;i++){const folder=folders.slice(0,i).join('/');if(!app.vault.getAbstractFileByPath(folder)) await app.vault.createFolder(folder);}const file=await app.vault.create(path,${JSON.stringify(content)});return JSON.stringify({path:file.path,content:await app.vault.read(file)});})()`,
+        env
+    );
+    assertEqual(created.path, path, "Obsidian created the source note at an unexpected path.");
+    assertEqual(created.content, content, "Obsidian did not read back the source note content.");
+    const entry = await waitForLocalDatabaseEntry(cliBinary, env, path);
+    if (entry.children.length === 0) throw new Error(`The source note did not create Chunks: ${path}`);
+    await pushLocalChanges(cliBinary, env);
+    await waitForCouchDbDocs(couchDb, dbName, (documents) => {
+        const ids = new Set(documents.map((document) => document._id));
+        return ids.has(entry.id) && entry.children.every((childId) => ids.has(childId));
+    });
+    const documents = await Promise.all(
+        [...new Set([entry.id, ...entry.children])].map((documentId) =>
+            fetchCouchDbDocument(couchDb, dbName, documentId)
+        )
+    );
+    return { entry, documents };
+}
+
+async function injectCapturedNote(
+    couchDb: CouchDbConfig,
+    dbName: string,
+    note: CapturedNote,
+    publishedChunkIds: Set<string>
+): Promise<void> {
+    const chunks = note.documents.filter((document) => document._id !== note.entry.id);
+    const metadata = note.documents.find((document) => document._id === note.entry.id);
+    if (!metadata) throw new Error(`The source note metadata was not captured: ${note.entry.path}`);
+    for (const document of chunks) {
+        if (publishedChunkIds.has(document._id)) continue;
+        const freshDocument = { ...document };
+        delete freshDocument._rev;
+        await putCouchDbDocument(couchDb, dbName, freshDocument);
+        publishedChunkIds.add(document._id);
+    }
+    const freshMetadata = { ...metadata };
+    delete freshMetadata._rev;
+    await putCouchDbDocument(couchDb, dbName, freshMetadata);
+}
+
+async function installObserver(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
+    await evalObsidianJson(
+        cliBinary,
+        [
+            "(()=>{",
+            `const key=${JSON.stringify(observerKey)};`,
+            `const readyEvent=${JSON.stringify(EVENT_APPLICATION_READY)};`,
+            "const previous=globalThis[key];",
+            "if(previous) previous.unsubscribeReady?.();",
+            "const state={readinessEvents:0,unsubscribeReady:null};",
+            "state.unsubscribeReady=app.plugins.plugins['obsidian-livesync'].core.services.context.events.onEvent(readyEvent,()=>state.readinessEvents++);",
+            "globalThis[key]=state;",
+            "return JSON.stringify(true);",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function readObservation(cliBinary: string, env: NodeJS.ProcessEnv, path: string): Promise<ReadinessObservation> {
+    return await evalObsidianJson<ReadinessObservation>(
+        cliBinary,
+        [
+            "(async()=>{",
+            `const key=${JSON.stringify(observerKey)};`,
+            `const path=${JSON.stringify(path)};`,
+            "const state=globalThis[key];",
+            "const file=app.vault.getAbstractFileByPath(path);",
+            "return JSON.stringify({readinessEvents:state?.readinessEvents??0,content:file?await app.vault.read(file):null});",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function resetReadiness(cliBinary: string, env: NodeJS.ProcessEnv, suspendReflecting: boolean): Promise<void> {
+    const result = await evalObsidianJson<{ ready: boolean }>(
+        cliBinary,
+        [
+            "(async()=>{",
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            ...(suspendReflecting
+                ? [
+                      "await core.services.setting.applyPartial({suspendParseReplicationResult:true},true);",
+                      "await core.services.control.applySettings();",
+                      "await core.services.replication.startContinuous({trigger:'daemon',interaction:{kind:'forbidden'}});",
+                  ]
+                : []),
+            "core.services.appLifecycle.resetIsReady();",
+            "return JSON.stringify({ready:core.services.appLifecycle.isReady()});",
+            "})()",
+        ].join(""),
+        env
+    );
+    assertEqual(result.ready, false, "The target application remained ready after resetIsReady().");
+}
+
+async function waitForReceivedWhileUnready(
+    cliBinary: string,
+    env: NodeJS.ProcessEnv,
+    target: TemporaryVault,
+    path: string
+): Promise<void> {
+    const entry = await waitForLocalDatabaseEntry(cliBinary, env, path);
+    if (entry.children.length === 0) throw new Error(`The target received metadata without Chunks: ${path}`);
+    const readiness = await evalObsidianJson<{ ready: boolean }>(
+        cliBinary,
+        "(()=>JSON.stringify({ready:app.plugins.plugins['obsidian-livesync'].core.services.appLifecycle.isReady()}))()",
+        env
+    );
+    assertEqual(readiness.ready, false, `The target became ready before applying ${path}.`);
+    await assertVaultPathStaysMissing(target, path, 750);
+}
+
+async function startContinuousReplication(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
+    const result = await evalObsidianJson<{ status: string }>(
+        cliBinary,
+        [
+            "(async()=>{",
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            "await core.services.setting.applyExternalSettings({liveSync:true},true);",
+            "await core.services.control.applySettings();",
+            "const result=await core.services.replication.startContinuous({trigger:'daemon',interaction:{kind:'forbidden'}});",
+            "return JSON.stringify(result);",
+            "})()",
+        ].join(""),
+        env
+    );
+    assertEqual(result.status, "completed", `Continuous replication did not start: ${JSON.stringify(result)}.`);
+
+    const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_REMOTE_ACTIVITY_TIMEOUT_MS ?? 30000);
+    let active = false;
+    while (!active && Date.now() < deadline) {
+        active = await evalObsidianJson<boolean>(
+            cliBinary,
+            "(()=>JSON.stringify(!!app.plugins.plugins['obsidian-livesync'].core.services.replicator.getActiveReplicator()))()",
+            env
+        );
+        if (!active) await new Promise((resolve) => setTimeout(resolve, 250));
+    }
+    if (!active) throw new Error("Timed out waiting for the target Replicator to become active.");
+}
+
+async function markReadyTwice(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
+    const result = await evalObsidianJson<{ ready: boolean }>(
+        cliBinary,
+        [
+            "(()=>{",
+            "const lifecycle=app.plugins.plugins['obsidian-livesync'].core.services.appLifecycle;",
+            "lifecycle.markIsReady();",
+            "lifecycle.markIsReady();",
+            "return JSON.stringify({ready:lifecycle.isReady()});",
+            "})()",
+        ].join(""),
+        env
+    );
+    assertEqual(result.ready, true, "Commonlib did not establish application readiness.");
+}
+
+async function removeObserver(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
+    await evalObsidianJson(
+        cliBinary,
+        [
+            "(()=>{",
+            `const key=${JSON.stringify(observerKey)};`,
+            "const state=globalThis[key];",
+            "if(state){state.unsubscribeReady?.();delete globalThis[key];}",
+            "return JSON.stringify(true);",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function assertApplied(
+    cliBinary: string,
+    env: NodeJS.ProcessEnv,
+    vault: TemporaryVault,
+    path: string,
+    expectedContent: string,
+    expectedReadyEvents: number
+): Promise<void> {
+    const beforeReflection = await readObservation(cliBinary, env, path);
+    assertEqual(
+        beforeReflection.readinessEvents,
+        expectedReadyEvents,
+        "The expected Commonlib readiness transition was not observed before reflection."
+    );
+    await waitForVaultContent(vault, path, expectedContent);
+    await new Promise((resolve) => setTimeout(resolve, 500));
+    const observation = await readObservation(cliBinary, env, path);
+    assertEqual(observation.content, expectedContent, `Obsidian Vault read-back was incorrect for ${path}.`);
+    assertEqual(
+        observation.readinessEvents,
+        expectedReadyEvents,
+        "markIsReady emitted an unexpected number of readiness transitions."
+    );
+}
+
+async function main(): Promise<void> {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`);
+
+    const couchDb = await loadCouchDbConfig();
+    const sourceDbName = makeUniqueDatabaseName(couchDb.dbPrefix, "received-change-readiness-source");
+    const targetDbName = makeUniqueDatabaseName(couchDb.dbPrefix, "received-change-readiness-target");
+    const couchDbSettings = {
+        uri: couchDb.uri,
+        username: couchDb.username,
+        password: couchDb.password,
+    };
+    const sourceCouchDbSettings = { ...couchDbSettings, dbName: sourceDbName };
+    const targetCouchDbSettings = { ...couchDbSettings, dbName: targetDbName };
+    const sourceVault = await createTemporaryVault("obsidian-livesync-readiness-source-");
+    const targetVault = await createTemporaryVault("obsidian-livesync-readiness-target-");
+    let source: ObsidianLiveSyncSession | undefined;
+    let target: ObsidianLiveSyncSession | undefined;
+    const publishedChunkIds = new Set<string>();
+
+    try {
+        await assertCouchDbReachable(couchDb);
+        await createCouchDbDatabase(couchDb, sourceDbName);
+        await createCouchDbDatabase(couchDb, targetDbName);
+        source = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: cli.binary,
+            vault: sourceVault,
+            startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
+            pluginData: createE2eCouchDbPluginData(sourceCouchDbSettings),
+            localStorageEntries: createE2eObsidianDeviceLocalState(sourceVault.name),
+        });
+        await waitForLiveSyncCoreReady(cli.binary, source.cliEnv);
+        await prepareRemote(cli.binary, source.cliEnv);
+
+        const readyNote = await createAndUploadNote(
+            cli.binary,
+            source.cliEnv,
+            couchDb,
+            sourceDbName,
+            readyPath,
+            readyContent
+        );
+        const suspendedNote = await createAndUploadNote(
+            cli.binary,
+            source.cliEnv,
+            couchDb,
+            sourceDbName,
+            suspendedPath,
+            suspendedContent
+        );
+        await source.app.stop();
+        source = undefined;
+
+        target = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: cli.binary,
+            vault: targetVault,
+            startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
+            pluginData: createE2eCouchDbPluginData(targetCouchDbSettings),
+            localStorageEntries: createE2eObsidianDeviceLocalState(targetVault.name),
+        });
+        await waitForLiveSyncCoreReady(cli.binary, target.cliEnv);
+        await prepareRemote(cli.binary, target.cliEnv);
+        await installObserver(cli.binary, target.cliEnv);
+        await startContinuousReplication(cli.binary, target.cliEnv);
+
+        await resetReadiness(cli.binary, target.cliEnv, false);
+        await injectCapturedNote(couchDb, targetDbName, readyNote, publishedChunkIds);
+        await waitForReceivedWhileUnready(cli.binary, target.cliEnv, targetVault, readyPath);
+        await markReadyTwice(cli.binary, target.cliEnv);
+        await assertApplied(cli.binary, target.cliEnv, targetVault, readyPath, readyContent, 1);
+
+        await resetReadiness(cli.binary, target.cliEnv, true);
+        await injectCapturedNote(couchDb, targetDbName, suspendedNote, publishedChunkIds);
+        await waitForReceivedWhileUnready(cli.binary, target.cliEnv, targetVault, suspendedPath);
+        await markReadyTwice(cli.binary, target.cliEnv);
+        await assertVaultPathStaysMissing(targetVault, suspendedPath, 1000);
+        const suspendedObservation = await readObservation(cli.binary, target.cliEnv, suspendedPath);
+        assertEqual(suspendedObservation.readinessEvents, 2, "The resumed-ready transition was not observed once.");
+        assertEqual(suspendedObservation.content, null, "Readiness bypassed explicit database-reflection suspension.");
+
+        await evalObsidianJson(
+            cli.binary,
+            "(async()=>{const setting=app.plugins.plugins['obsidian-livesync'].core.services.setting;await setting.applyPartial({suspendParseReplicationResult:false},true);return JSON.stringify(true);})()",
+            target.cliEnv
+        );
+        await assertApplied(cli.binary, target.cliEnv, targetVault, suspendedPath, suspendedContent, 2);
+
+        console.log(
+            "Received-change readiness: queued changes resumed once per readiness transition; explicit suspension held until settings resumed."
+        );
+    } finally {
+        if (target) {
+            await removeObserver(cli.binary, target.cliEnv).catch(() => undefined);
+            await target.app.stop();
+        }
+        if (source) await source.app.stop();
+        await sourceVault.dispose();
+        await targetVault.dispose();
+        if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
+            await Promise.all(
+                [sourceDbName, targetDbName].map((dbName) => deleteCouchDbDatabase(couchDb, dbName))
+            ).catch((error: unknown) => {
+                console.warn(error instanceof Error ? error.message : error);
+            });
+        }
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exit(1);
+});

test/e2e-obsidian/scripts/run-focused.ts

@@ -17,6 +17,7 @@ const focusedScenarios = new Set([
     "p2p-pane",
     "vault-reflection",
     "couchdb-upload",
+    "chunk-fetch-retry",
     "couchdb-manual-setup-workflow",
     "cli-to-obsidian-sync",
     "minio-upload",
@@ -34,6 +35,7 @@ const focusedScenarios = new Set([
     "security-seed-reconnect",
     "hidden-file-snippet-sync",
     "customisation-sync",
+    "received-change-readiness",
     "internal-metadata-doctor",
     "setting-markdown-export",
     "upgrade-from-stable",