feat: distinguish remote operation and request activity

2026-07-16 17:46:07 +00:00 · 2026-07-16 04:09:53 +00:00
parent 9ffde2dd3e
commit 7439b75cd0
15 changed files with 579 additions and 91 deletions
@@ -66,7 +66,7 @@ npm run test:e2e:obsidian:local-suite:services

 `test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, configures Self-hosted LiveSync through `obsidian-cli eval`, creates a note in real Obsidian, commits the note into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the metadata document and its chunk documents.

-The same workflow checks the remote-activity status boundary. It holds the real one-shot replication immediately before its replicator call, confirms that the status bar contains `📲` while finite replication is active, releases it, and then requires the finite and bounded activity counts to return to zero, the HTTP request and response counts to balance, and `📲` to disappear. It then creates a remote-only chunk, holds the real on-demand fetch immediately before its remote call, makes the same active and idle assertions, and verifies that the fetched chunk is written into the local database. These gates make the active state deterministic without replacing either remote operation.
+The same workflow checks the two remote-activity status boundaries. It first holds a real CouchDB request at the selected fetch implementation and confirms that `🌐N` is visible while `📲` is absent. It then holds the real one-shot replication immediately before its replicator call, confirms that `📲` is visible while no physical request is active, releases it, and requires the finite and bounded activity counts to return to zero, the request and response counts to balance, and both indicators to disappear. Finally, it creates a remote-only chunk, holds the real on-demand fetch immediately before its remote call, makes the same logical active and idle assertions, and verifies that the fetched chunk is written into the local database. These gates make the active states deterministic without replacing the remote request or operation.

 If this status workflow fails while Obsidian is running, it writes a full-page screenshot and a JSON snapshot of the status text and counters under `/tmp/obsidian-livesync-e2e`. Set `E2E_OBSIDIAN_DIAGNOSTICS_DIR` to use another directory.

@@ -87,7 +87,7 @@ LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) --
  npm run test:e2e:obsidian:cli-to-obsidian-sync
 ```

-`test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service, configures Self-hosted LiveSync for Object Storage through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix.
+`test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service, configures Self-hosted LiveSync for Object Storage through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. Adapter tests separately observe an in-progress SDK command, while this real-runtime workflow verifies the resulting request counters advance and rebalance.

 `test:e2e:obsidian:startup-scan` configures a temporary CouchDB database, stops Obsidian, writes a note directly into the vault, restarts Obsidian, and verifies from CouchDB that the boot-time scan picked up the offline file.

@@ -2,8 +2,24 @@ import { mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import type { Page } from "playwright";
 import { withObsidianPage } from "./ui.ts";
+import {
+    REMOTE_OPERATION_ACTIVITY_ICON,
+    REMOTE_REQUEST_ACTIVITY_ICON,
+} from "../../../src/modules/features/RemoteActivityStatus.ts";

 export const REMOTE_ACTIVITY_E2E_STATE_KEY = "__livesyncE2ERemoteActivity";
+export const REMOTE_ACTIVITY_GATE_KIND = {
+    chunkFetch: "chunk-fetch",
+    oneShot: "one-shot",
+    trackedRequest: "tracked-request",
+} as const;
+export const REMOTE_ACTIVITY_EXPECTED_STATE = {
+    chunkFetchActive: "chunk-fetch-active",
+    finiteReplicationActive: "finite-replication-active",
+    idle: "idle",
+    trackedRequestActive: "tracked-request-active",
+} as const;
+export type RemoteActivityGateKind = (typeof REMOTE_ACTIVITY_GATE_KIND)[keyof typeof REMOTE_ACTIVITY_GATE_KIND];

 export type RemoteActivitySnapshot = {
    boundedRemoteActivityCount: number;
@@ -11,14 +27,17 @@ export type RemoteActivitySnapshot = {
    gateDone?: boolean;
    gateEntered?: boolean;
    gateError?: string;
-    gateKind?: string;
+    gateKind?: RemoteActivityGateKind;
    requestCount: number;
+    remoteOperationIndicatorVisible: boolean;
+    remoteRequestIndicatorVisible: boolean;
    responseCount: number;
    statusBarFound: boolean;
    statusBarText: string;
 };

-export type ExpectedRemoteActivityState = "chunk-fetch-active" | "finite-replication-active" | "idle";
+export type ExpectedRemoteActivityState =
+    (typeof REMOTE_ACTIVITY_EXPECTED_STATE)[keyof typeof REMOTE_ACTIVITY_EXPECTED_STATE];

 type RuntimeCounter = { value?: number };

@@ -39,7 +58,7 @@ type RuntimeGate = {
    done?: boolean;
    entered?: boolean;
    error?: string;
-    kind?: string;
+    kind?: RemoteActivityGateKind;
 };

 type RendererGlobals = typeof globalThis & {
@@ -52,7 +71,7 @@ type RendererGlobals = typeof globalThis & {

 async function readRemoteActivitySnapshotFromPage(page: Page): Promise<RemoteActivitySnapshot> {
    return await page.evaluate(
-        ({ pluginId, stateKey }) => {
+        ({ operationIcon, pluginId, requestIcon, stateKey }) => {
            const globals = globalThis as RendererGlobals;
            const core = globals.app?.plugins?.plugins?.[pluginId]?.core;
            if (!core) throw new Error(`Obsidian plug-in is not loaded: ${pluginId}`);
@@ -68,12 +87,23 @@ async function readRemoteActivitySnapshotFromPage(page: Page): Promise<RemoteAct
                gateError: gate?.error,
                gateKind: gate?.kind,
                requestCount: Number(core.services?.API?.requestCount?.value ?? -1),
+                remoteOperationIndicatorVisible: statusBars.some((element) =>
+                    (element.textContent ?? "").includes(operationIcon)
+                ),
+                remoteRequestIndicatorVisible: statusBars.some((element) =>
+                    (element.textContent ?? "").includes(requestIcon)
+                ),
                responseCount: Number(core.services?.API?.responseCount?.value ?? -1),
                statusBarFound: statusBars.length > 0,
                statusBarText: statusBars.map((element) => element.textContent ?? "").join("\n"),
            } satisfies RemoteActivitySnapshot;
        },
-        { pluginId: "obsidian-livesync", stateKey: REMOTE_ACTIVITY_E2E_STATE_KEY }
+        {
+            operationIcon: REMOTE_OPERATION_ACTIVITY_ICON,
+            pluginId: "obsidian-livesync",
+            requestIcon: REMOTE_REQUEST_ACTIVITY_ICON,
+            stateKey: REMOTE_ACTIVITY_E2E_STATE_KEY,
+        }
    );
 }

@@ -105,7 +135,7 @@ export async function waitForRemoteActivityState(
    try {
        return await withObsidianPage(port, async (page) => {
            await page.waitForFunction(
-                ({ expectedState, pluginId, stateKey }) => {
+                ({ expectedState, expectedStates, gateKinds, operationIcon, pluginId, requestIcon, stateKey }) => {
                    const globals = globalThis as RendererGlobals;
                    const core = globals.app?.plugins?.plugins?.[pluginId]?.core;
                    if (!core) return false;
@@ -117,31 +147,58 @@ export async function waitForRemoteActivityState(
                    const finite = Number(core.services?.replicator?.finiteReplicationActivityCount?.value ?? -1);
                    const requests = Number(core.services?.API?.requestCount?.value ?? -1);
                    const responses = Number(core.services?.API?.responseCount?.value ?? -1);
-                    const iconVisible = statusBarText.includes("📲");
+                    const operationIconVisible = statusBarText.includes(operationIcon);
+                    const requestIconVisible = statusBarText.includes(requestIcon);
+                    const trackedRequests = Math.max(0, requests - responses);

-                    if (expectedState === "finite-replication-active") {
+                    if (expectedState === expectedStates.finiteReplicationActive) {
                        return (
-                            gate?.kind === "one-shot" &&
+                            gate?.kind === gateKinds.oneShot &&
                            gate.entered === true &&
                            bounded > 0 &&
                            finite > 0 &&
-                            iconVisible
+                            operationIconVisible &&
+                            !requestIconVisible &&
+                            trackedRequests === 0
                        );
                    }
-                    if (expectedState === "chunk-fetch-active") {
+                    if (expectedState === expectedStates.chunkFetchActive) {
                        return (
-                            gate?.kind === "chunk-fetch" &&
+                            gate?.kind === gateKinds.chunkFetch &&
                            gate.entered === true &&
                            bounded > 0 &&
                            finite === 0 &&
-                            iconVisible
+                            operationIconVisible &&
+                            !requestIconVisible &&
+                            trackedRequests === 0
                        );
                    }
-                    return bounded === 0 && finite === 0 && requests === responses && !iconVisible;
+                    if (expectedState === expectedStates.trackedRequestActive) {
+                        return (
+                            gate?.kind === gateKinds.trackedRequest &&
+                            gate.entered === true &&
+                            bounded === 0 &&
+                            finite === 0 &&
+                            trackedRequests > 0 &&
+                            !operationIconVisible &&
+                            statusBarText.includes(`${requestIcon}${trackedRequests}`)
+                        );
+                    }
+                    return (
+                        bounded === 0 &&
+                        finite === 0 &&
+                        requests === responses &&
+                        !operationIconVisible &&
+                        !requestIconVisible
+                    );
                },
                {
                    expectedState: expected,
+                    expectedStates: REMOTE_ACTIVITY_EXPECTED_STATE,
+                    gateKinds: REMOTE_ACTIVITY_GATE_KIND,
+                    operationIcon: REMOTE_OPERATION_ACTIVITY_ICON,
                    pluginId: "obsidian-livesync",
+                    requestIcon: REMOTE_REQUEST_ACTIVITY_ICON,
                    stateKey: REMOTE_ACTIVITY_E2E_STATE_KEY,
                },
                { timeout: timeoutMs }
@@ -1,11 +1,15 @@
 import { evalObsidianJson } from "./cli.ts";
-import { REMOTE_ACTIVITY_E2E_STATE_KEY } from "./remoteActivity.ts";
+import {
+    REMOTE_ACTIVITY_E2E_STATE_KEY,
+    REMOTE_ACTIVITY_GATE_KIND,
+    type RemoteActivityGateKind,
+} from "./remoteActivity.ts";

 export type HeldRemoteActivityResult = {
    done: boolean;
    entered: boolean;
    error?: string;
-    kind: "chunk-fetch" | "one-shot";
+    kind: RemoteActivityGateKind;
    requestedIds?: string[];
    result?: boolean;
    resultCount?: number;
@@ -27,7 +31,7 @@ export async function startHeldOneShotReplication(cliBinary: string, env: NodeJS
            "const original=replicator.openReplication;",
            "let releaseGate;",
            "const gate=new Promise((resolve)=>{releaseGate=resolve;});",
-            "const state={kind:'one-shot',entered:false,done:false,released:false,error:undefined,result:undefined,promise:undefined,release:undefined,restore:undefined};",
+            `const state={kind:${JSON.stringify(REMOTE_ACTIVITY_GATE_KIND.oneShot)},entered:false,done:false,released:false,error:undefined,result:undefined,promise:undefined,release:undefined,restore:undefined};`,
            "state.release=()=>{if(!state.released){state.released=true;releaseGate();}};",
            "state.restore=()=>{replicator.openReplication=original;};",
            "host[stateKey]=state;",
@@ -74,7 +78,7 @@ export async function startHeldChunkFetch(cliBinary: string, env: NodeJS.Process
            "let resolveDone;",
            "const gate=new Promise((resolve)=>{releaseGate=resolve;});",
            "const donePromise=new Promise((resolve)=>{resolveDone=resolve;});",
-            "const state={kind:'chunk-fetch',entered:false,done:false,released:false,error:undefined,resultCount:undefined,requestedIds:undefined,promise:donePromise,release:undefined,restore:undefined};",
+            `const state={kind:${JSON.stringify(REMOTE_ACTIVITY_GATE_KIND.chunkFetch)},entered:false,done:false,released:false,error:undefined,resultCount:undefined,requestedIds:undefined,promise:donePromise,release:undefined,restore:undefined};`,
            "state.release=()=>{if(!state.released){state.released=true;releaseGate();}};",
            "state.restore=()=>{replicator.fetchRemoteChunks=original;};",
            "host[stateKey]=state;",
@@ -103,6 +107,51 @@ export async function startHeldChunkFetch(cliBinary: string, env: NodeJS.Process
    );
 }

+export async function startHeldTrackedRequest(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
+    await evalObsidianJson<{ started: boolean }>(
+        cliBinary,
+        [
+            "(async()=>{",
+            `const stateKey=${stateKeySource};`,
+            "const host=globalThis;",
+            "if(host[stateKey]) throw new Error('A remote activity E2E gate is already installed.');",
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            "const remote=core.services.remote;",
+            "const api=core.services.API;",
+            "const settings=core.services.setting.currentSettings();",
+            "const original=api.webCompatFetch;",
+            "let releaseGate;",
+            "const gate=new Promise((resolve)=>{releaseGate=resolve;});",
+            `const state={kind:${JSON.stringify(REMOTE_ACTIVITY_GATE_KIND.trackedRequest)},entered:false,done:false,released:false,error:undefined,result:undefined,promise:undefined,release:undefined,restore:undefined};`,
+            "state.release=()=>{if(!state.released){state.released=true;releaseGate();}};",
+            "state.restore=()=>{api.webCompatFetch=original;};",
+            "host[stateKey]=state;",
+            "api.webCompatFetch=async function(...args){",
+            "state.entered=true;",
+            "await gate;",
+            "return await original.apply(this,args);",
+            "};",
+            "state.promise=(async()=>{",
+            "try{",
+            "const base=String(settings.couchDB_URI).replace(/\\/$/,'');",
+            "const database=encodeURIComponent(settings.couchDB_DBNAME);",
+            "const credentials=btoa(`${settings.couchDB_USER}:${settings.couchDB_PASSWORD}`);",
+            "const response=await remote.performFetch(`${base}/${database}/_all_docs?limit=0`,{headers:{Authorization:`Basic ${credentials}`}});",
+            "state.result=response.ok;",
+            "}catch(error){",
+            "state.error=error instanceof Error?error.message:String(error);",
+            "}finally{",
+            "state.restore();",
+            "state.done=true;",
+            "}",
+            "})();",
+            "return JSON.stringify({started:true});",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
 export async function finishHeldRemoteActivity(
    cliBinary: string,
    env: NodeJS.ProcessEnv
@@ -16,13 +16,18 @@ import {
    waitForLiveSyncCoreReady,
    type LocalDatabaseEntry,
 } from "../runner/liveSyncWorkflow.ts";
-import { captureRemoteActivityDiagnostics, waitForRemoteActivityState } from "../runner/remoteActivity.ts";
+import {
+    REMOTE_ACTIVITY_EXPECTED_STATE,
+    captureRemoteActivityDiagnostics,
+    waitForRemoteActivityState,
+} from "../runner/remoteActivity.ts";
 import {
    cleanUpHeldRemoteActivity,
    clearHeldRemoteActivity,
    finishHeldRemoteActivity,
    startHeldChunkFetch,
    startHeldOneShotReplication,
+    startHeldTrackedRequest,
    waitForRestoredChunk,
 } from "../runner/remoteActivityWorkflow.ts";
 import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
@@ -115,21 +120,46 @@ async function main(): Promise<void> {

        await prepareRemote(cli.binary, session.cliEnv);
        activityStage = "initial-idle";
-        const initialIdle = await waitForRemoteActivityState(session.remoteDebuggingPort, "idle");
+        const initialIdle = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.idle
+        );
        const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv);

+        activityStage = REMOTE_ACTIVITY_EXPECTED_STATE.trackedRequestActive;
+        await startHeldTrackedRequest(cli.binary, session.cliEnv);
+        const trackedRequestActive = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.trackedRequestActive
+        );
+        const trackedRequestResult = await finishHeldRemoteActivity(cli.binary, session.cliEnv);
+        assertEqual(trackedRequestResult.error, undefined, "The observed CouchDB request failed.");
+        assertEqual(trackedRequestResult.result, true, "The observed CouchDB request did not report success.");
+        activityStage = "tracked-request-idle";
+        const trackedRequestIdle = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.idle
+        );
+        if (trackedRequestIdle.requestCount <= initialIdle.requestCount) {
+            throw new Error("The held CouchDB request did not advance the tracked remote-request count.");
+        }
+        await clearHeldRemoteActivity(cli.binary, session.cliEnv);
+
        activityStage = "one-shot-active";
        await startHeldOneShotReplication(cli.binary, session.cliEnv);
        const oneShotActive = await waitForRemoteActivityState(
            session.remoteDebuggingPort,
-            "finite-replication-active"
+            REMOTE_ACTIVITY_EXPECTED_STATE.finiteReplicationActive
        );
        const oneShotResult = await finishHeldRemoteActivity(cli.binary, session.cliEnv);
        assertEqual(oneShotResult.error, undefined, "One-shot replication failed while its activity was observed.");
        assertEqual(oneShotResult.result, true, "One-shot replication did not report success.");
        activityStage = "one-shot-idle";
-        const oneShotIdle = await waitForRemoteActivityState(session.remoteDebuggingPort, "idle");
-        if (oneShotIdle.requestCount <= initialIdle.requestCount) {
+        const oneShotIdle = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.idle
+        );
+        if (oneShotIdle.requestCount <= trackedRequestIdle.requestCount) {
            throw new Error("One-shot replication did not make an observed remote request.");
        }
        await clearHeldRemoteActivity(cli.binary, session.cliEnv);
@@ -154,9 +184,12 @@ async function main(): Promise<void> {
        const { _rev: _sourceRevision, ...remoteOnlyChunk } = sourceChunk;
        const chunkId = `h:e2e-remote-activity-${Date.now().toString(36)}`;
        await putCouchDbDocument(couchDb, dbName, { ...remoteOnlyChunk, _id: chunkId });
-        activityStage = "chunk-fetch-active";
+        activityStage = REMOTE_ACTIVITY_EXPECTED_STATE.chunkFetchActive;
        await startHeldChunkFetch(cli.binary, session.cliEnv, chunkId);
-        const chunkFetchActive = await waitForRemoteActivityState(session.remoteDebuggingPort, "chunk-fetch-active");
+        const chunkFetchActive = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.chunkFetchActive
+        );
        const chunkFetchResult = await finishHeldRemoteActivity(cli.binary, session.cliEnv);
        assertEqual(
            chunkFetchResult.error,
@@ -172,7 +205,10 @@ async function main(): Promise<void> {
        const restoredChunk = await waitForRestoredChunk(cli.binary, session.cliEnv, chunkId);
        assertEqual(restoredChunk.id, chunkId, "The restored chunk ID did not match the requested chunk.");
        activityStage = "chunk-fetch-idle";
-        const chunkFetchIdle = await waitForRemoteActivityState(session.remoteDebuggingPort, "idle");
+        const chunkFetchIdle = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.idle
+        );
        if (chunkFetchIdle.requestCount <= oneShotIdle.requestCount) {
            throw new Error("On-demand chunk fetching did not make an observed remote request.");
        }
@@ -183,6 +219,7 @@ async function main(): Promise<void> {
        );
        console.log(
            [
+                `Tracked request: ${trackedRequestActive.statusBarText.trim()} -> idle`,
                `One-shot activity: ${oneShotActive.statusBarText.trim()} -> idle`,
                `Chunk-fetch activity: ${chunkFetchActive.statusBarText.trim()} -> idle`,
                `Balanced remote requests: ${chunkFetchIdle.requestCount}/${chunkFetchIdle.responseCount}`,
@@ -17,6 +17,7 @@ import {
 } from "../runner/objectStorage.ts";
 import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
 import { createTemporaryVault } from "../runner/vault.ts";
+import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../runner/remoteActivity.ts";

 process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";

@@ -115,13 +116,29 @@ async function main(): Promise<void> {
        assertEqual(configured.liveSync, false, "LiveSync should remain disabled during this one-shot workflow.");

        await prepareRemote(cli.binary, session.cliEnv);
+        const activityBeforeUpload = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.idle
+        );
        const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv);
        await pushLocalChanges(cli.binary, session.cliEnv);
+        const activityAfterUpload = await waitForRemoteActivityState(
+            session.remoteDebuggingPort,
+            REMOTE_ACTIVITY_EXPECTED_STATE.idle
+        );
+        if (activityAfterUpload.requestCount <= activityBeforeUpload.requestCount) {
+            throw new Error("Object Storage synchronisation did not advance the tracked remote-request count.");
+        }
+        assertEqual(
+            activityAfterUpload.responseCount,
+            activityAfterUpload.requestCount,
+            "Object Storage remote-request counters did not rebalance after synchronisation."
+        );

        const keys = await waitForObjectStorageObjects(bucketPrefix);

        console.log(
-            `Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s))`
+            `Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}`
        );
    } finally {
        if (session) {