Merge pull request #1308 from vrtmrz/fix/request-api-cache-control

Honour no-store reads through Obsidian's Internal API

src/modules/services/ObsidianAPIService.ts

@@ -16,6 +16,43 @@ declare module "obsidian" {
     }
 }
 
+const cacheReuseDirectives = new Set(["max-age", "max-stale", "min-fresh", "only-if-cached"]);
+
+function splitHeaderDirectives(value: string): string[] {
+    const directives: string[] = [];
+    let start = 0;
+    let inQuotes = false;
+    let escaped = false;
+
+    for (let index = 0; index < value.length; index++) {
+        const character = value[index];
+        if (inQuotes && escaped) {
+            escaped = false;
+            continue;
+        }
+        if (inQuotes && character === "\\") {
+            escaped = true;
+            continue;
+        }
+        if (character === '"') {
+            inQuotes = !inQuotes;
+        } else if (character === "," && !inQuotes) {
+            const directive = value.slice(start, index).trim();
+            if (directive) directives.push(directive);
+            start = index + 1;
+        }
+    }
+
+    const lastDirective = value.slice(start).trim();
+    if (lastDirective) directives.push(lastDirective);
+    return directives;
+}
+
+function getDirectiveName(directive: string): string {
+    const equalsIndex = directive.indexOf("=");
+    return (equalsIndex === -1 ? directive : directive.slice(0, equalsIndex)).trim().toLowerCase();
+}
+
 export class ObsidianAPIService extends InjectableAPIService<ObsidianServiceContext> {
     _customHandler: ObsHttpHandler | undefined;
     _confirmInstance: Confirm;
@@ -182,6 +219,30 @@ export class ObsidianAPIService extends InjectableAPIService<ObsidianServiceCont
         delete transformedHeaders["Host"];
         delete transformedHeaders["content-length"];
         delete transformedHeaders["Content-Length"];
+        const cacheMode = opts?.cache ?? (req instanceof Request ? req.cache : undefined);
+        if (cacheMode === "no-store" && ["GET", "HEAD"].includes(method.toUpperCase())) {
+            // requestUrl has no cache option; no-store alone can reuse an existing cached response.
+            const cacheControlKeys = Object.keys(transformedHeaders).filter(
+                (key) => key.toLowerCase() === "cache-control"
+            );
+            const cacheControlDirectives = cacheControlKeys
+                .flatMap((key) => splitHeaderDirectives(transformedHeaders[key]))
+                .filter((directive) => {
+                    const name = getDirectiveName(directive);
+                    return !cacheReuseDirectives.has(name) && name !== "no-cache" && name !== "no-store";
+                });
+            const cacheControlKey = cacheControlKeys[0] ?? "cache-control";
+            for (const key of cacheControlKeys) delete transformedHeaders[key];
+            transformedHeaders[cacheControlKey] = [...cacheControlDirectives, "no-cache", "no-store"].join(", ");
+
+            const pragmaKeys = Object.keys(transformedHeaders).filter((key) => key.toLowerCase() === "pragma");
+            const pragmaDirectives = pragmaKeys
+                .flatMap((key) => splitHeaderDirectives(transformedHeaders[key]))
+                .filter((directive) => getDirectiveName(directive) !== "no-cache");
+            const pragmaKey = pragmaKeys[0] ?? "pragma";
+            for (const key of pragmaKeys) delete transformedHeaders[key];
+            transformedHeaders[pragmaKey] = [...pragmaDirectives, "no-cache"].join(", ");
+        }
         const contentType =
             transformedHeaders["content-type"] ?? transformedHeaders["Content-Type"] ?? "application/json";
         const requestParam: RequestUrlParam = {

src/modules/services/ObsidianAPIService.unit.spec.ts

@@ -1,6 +1,14 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { RequestUrlParam } from "@/deps";
 
 const mocks = vi.hoisted(() => ({
+    requestUrl: vi.fn<
+        (request: RequestUrlParam) => Promise<{
+            status: number;
+            headers: Record<string, string>;
+            arrayBuffer: ArrayBuffer;
+        }>
+    >(),
     platform: {
         isMobile: false,
     },
@@ -8,12 +16,12 @@ const mocks = vi.hoisted(() => ({
 
 vi.mock("@/deps.ts", () => ({
     Platform: mocks.platform,
-    requestUrl: vi.fn(),
+    requestUrl: mocks.requestUrl,
 }));
 
 vi.mock("@/deps", () => ({
     Platform: mocks.platform,
-    requestUrl: vi.fn(),
+    requestUrl: mocks.requestUrl,
 }));
 
 vi.mock("@/modules/essentialObsidian/APILib/ObsHttpHandler", () => ({
@@ -36,6 +44,8 @@ function createService(workspace: Record<string, unknown>, isMobile = false): Ob
 beforeEach(() => {
     mocks.platform.isMobile = false;
     vi.clearAllMocks();
+    mocks.requestUrl.mockReset();
+    mocks.requestUrl.mockResolvedValue({ status: 200, headers: {}, arrayBuffer: new ArrayBuffer(0) });
 });
 
 describe("ObsidianAPIService.showWindowOnRight", () => {
@@ -65,3 +75,115 @@ describe("ObsidianAPIService.showWindowOnRight", () => {
         expect(workspace.revealLeaf).toHaveBeenCalledWith(rightLeaf);
     });
 });
+
+describe("ObsidianAPIService.nativeFetch cache control", () => {
+    const url = "https://couchdb.example.test/vault/f%3Ametadata";
+
+    function sentHeaders(): Headers {
+        expect(mocks.requestUrl).toHaveBeenCalledOnce();
+        return new Headers(mocks.requestUrl.mock.calls[0][0].headers);
+    }
+
+    it.each(["GET", "HEAD", "get", "head"])("bypasses cached responses for an explicit no-store %s", async (method) => {
+        const headers = {
+            Authorization: "Basic fixture",
+            "Cache-Control": "public, max-age=3600",
+            "cAcHe-CoNtRoL": "max-stale",
+            Pragma: "cache",
+        };
+
+        await createService({}).nativeFetch(url, { method, cache: "no-store", headers });
+
+        expect(sentHeaders().get("cache-control")).toBe("public, no-cache, no-store");
+        expect(sentHeaders().get("pragma")).toBe("cache, no-cache");
+        expect(sentHeaders().get("authorization")).toBe("Basic fixture");
+        expect(headers["Cache-Control"]).toBe("public, max-age=3600");
+        expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method });
+    });
+
+    it("preserves non-conflicting cache directives when enforcing no-store", async () => {
+        const headers = {
+            "Cache-Control": 'max-age=3600, no-transform, custom="a,b"',
+            "cAcHe-CoNtRoL": 'max-stale=30, min-fresh=5, only-if-cached, custom-two="max-age=1, only-if-cached"',
+            Pragma: 'legacy="a,b", no-cache',
+            "X-Custom-Header": "preserved",
+        };
+        const originalHeaders = { ...headers };
+
+        await createService({}).nativeFetch(url, { cache: "no-store", headers });
+
+        const sent = mocks.requestUrl.mock.calls[0][0].headers ?? {};
+        expect(sent["Cache-Control"]).toBe(
+            'no-transform, custom="a,b", custom-two="max-age=1, only-if-cached", no-cache, no-store'
+        );
+        expect(sent["cAcHe-CoNtRoL"]).toBeUndefined();
+        expect(sent.Pragma).toBe('legacy="a,b", no-cache');
+        expect(sent["X-Custom-Header"]).toBe("preserved");
+        expect(headers).toEqual(originalHeaders);
+    });
+
+    it("uses the Request cache mode when the options do not override it", async () => {
+        const request = new Request(url, {
+            method: "HEAD",
+            cache: "no-store",
+            headers: { "X-Fixture": "request" },
+        });
+        const headers = new Headers({ "X-Other": "options" });
+
+        await createService({}).nativeFetch(request, { headers });
+
+        expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store");
+        expect(sentHeaders().get("pragma")).toBe("no-cache");
+        expect(sentHeaders().get("x-fixture")).toBe("request");
+        expect(sentHeaders().get("x-other")).toBe("options");
+        expect(headers.has("cache-control")).toBe(false);
+        expect(request.headers.has("cache-control")).toBe(false);
+    });
+
+    it("lets the options request no-store for an otherwise cacheable Request", async () => {
+        await createService({}).nativeFetch(new Request(url, { cache: "force-cache" }), { cache: "no-store" });
+
+        expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store");
+    });
+
+    it("lets an explicit default cache mode override a no-store Request", async () => {
+        await createService({}).nativeFetch(new Request(url, { cache: "no-store" }), { cache: "default" });
+
+        expect(sentHeaders().has("cache-control")).toBe(false);
+        expect(sentHeaders().has("pragma")).toBe(false);
+    });
+
+    it.each([undefined, "default", "no-cache", "reload", "force-cache"] as const)(
+        "preserves caller headers for cache mode %s",
+        async (cache) => {
+            const headers = { "Cache-Control": "max-age=60", Pragma: "fixture" };
+
+            await createService({}).nativeFetch(url, { cache, headers });
+
+            expect(mocks.requestUrl.mock.calls[0][0].headers).toEqual(headers);
+        }
+    );
+
+    it("does not add cache directives to an ordinary request", async () => {
+        await createService({}).nativeFetch(url);
+
+        expect(sentHeaders().has("cache-control")).toBe(false);
+        expect(sentHeaders().has("pragma")).toBe(false);
+    });
+
+    it.each(["POST", "PUT", "DELETE"])("preserves a no-store %s request", async (method) => {
+        const headers = { "Cache-Control": "no-transform" };
+
+        await createService({}).nativeFetch(url, { method, cache: "no-store", headers });
+
+        expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method, headers });
+        expect(sentHeaders().has("pragma")).toBe(false);
+    });
+
+    it("uses the effective method from the options", async () => {
+        await createService({}).nativeFetch(new Request(url, { cache: "no-store" }), { method: "POST" });
+
+        expect(sentHeaders().has("cache-control")).toBe(false);
+        expect(sentHeaders().has("pragma")).toBe(false);
+    });
+});

test/e2e-obsidian/README.md

@@ -174,6 +174,8 @@ The same workflow checks the two remote-activity status boundaries. It first hol
 
 `test:e2e:obsidian:couchdb-http-cache` checks replication after rebuilding an encrypted CouchDB remote while an earlier HTTP response remains cached. It enables internal Metadata encryption with Hidden File Sync and Customisation Sync, verifies that the Rebuild replaces the Security Seed and ciphertext while retaining the document revision, and requires the replication pre-check and subsequent sends to succeed without clearing the browser cache. It uses the CouchDB fixture variables, an isolated Vault and profile, and a unique remote database. The raw command uses the current built plug-in by default; `E2E_OBSIDIAN_ARTIFACT_ROOT` can select an exact earlier build for regression comparison.
 
+Run `npm run test:e2e:obsidian:focused -- couchdb-http-cache --request-api` to warm Obsidian's Internal API cache and perform the same workflow with **Use Internal API** enabled. Both modes require an explicit `no-store` read to return the rebuilt ciphertext. The default mode also retains the stale browser-cache assertion; the Internal API mode permits reads during Rebuild to refresh that cache.
+
 `npm run test:e2e:obsidian:focused -- chunk-fetch-retry` checks delayed Chunk availability through a real CouchDB service and Obsidian. It creates a Metadata-only remote fixture, starts ordinary one-shot replication with `readChunksOnline`, and inserts the missing Chunk only after a real fetch has returned an empty result. A pass-through observer records the replicator's call times and results without substituting responses or adding waits. The fixture sets the existing minimum request interval to 500 ms to keep the real retry status observable even if finite completion expedites the final probe. The actual status bar must show zero initial requests (`🛄`) and one retry (`🔁`), and both counts must return to zero after delivery ends. On-demand replication excludes Chunk documents from the ordinary pull, so the delayed Chunk must arrive through the observed fetch and produce the exact Vault content.
 
 If finite replication was already inactive when the initial lookup began, the scenario requires a retry at least two seconds later and no physical request slot occupied during backoff. If finite replication ends during the initial lookup or the following backoff, the retry must instead be a post-completion final probe before the two-second delay would expire. This distinction is determined from the observed finite-count transitions, not an assumed ordering between replication and HTTP completion.

test/e2e-obsidian/scripts/couchdb-http-cache.ts

@@ -28,6 +28,8 @@ import { createTemporaryVault } from "../runner/vault.ts";
 
 process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000";
 
+const useRequestAPI = process.argv.includes("--request-api");
+
 const customPath = ".obsidian/snippets/http-cache.css";
 const nextCustomPath = ".obsidian/snippets/http-cache-new.css";
 const settings = {
@@ -45,7 +47,7 @@ const settings = {
     syncInternalFilesInterval: 0,
     syncInternalFilesTargetPatterns: "^\\.http-cache(?:/|$)",
     watchInternalFileChanges: false,
-    useRequestAPI: false,
+    useRequestAPI,
     autoAcceptCompatibleTweak: false,
     doctorProcessedVersion: DoctorRegulation.version,
     deviceAndVaultName: "http-cache-source",
@@ -74,10 +76,11 @@ async function main(): Promise<void> {
                 .storeCustomizationFiles(path,core.services.setting.getDeviceAndVaultName());
             return JSON.stringify(true);`);
     };
-    const browserRead = async (id: string, cache: RequestCache): Promise<CouchDbDocument> => {
+    const readCachedDocument = async (id: string, cache: RequestCache): Promise<CouchDbDocument> => {
         const url = `${couchDb.uri}/${encodeURIComponent(dbName)}/${encodeURIComponent(id)}`;
         const authorization = `Basic ${Buffer.from(`${couchDb.username}:${couchDb.password}`).toString("base64")}`;
-        return await evaluate(`const response=await fetch(${JSON.stringify(url)},{
+        const fetchFunction = useRequestAPI ? "core.services.API.nativeFetch.bind(core.services.API)" : "fetch";
+        return await evaluate(`const response=await ${fetchFunction}(${JSON.stringify(url)},{
             headers:{authorization:${JSON.stringify(authorization)}},credentials:'include',cache:${JSON.stringify(cache)}});
             if(!response.ok)throw new Error('Fixture read failed: '+response.status);
             return JSON.stringify(await response.json());`);
@@ -110,7 +113,7 @@ async function main(): Promise<void> {
         assert.ok(original?.path, "The encrypted ordinary Metadata fixture is missing.");
         const initialParameters = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS);
         // Populate the HTTP cache as an earlier plug-in version would have done.
-        assert.deepEqual(await browserRead(original._id, "reload"), original);
+        assert.deepEqual(await readCachedDocument(original._id, "reload"), original);
 
         await evaluate(`await core.services.setting.applyPartial({encryptInternalMetadata:true},true);
             await core.services.control.applySettings();
@@ -128,11 +131,13 @@ async function main(): Promise<void> {
         assert.notEqual(rebuiltParameters.pbkdf2salt, initialParameters.pbkdf2salt, "The Seed did not change.");
         assert.equal(rebuilt._rev, original._rev, "The document revision was not retained.");
         assert.notEqual(rebuilt.path, original.path, "The remote ciphertext did not change.");
-        assert.equal(
-            (await browserRead(original._id, "default")).path,
-            original.path,
-            "The stale browser HTTP cache fixture is no longer present."
-        );
+        if (!useRequestAPI) {
+            assert.equal(
+                (await readCachedDocument(original._id, "default")).path,
+                original.path,
+                "The stale browser HTTP cache fixture is no longer present."
+            );
+        }
 
         await evaluate(`await core.services.setting.applyPartial({usePluginSync:true,syncInternalFiles:true},true);
             await core.services.control.applySettings();return JSON.stringify(true);`);
@@ -143,9 +148,12 @@ async function main(): Promise<void> {
             if(opened)await opened.close();return JSON.stringify({admitted:!!opened});
             }catch(error){return JSON.stringify({admitted:false,error:error.name});}`);
         assert.ok(admission.admitted, `Replication admission failed: ${JSON.stringify(admission)}`);
+        assert.equal((await readCachedDocument(original._id, "no-store")).path, rebuilt.path);
         await pushLocalChanges(cliBinary, session.cliEnv);
         await pushLocalChanges(cliBinary, session.cliEnv);
-        console.log("Rebuild and subsequent replication succeeded with stale ciphertext still in the HTTP cache.");
+        console.log(
+            `Rebuild and subsequent replication succeeded after warming the ${useRequestAPI ? "Internal API" : "browser"} HTTP cache.`
+        );
     } finally {
         await session?.app.stop();
         await vault.dispose();

updates.md

@@ -12,6 +12,9 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
 
 ## Unreleased
 
+- We can continue CouchDB synchronisation after rebuilding an encrypted remote when **Use Internal API** is enabled. (#1234)
+    - CouchDB read requests now carry cache-control headers to avoid reusing responses encrypted with the previous Security Seed.
+
 - CLI `fetch` and `rebuild` retain their selected local database for subsequent commands even without `--write-settings`, preventing the empty-database restart and file deletion reported in #1304.
     - Runtime state is separated by database and Vault. Interrupted reconstruction blocks ordinary synchronisation until recovery, and a pending settings commit can be completed on restart.
     - Older runtime state with no recorded owner is preserved and requires explicit reconstruction from a verified source before ordinary writes resume. Back up unique edits before choosing Fetch or Rebuild.
@@ -22,6 +25,7 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
 
 - Files waiting for encrypted Chunks which arrive after their Metadata receive the decrypted contents once the local database is ready.
     - Commonlib 0.2.3 restarts the database change feed after initialisation installs the decryption transforms. This corrects delivery of later Chunks; it does not replay earlier events. ([Commonlib #191](https://github.com/vrtmrz/livesync-commonlib/pull/191))
+
 - The CLI mirror now starts correctly on Windows when the Vault path is omitted and the database directory is also used as the Vault. It checks logical Vault paths using `/` separators, correctly excluding internal database files and nested hidden directories even when the host uses Windows path separators.
 
 ## 1.0.43