From 465def9d140d5342236d3ed6dc15cdf7e80ae899 Mon Sep 17 00:00:00 2001 From: Chihiro WATANABE Date: Tue, 3 Nov 2026 09:41:00 +0900 Subject: [PATCH 1/8] fix(i18n): correct inverted description for 'Do not split chunks in the background' setting --- docs/settings.md | 2 +- src/common/messages/combinedMessages.prod.ts | 2 +- src/common/messagesJson/en.json | 2 +- src/common/messagesYAML/en.yaml | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index e197d669..a97ffd3f 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -1224,7 +1224,7 @@ Enable this setting to process files with size mismatches, which can sometimes b #### Do not split chunks in the background Setting key: disableWorkerForGeneratingChunks -If disabled(toggled), chunks will be split on the UI thread (Previous behaviour). +If enabled, chunks will be split on the UI thread (previous behaviour). #### Process small files in the foreground diff --git a/src/common/messages/combinedMessages.prod.ts b/src/common/messages/combinedMessages.prod.ts index b808f305..c67eead0 100644 --- a/src/common/messages/combinedMessages.prod.ts +++ b/src/common/messages/combinedMessages.prod.ts @@ -3167,7 +3167,7 @@ export const allMessages: Readonly Date: Tue, 3 Nov 2026 01:04:36 +0000 Subject: [PATCH 2/8] Add CouchDB cache compatibility controls and native revalidation --- docs/settings.md | 8 + src/common/replicatorConfigurationIdentity.ts | 1 + ...plicatorConfigurationIdentity.unit.spec.ts | 12 ++ .../SettingDialogue/PanePowerUsers.ts | 1 + src/modules/services/ObsidianAPIService.ts | 15 +- .../services/ObsidianAPIService.unit.spec.ts | 63 +++++- test/e2e-obsidian/README.md | 4 +- .../scripts/couchdb-http-cache.ts | 184 +++++++++++++++++- updates.md | 6 + 9 files changed, 283 insertions(+), 11 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index e197d669..e28858b1 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -1096,6 +1096,14 @@ the separate P2P or Object Storage transfer units. Setting key: useTimeouts If this option is enabled, PouchDB will hold the connection open for 60 seconds, and if no change arrives in that time, close and reopen the socket, instead of holding it open indefinitely. Useful when a proxy limits request duration but can increase resource usage. +#### Use headers only for HTTP cache control (compatibility) + +Setting key: useCouchDBCacheControlHeadersOnly + +Disabled by default. Encrypted CouchDB reads normally use a cache query parameter derived from the plug-in startup time and current Security Seed, and revalidate cached responses. This separates responses from before a remote Rebuild even when document revisions remain unchanged. + +Enable this only if your server or proxy rejects that additional query parameter. Requests then retain their original URLs and use `no-store` with revalidation headers. Some native caches can still reuse older encrypted responses in this mode, so decryption failures after a remote Rebuild may recur. Seed refresh before replication remains active in both modes. Changing this setting does not require a database rebuild. + ### 3. Configuration Encryption #### Encrypting sensitive configuration items diff --git a/src/common/replicatorConfigurationIdentity.ts b/src/common/replicatorConfigurationIdentity.ts index 3cd8cb69..d2a1b1cd 100644 --- a/src/common/replicatorConfigurationIdentity.ts +++ b/src/common/replicatorConfigurationIdentity.ts @@ -82,6 +82,7 @@ export function getCouchDBReplicatorConfigurationIdentity(settings: RemoteDBSett authentication, projectHeaders(settings.couchDB_CustomHeaders), settings.useRequestAPI, + settings.useCouchDBCacheControlHeadersOnly === true, settings.disableRequestURI, projectRemoteSecurity(settings), usesEncryptedInternalMetadata(settings), diff --git a/src/common/replicatorConfigurationIdentity.unit.spec.ts b/src/common/replicatorConfigurationIdentity.unit.spec.ts index 014188e0..13bd92c2 100644 --- a/src/common/replicatorConfigurationIdentity.unit.spec.ts +++ b/src/common/replicatorConfigurationIdentity.unit.spec.ts @@ -7,6 +7,17 @@ import { } from "./replicatorConfigurationIdentity"; describe("active Replicator configuration identity", () => { + it("treats an absent headers-only CouchDB cache switch as off and leaves Object Storage unaffected", () => { + const absent = configuredSettings(); + Reflect.deleteProperty(absent, "useCouchDBCacheControlHeadersOnly"); + expect(getCouchDBReplicatorConfigurationIdentity({ ...absent, useCouchDBCacheControlHeadersOnly: false })).toBe( + getCouchDBReplicatorConfigurationIdentity(absent) + ); + expect(getObjectStorageReplicatorConfigurationIdentity({ ...absent, useCouchDBCacheControlHeadersOnly: true })).toBe( + getObjectStorageReplicatorConfigurationIdentity(absent) + ); + }); + it("replaces a connection when the path identity mode changes", () => { const previous = configuredSettings({ pathNormalizationVersion: 0 }); const next = { ...previous, pathNormalizationVersion: 1 as const }; @@ -73,6 +84,7 @@ describe("active Replicator configuration identity", () => { ["couchDB_PASSWORD", "secret-b"], ["couchDB_CustomHeaders", "X-First: changed"], ["useRequestAPI", true], + ["useCouchDBCacheControlHeadersOnly", true], ["disableRequestURI", true], ["encrypt", false], ["passphrase", "encryption-b"], diff --git a/src/modules/features/SettingDialogue/PanePowerUsers.ts b/src/modules/features/SettingDialogue/PanePowerUsers.ts index b254544f..7129972a 100644 --- a/src/modules/features/SettingDialogue/PanePowerUsers.ts +++ b/src/modules/features/SettingDialogue/PanePowerUsers.ts @@ -28,6 +28,7 @@ export function panePowerUsers( onUpdate: this.onlyOnCouchDB, }); new Setting(paneEl).autoWireToggle("useTimeouts", { onUpdate: this.onlyOnCouchDB }); + new Setting(paneEl).autoWireToggle("useCouchDBCacheControlHeadersOnly", { onUpdate: this.onlyOnCouchDB }); }); void addPanel(paneEl, $msg("Configuration Encryption")).then((paneEl) => { const passphrase_options: Record = { diff --git a/src/modules/services/ObsidianAPIService.ts b/src/modules/services/ObsidianAPIService.ts index ebf4e011..50c0ea3a 100644 --- a/src/modules/services/ObsidianAPIService.ts +++ b/src/modules/services/ObsidianAPIService.ts @@ -220,8 +220,9 @@ export class ObsidianAPIService extends InjectableAPIService key.toLowerCase() === "cache-control" ); @@ -229,11 +230,19 @@ export class ObsidianAPIService extends InjectableAPIService splitHeaderDirectives(transformedHeaders[key])) .filter((directive) => { const name = getDirectiveName(directive); - return !cacheReuseDirectives.has(name) && name !== "no-cache" && name !== "no-store"; + return ( + !cacheReuseDirectives.has(name) && + name !== "no-cache" && + (!enforceNoStore || name !== "no-store") + ); }); const cacheControlKey = cacheControlKeys[0] ?? "cache-control"; for (const key of cacheControlKeys) delete transformedHeaders[key]; - transformedHeaders[cacheControlKey] = [...cacheControlDirectives, "no-cache", "no-store"].join(", "); + transformedHeaders[cacheControlKey] = [ + ...cacheControlDirectives, + "no-cache", + ...(enforceNoStore ? ["no-store"] : []), + ].join(", "); const pragmaKeys = Object.keys(transformedHeaders).filter((key) => key.toLowerCase() === "pragma"); const pragmaDirectives = pragmaKeys diff --git a/src/modules/services/ObsidianAPIService.unit.spec.ts b/src/modules/services/ObsidianAPIService.unit.spec.ts index 10f13081..9eb4aa53 100644 --- a/src/modules/services/ObsidianAPIService.unit.spec.ts +++ b/src/modules/services/ObsidianAPIService.unit.spec.ts @@ -101,6 +101,35 @@ describe("ObsidianAPIService.nativeFetch cache control", () => { expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method }); }); + it.each(["GET", "HEAD", "get", "head"])("revalidates an explicit no-cache %s request", async (method) => { + const headers = { + "Cache-Control": 'public, max-age=3600, no-transform, custom="a,b"', + "cAcHe-CoNtRoL": 'max-stale=30, min-fresh=5, only-if-cached, custom-two="max-age=1, only-if-cached"', + Pragma: "cache", + }; + const originalHeaders = { ...headers }; + + await createService({}).nativeFetch(url, { method, cache: "no-cache", headers }); + + expect(sentHeaders().get("cache-control")).toBe( + 'public, no-transform, custom="a,b", custom-two="max-age=1, only-if-cached", no-cache' + ); + expect(sentHeaders().get("pragma")).toBe("cache, no-cache"); + expect(headers).toEqual(originalHeaders); + expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method }); + }); + + it("preserves an explicit no-store directive when enforcing no-cache", async () => { + const headers = { "Cache-Control": "no-store, max-age=3600", Pragma: "legacy" }; + const originalHeaders = { ...headers }; + + await createService({}).nativeFetch(url, { cache: "no-cache", headers }); + + expect(sentHeaders().get("cache-control")).toBe("no-store, no-cache"); + expect(sentHeaders().get("pragma")).toBe("legacy, no-cache"); + expect(headers).toEqual(originalHeaders); + }); + it("preserves non-conflicting cache directives when enforcing no-store", async () => { const headers = { "Cache-Control": 'max-age=3600, no-transform, custom="a,b"', @@ -140,6 +169,30 @@ describe("ObsidianAPIService.nativeFetch cache control", () => { expect(request.headers.has("cache-control")).toBe(false); }); + it("uses a Request no-cache mode when the options do not override it", async () => { + const request = new Request(url, { + method: "HEAD", + cache: "no-cache", + headers: { "Cache-Control": "max-age=60", "X-Fixture": "request" }, + }); + const headers = new Headers({ "X-Other": "options" }); + + await createService({}).nativeFetch(request, { headers }); + + expect(sentHeaders().get("cache-control")).toBe("no-cache"); + expect(sentHeaders().get("x-fixture")).toBe("request"); + expect(sentHeaders().get("x-other")).toBe("options"); + expect(request.headers.get("cache-control")).toBe("max-age=60"); + expect(headers.has("cache-control")).toBe(false); + }); + + it("lets an explicit default cache mode override a no-cache Request", async () => { + await createService({}).nativeFetch(new Request(url, { cache: "no-cache" }), { cache: "default" }); + + expect(sentHeaders().has("cache-control")).toBe(false); + expect(sentHeaders().has("pragma")).toBe(false); + }); + it("lets the options request no-store for an otherwise cacheable Request", async () => { await createService({}).nativeFetch(new Request(url, { cache: "force-cache" }), { cache: "no-store" }); @@ -153,7 +206,7 @@ describe("ObsidianAPIService.nativeFetch cache control", () => { expect(sentHeaders().has("pragma")).toBe(false); }); - it.each([undefined, "default", "no-cache", "reload", "force-cache"] as const)( + it.each([undefined, "default", "reload", "force-cache"] as const)( "preserves caller headers for cache mode %s", async (cache) => { const headers = { "Cache-Control": "max-age=60", Pragma: "fixture" }; @@ -171,6 +224,14 @@ describe("ObsidianAPIService.nativeFetch cache control", () => { expect(sentHeaders().has("pragma")).toBe(false); }); + it.each(["POST", "PUT", "DELETE"])("preserves a no-cache %s request", async (method) => { + const headers = { "Cache-Control": "max-age=60, no-transform", Pragma: "fixture" }; + + await createService({}).nativeFetch(url, { method, cache: "no-cache", headers }); + + expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method, headers }); + }); + it.each(["POST", "PUT", "DELETE"])("preserves a no-store %s request", async (method) => { const headers = { "Cache-Control": "no-transform" }; diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 1e4fc3a4..758f7e61 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -172,9 +172,9 @@ The Harness also measures ID generation with fixed in-memory data on desktop and The same workflow checks the two remote-activity status boundaries. It first holds a real CouchDB request at the selected fetch implementation and confirms that `🌐N` is visible while `📲` is absent. It then holds the real one-shot replication immediately before its replicator call, confirms that `📲` is visible while no physical request is active, releases it, and requires the finite and bounded activity counts to return to zero, the request and response counts to balance, and both indicators to disappear. Finally, it creates a remote-only chunk, holds the real on-demand fetch immediately before its remote call, makes the same logical active and idle assertions, and verifies that the fetched chunk is written into the local database. These gates make the active states deterministic without replacing the remote request or operation. -`test:e2e:obsidian:couchdb-http-cache` checks replication after rebuilding an encrypted CouchDB remote while an earlier HTTP response remains cached. It enables internal Metadata encryption with Hidden File Sync and Customisation Sync, verifies that the Rebuild replaces the Security Seed and ciphertext while retaining the document revision, and requires the replication pre-check and subsequent sends to succeed without clearing the browser cache. It uses the CouchDB fixture variables, an isolated Vault and profile, and a unique remote database. The raw command uses the current built plug-in by default; `E2E_OBSIDIAN_ARTIFACT_ROOT` can select an exact earlier build for regression comparison. +`test:e2e:obsidian:couchdb-http-cache` checks replication after rebuilding an encrypted CouchDB remote while an earlier HTTP response remains cached. It enables internal Metadata encryption with Hidden File Sync and Customisation Sync, verifies that the Rebuild replaces the Security Seed and ciphertext while retaining the document revision, and requires the replication pre-check and subsequent sends to succeed without clearing the browser cache. It uses the CouchDB fixture variables, an isolated Vault and profile, and a unique remote database. The raw command uses the current built plug-in by default; `E2E_OBSIDIAN_ARTIFACT_ROOT` can select an exact earlier build for regression comparison. It also verifies the combined startup-time/Seed hash, the default-off headers-only compatibility switch through the visible settings UI, its saved value after reopening, and the resulting request URLs and cache modes in both switch positions. -Run `npm run test:e2e:obsidian:focused -- couchdb-http-cache --request-api` to warm Obsidian's Internal API cache and perform the same workflow with **Use Internal API** enabled. Both modes require an explicit `no-store` read to return the rebuilt ciphertext. The default mode also retains the stale browser-cache assertion; the Internal API mode permits reads during Rebuild to refresh that cache. +Run `npm run test:e2e:obsidian:focused -- couchdb-http-cache --request-api` to warm Obsidian's Internal API cache and perform the same workflow with **Use Internal API** enabled. Both modes read decoded Metadata through owned Commonlib connections and observe the actual request URLs. They require the cache URL to change after Security Seed replacement and remain stable across new connections with an unchanged Seed, including after Standard Fetch. The ordinary Vault file must retain its expected contents. The final Standard Fetch phase includes Chunk bodies rather than leaving them for on-demand retrieval, so strict local reconciliation can read the file after database replacement. The default mode also retains the stale browser-cache assertion; the Internal API mode permits reads during Rebuild to refresh that cache. Native adapters may return a full body rather than 304 during revalidation. `npm run test:e2e:obsidian:focused -- chunk-fetch-retry` checks delayed Chunk availability through a real CouchDB service and Obsidian. It creates a Metadata-only remote fixture, starts ordinary one-shot replication with `readChunksOnline`, and inserts the missing Chunk only after a real fetch has returned an empty result. A pass-through observer records the replicator's call times and results without substituting responses or adding waits. The fixture sets the existing minimum request interval to 500 ms to keep the real retry status observable even if finite completion expedites the final probe. The actual status bar must show zero initial requests (`🛄`) and one retry (`🔁`), and both counts must return to zero after delivery ends. On-demand replication excludes Chunk documents from the ordinary pull, so the delayed Chunk must arrive through the observed fetch and produce the exact Vault content. diff --git a/test/e2e-obsidian/scripts/couchdb-http-cache.ts b/test/e2e-obsidian/scripts/couchdb-http-cache.ts index 25a1ec81..03445fb9 100644 --- a/test/e2e-obsidian/scripts/couchdb-http-cache.ts +++ b/test/e2e-obsidian/scripts/couchdb-http-cache.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc"; import { DOCID_SYNC_PARAMETERS } from "@vrtmrz/livesync-commonlib/compat/common/types"; @@ -25,13 +25,17 @@ import { } from "../runner/liveSyncWorkflow.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { createTemporaryVault } from "../runner/vault.ts"; +import { obsidianRemoteDebuggingPort, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; const useRequestAPI = process.argv.includes("--request-api"); +type ObservedRequest = { transport: string; method: string; url: string; cache?: RequestCache }; const customPath = ".obsidian/snippets/http-cache.css"; const nextCustomPath = ".obsidian/snippets/http-cache-new.css"; +const ordinaryPath = "example.md"; +const ordinaryContents = "/* HTTP cache fixture */\n"; const settings = { encrypt: true, passphrase: "couchdb-http-cache-fixture-secret", @@ -48,6 +52,7 @@ const settings = { syncInternalFilesTargetPatterns: "^\\.http-cache(?:/|$)", watchInternalFileChanges: false, useRequestAPI, + usePowerUserMode: true, autoAcceptCompatibleTweak: false, doctorProcessedVersion: DoctorRegulation.version, deviceAndVaultName: "http-cache-source", @@ -62,6 +67,7 @@ async function main(): Promise { const connection = { ...couchDb, dbName }; const vault = await createTemporaryVault(); let session: ObsidianLiveSyncSession | undefined; + let observingRequests = false; const evaluate = async (body: string): Promise => { if (!session) throw new Error("No active Obsidian session."); return await evalObsidianJson( @@ -85,12 +91,102 @@ async function main(): Promise { if(!response.ok)throw new Error('Fixture read failed: '+response.status); return JSON.stringify(await response.json());`); }; + const observeCommonlibRequests = async (): Promise => { + await evaluate(`return (() => { + const api=core.services.API; + const state={requests:[],originals:{},restore:undefined}; + for(const name of ['webCompatFetch','nativeFetch']) { + state.originals[name]=api[name]; + api[name]=function(request,options) { + const url=typeof request==='string'?request:request.url; + const method=options?.method??(typeof request==='string'?'GET':request.method); + state.requests.push({transport:name,method,url,cache:options?.cache}); + return state.originals[name].call(this,request,options); + }; + } + state.restore=()=>{for(const name of Object.keys(state.originals)) api[name]=state.originals[name];}; + globalThis.couchDBHttpCacheObservations=state; + return JSON.stringify(true); + })()`); + observingRequests = true; + }; + const restoreCommonlibRequests = async (): Promise => { + await evaluate(`return (() => { + const state=globalThis.couchDBHttpCacheObservations; + if(state)state.restore(); + delete globalThis.couchDBHttpCacheObservations; + return JSON.stringify(true); + })()`); + observingRequests = false; + }; + const readCommonlibDocument = async (id: string) => + await evaluate<{ + document: CouchDbDocument; + requests: ObservedRequest[]; + }>(`const state=globalThis.couchDBHttpCacheObservations; + if(!state)throw new Error('HTTP request observation is not active'); + const firstRequest=state.requests.length; + const replicator=core.services.replicator.getActiveReplicator(); + const connection=await replicator.connectRemoteCouchDBWithSetting( + core.services.setting.currentSettings(),false,false,true); + if(typeof connection==='string')throw new Error('Could not open the owned Commonlib connection: '+connection); + try { + const document=await connection.db.get(${JSON.stringify(id)}); + return JSON.stringify({document,requests:state.requests.slice(firstRequest)}); + } finally { + await connection.close(); + }`); + + const metadataRequest = ( + read: Awaited>, + id: string + ): ObservedRequest => { + const request = read.requests + .filter((candidate) => { + const path = new URL(candidate.url).pathname; + return decodeURIComponent(path.slice(path.lastIndexOf("/") + 1)) === id; + }) + .slice(-1)[0]; + assert.ok(request, `The owned Commonlib read did not request Metadata ${id}.`); + assert.equal(request.transport, useRequestAPI ? "nativeFetch" : "webCompatFetch"); + assert.equal(request.method.toUpperCase(), "GET"); + return request; + }; + const setHeadersOnlyThroughUI = async (enabled: boolean, expectedBefore: boolean): Promise => { + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const navigator = await openLiveSyncSettings(page); + try { + const pane = await navigator.openPage("Power users"); + const row = pane.locator(".setting-item").filter({ + hasText: "Use headers only for HTTP cache control (compatibility)", + }); + await row.waitFor({ state: "visible" }); + assert.equal(await row.locator(".checkbox-container.is-enabled").count(), Number(expectedBefore)); + await row.locator(".checkbox-container").click(); + await page.waitForFunction( + `app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings().useCouchDBCacheControlHeadersOnly === ${enabled}` + ); + } finally { + await navigator.close(); + } + }); + const saved = JSON.parse(await readFile(join(vault.path, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8")); + assert.equal(saved.useCouchDBCacheControlHeadersOnly, enabled, "The compatibility switch was not saved."); + }; + const assertLogicalDocument = (document: CouchDbDocument, original: CouchDbDocument): void => { + assert.equal(document._id, original._id); + assert.equal(document._rev, original._rev, "The logical Metadata revision changed."); + assert.equal(document.path, ordinaryPath, "The owned connection did not return the logical file path."); + assert.equal(document.type, original.type); + assert.equal(document.size, Buffer.byteLength(ordinaryContents)); + assert.ok(Array.isArray(document.children) && document.children.length > 0); + }; try { await createCouchDbDatabase(couchDb, dbName); - for (const path of ["example.md", ".http-cache/example.json", customPath]) { + for (const path of [ordinaryPath, ".http-cache/example.json", customPath]) { await mkdir(dirname(join(vault.path, path)), { recursive: true }); - await writeFile(join(vault.path, path), "/* HTTP cache fixture */\n"); + await writeFile(join(vault.path, path), ordinaryContents); } session = await startObsidianLiveSyncSession({ binary, @@ -114,6 +210,27 @@ async function main(): Promise { const initialParameters = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); // Populate the HTTP cache as an earlier plug-in version would have done. assert.deepEqual(await readCachedDocument(original._id, "reload"), original); + await observeCommonlibRequests(); + const initialRead = await readCommonlibDocument(original._id); + assertLogicalDocument(initialRead.document, original); + const initialMetadataRequest = metadataRequest(initialRead, original._id); + const initialEpoch = new URL(initialMetadataRequest.url).searchParams.get("_livesync_epoch"); + assert.ok(initialEpoch, "The ordinary Metadata read did not include a Seed cache namespace."); + assert.match(initialEpoch, /^[0-9a-f]{64}$/, "The URL must contain only the combined hash."); + + await setHeadersOnlyThroughUI(true, false); + const compatibilityRead = await readCommonlibDocument(original._id); + assertLogicalDocument(compatibilityRead.document, original); + const compatibilityRequest = metadataRequest(compatibilityRead, original._id); + assert.equal(new URL(compatibilityRequest.url).searchParams.has("_livesync_epoch"), false); + assert.equal(compatibilityRequest.cache, "no-store"); + + await setHeadersOnlyThroughUI(false, true); + const restoredRead = await readCommonlibDocument(original._id); + assertLogicalDocument(restoredRead.document, original); + const restoredRequest = metadataRequest(restoredRead, original._id); + assert.equal(new URL(restoredRequest.url).searchParams.get("_livesync_epoch"), initialEpoch); + assert.equal(restoredRequest.cache, "no-cache"); await evaluate(`await core.services.setting.applyPartial({encryptInternalMetadata:true},true); await core.services.control.applySettings(); @@ -138,6 +255,17 @@ async function main(): Promise { "The stale browser HTTP cache fixture is no longer present." ); } + const rebuiltRead = await readCommonlibDocument(original._id); + assertLogicalDocument(rebuiltRead.document, original); + assert.deepEqual( + rebuiltRead.document.children, + initialRead.document.children, + "The logical file content references changed during Rebuild." + ); + const rebuiltMetadataRequest = metadataRequest(rebuiltRead, original._id); + const rebuiltEpoch = new URL(rebuiltMetadataRequest.url).searchParams.get("_livesync_epoch"); + assert.ok(rebuiltEpoch, "The post-Rebuild Metadata read did not include a Seed cache namespace."); + assert.notEqual(rebuiltEpoch, initialEpoch, "The cache namespace did not change with the Security Seed."); await evaluate(`await core.services.setting.applyPartial({usePluginSync:true,syncInternalFiles:true},true); await core.services.control.applySettings();return JSON.stringify(true);`); @@ -148,13 +276,59 @@ async function main(): Promise { if(opened)await opened.close();return JSON.stringify({admitted:!!opened}); }catch(error){return JSON.stringify({admitted:false,error:error.name});}`); assert.ok(admission.admitted, `Replication admission failed: ${JSON.stringify(admission)}`); - assert.equal((await readCachedDocument(original._id, "no-store")).path, rebuilt.path); + const repeatedRead = await readCommonlibDocument(original._id); + assertLogicalDocument(repeatedRead.document, original); + assert.deepEqual( + repeatedRead.document.children, + initialRead.document.children, + "The logical file content references changed after replication admission." + ); + const repeatedMetadataRequest = metadataRequest(repeatedRead, original._id); + const repeatedEpoch = new URL(repeatedMetadataRequest.url).searchParams.get("_livesync_epoch"); + assert.equal( + repeatedEpoch, + rebuiltEpoch, + "A new connection at the same Seed did not reuse its cache namespace." + ); await pushLocalChanges(cliBinary, session.cliEnv); await pushLocalChanges(cliBinary, session.cliEnv); + // Include Chunk bodies for strict reconciliation after replacing the local database. + await evaluate( + `await core.services.setting.applyPartial({readChunksOnline:false},true);return JSON.stringify(true);` + ); + const fetchAttempt = await evaluate<{ succeeded: boolean; error?: string; diagnostics?: unknown }>(`try { + await core.rebuilder.fetchLocal(false,true);return JSON.stringify({succeeded:true}); + } catch(error) { + await core.services.API.showWindow('log-log'); + await new Promise(resolve=>setTimeout(resolve,100)); + const log=Array.from(document.querySelectorAll('.logpane .log pre')).map(e=>e.textContent??'').join('\\n').slice(-16000); + return JSON.stringify({succeeded:false,error:String(error),diagnostics:{ + databaseReady:core.services.database.isDatabaseReady(),appReady:core.services.appLifecycle.isReady(), + unresolvedMessages:(await core.services.appLifecycle.getUnresolvedMessages()).flat(),log}}); + }`); + assert.ok(fetchAttempt.succeeded, `Standard Fetch failed: ${JSON.stringify(fetchAttempt)}`); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + const fetchedRead = await readCommonlibDocument(original._id); + assertLogicalDocument(fetchedRead.document, original); + assert.deepEqual(fetchedRead.document.children, initialRead.document.children); + assert.equal( + new URL(metadataRequest(fetchedRead, original._id).url).searchParams.get("_livesync_epoch"), + rebuiltEpoch + ); + assert.equal(await readFile(join(vault.path, ordinaryPath), "utf8"), ordinaryContents); + await pushLocalChanges(cliBinary, session.cliEnv); + await restoreCommonlibRequests(); console.log( - `Rebuild and subsequent replication succeeded after warming the ${useRequestAPI ? "Internal API" : "browser"} HTTP cache.` + `Cache compatibility switch persistence, Rebuild, Standard Fetch, and subsequent replication succeeded with the ${useRequestAPI ? "Internal API" : "browser"} HTTP cache.` ); } finally { + if (observingRequests && session) { + try { + await restoreCommonlibRequests(); + } catch { + // The isolated Obsidian process is stopped immediately below. + } + } await session?.app.stop(); await vault.dispose(); await deleteCouchDbDatabase(couchDb, dbName); diff --git a/updates.md b/updates.md index fea046f5..e7c45a4c 100644 --- a/updates.md +++ b/updates.md @@ -14,6 +14,12 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ### Fixed +#### Synchronisation + +- We can synchronise and Fetch an encrypted CouchDB remote when the Internal API retains responses from before a Rebuild. (#1234) + - Metadata and Chunk reads use a cache URL derived from the startup time and Security Seed together. The URL remains stable within the same startup and Seed. + - The advanced CouchDB setting **Use headers only for HTTP cache control (compatibility)** is disabled by default. Enable it only if your server or proxy rejects the additional query parameter; affected native caches may then reuse stale encrypted responses after Rebuild. + #### Translations - The notice shown when a translation is available now displays the **Display Language** setting label correctly in English, French, Hebrew, Russian, and Simplified Chinese. (#1298) Thank you to @cwatanab for the contribution! From 8c51d66adf176aef5d3103e3d6e4a0bc4acd1ee1 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 3 Nov 2026 01:19:07 +0000 Subject: [PATCH 3/8] Document the legacy URL behaviour of cache compatibility --- docs/settings.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/settings.md b/docs/settings.md index e28858b1..974cabdd 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -1100,6 +1100,8 @@ If this option is enabled, PouchDB will hold the connection open for 60 seconds, Setting key: useCouchDBCacheControlHeadersOnly +Remove the query parameter used to invalidate cached responses (the same URL behaviour as in 1.0.44 and earlier). + Disabled by default. Encrypted CouchDB reads normally use a cache query parameter derived from the plug-in startup time and current Security Seed, and revalidate cached responses. This separates responses from before a remote Rebuild even when document revisions remain unchanged. Enable this only if your server or proxy rejects that additional query parameter. Requests then retain their original URLs and use `no-store` with revalidation headers. Some native caches can still reuse older encrypted responses in this mode, so decryption failures after a remote Rebuild may recur. Seed refresh before replication remains active in both modes. Changing this setting does not require a database rebuild. From 0e891998fd2abb9d2bcbf0c60ef56fe1b64f34e3 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 4 Nov 2026 02:09:00 +0000 Subject: [PATCH 4/8] Extract pure native request header helpers --- src/modules/services/ObsidianAPIService.ts | 78 +------------- src/modules/services/nativeRequestHeaders.ts | 100 ++++++++++++++++++ .../nativeRequestHeaders.unit.spec.ts | 84 +++++++++++++++ 3 files changed, 186 insertions(+), 76 deletions(-) create mode 100644 src/modules/services/nativeRequestHeaders.ts create mode 100644 src/modules/services/nativeRequestHeaders.unit.spec.ts diff --git a/src/modules/services/ObsidianAPIService.ts b/src/modules/services/ObsidianAPIService.ts index 50c0ea3a..5f5a5862 100644 --- a/src/modules/services/ObsidianAPIService.ts +++ b/src/modules/services/ObsidianAPIService.ts @@ -6,6 +6,7 @@ import { ObsidianConfirm } from "./ObsidianConfirm"; import type { Confirm } from "@vrtmrz/livesync-commonlib/compat/interfaces/Confirm"; import { requestUrl, type RequestUrlParam } from "@/deps"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +import { createNativeRequestHeaders } from "./nativeRequestHeaders"; // All Services will be migrated to be based on Plain Services, not Injectable Services. // This is a migration step. @@ -16,43 +17,6 @@ declare module "obsidian" { } } -const cacheReuseDirectives = new Set(["max-age", "max-stale", "min-fresh", "only-if-cached"]); - -function splitHeaderDirectives(value: string): string[] { - const directives: string[] = []; - let start = 0; - let inQuotes = false; - let escaped = false; - - for (let index = 0; index < value.length; index++) { - const character = value[index]; - if (inQuotes && escaped) { - escaped = false; - continue; - } - if (inQuotes && character === "\\") { - escaped = true; - continue; - } - if (character === '"') { - inQuotes = !inQuotes; - } else if (character === "," && !inQuotes) { - const directive = value.slice(start, index).trim(); - if (directive) directives.push(directive); - start = index + 1; - } - } - - const lastDirective = value.slice(start).trim(); - if (lastDirective) directives.push(lastDirective); - return directives; -} - -function getDirectiveName(directive: string): string { - const equalsIndex = directive.indexOf("="); - return (equalsIndex === -1 ? directive : directive.slice(0, equalsIndex)).trim().toLowerCase(); -} - export class ObsidianAPIService extends InjectableAPIService { _customHandler: ObsHttpHandler | undefined; _confirmInstance: Confirm; @@ -212,46 +176,8 @@ export class ObsidianAPIService extends InjectableAPIService key.toLowerCase() === "cache-control" - ); - const cacheControlDirectives = cacheControlKeys - .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) - .filter((directive) => { - const name = getDirectiveName(directive); - return ( - !cacheReuseDirectives.has(name) && - name !== "no-cache" && - (!enforceNoStore || name !== "no-store") - ); - }); - const cacheControlKey = cacheControlKeys[0] ?? "cache-control"; - for (const key of cacheControlKeys) delete transformedHeaders[key]; - transformedHeaders[cacheControlKey] = [ - ...cacheControlDirectives, - "no-cache", - ...(enforceNoStore ? ["no-store"] : []), - ].join(", "); - - const pragmaKeys = Object.keys(transformedHeaders).filter((key) => key.toLowerCase() === "pragma"); - const pragmaDirectives = pragmaKeys - .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) - .filter((directive) => getDirectiveName(directive) !== "no-cache"); - const pragmaKey = pragmaKeys[0] ?? "pragma"; - for (const key of pragmaKeys) delete transformedHeaders[key]; - transformedHeaders[pragmaKey] = [...pragmaDirectives, "no-cache"].join(", "); - } + const transformedHeaders = createNativeRequestHeaders(optHeaders, method, cacheMode); const contentType = transformedHeaders["content-type"] ?? transformedHeaders["Content-Type"] ?? "application/json"; const requestParam: RequestUrlParam = { diff --git a/src/modules/services/nativeRequestHeaders.ts b/src/modules/services/nativeRequestHeaders.ts new file mode 100644 index 00000000..2a125664 --- /dev/null +++ b/src/modules/services/nativeRequestHeaders.ts @@ -0,0 +1,100 @@ +const cacheReuseDirectives = new Set(["max-age", "max-stale", "min-fresh", "only-if-cached"]); + +function splitHeaderDirectives(value: string): string[] { + const directives: string[] = []; + let start = 0; + let inQuotes = false; + let escaped = false; + + for (let index = 0; index < value.length; index++) { + const character = value[index]; + if (inQuotes && escaped) { + escaped = false; + continue; + } + if (inQuotes && character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + inQuotes = !inQuotes; + } else if (character === "," && !inQuotes) { + const directive = value.slice(start, index).trim(); + if (directive) directives.push(directive); + start = index + 1; + } + } + + const lastDirective = value.slice(start).trim(); + if (lastDirective) directives.push(lastDirective); + return directives; +} + +function getDirectiveName(directive: string): string { + const equalsIndex = directive.indexOf("="); + return (equalsIndex === -1 ? directive : directive.slice(0, equalsIndex)).trim().toLowerCase(); +} + +/** + * Whether native GET or HEAD requests must revalidate an explicitly bypassed cache. + * + * @param method - HTTP method, compared without regard to letter case. + * @param cacheMode - Effective Request or options cache mode. + * @returns True only for GET/HEAD with `no-cache` or `no-store`. + */ +export function shouldRevalidateNativeRequest(method: string, cacheMode: RequestCache | undefined): boolean { + return (cacheMode === "no-cache" || cacheMode === "no-store") && ["GET", "HEAD"].includes(method.toUpperCase()); +} + +function shouldRetainCacheControlDirective(directive: string, enforceNoStore: boolean): boolean { + const name = getDirectiveName(directive); + return !cacheReuseDirectives.has(name) && name !== "no-cache" && (!enforceNoStore || name !== "no-store"); +} + +/** + * Copy request headers and apply the native fetch adapter's header/cache policy. + * + * @param headers - Merged input headers; this function does not mutate them. + * @param method - HTTP method used to decide whether cache revalidation applies. + * @param cacheMode - Effective Request or options cache mode. + * @returns Fresh headers with the adapter's native-controlled headers removed. + * Revalidated GET/HEAD requests retain non-conflicting directives and receive + * `no-cache`; `no-store` also adds that directive. Existing header spelling is retained. + */ +export function createNativeRequestHeaders( + headers: Readonly>, + method: string, + cacheMode: RequestCache | undefined +): Record { + const transformedHeaders = { ...headers }; + // Delete headers that should not be sent by native fetch, + // they are controlled by the browser and may cause CORS preflight failure if sent manually. + delete transformedHeaders["host"]; + delete transformedHeaders["Host"]; + delete transformedHeaders["content-length"]; + delete transformedHeaders["Content-Length"]; + if (shouldRevalidateNativeRequest(method, cacheMode)) { + // requestUrl has no cache option; no-store alone can reuse an existing cached response. + const enforceNoStore = cacheMode === "no-store"; + const cacheControlKeys = Object.keys(transformedHeaders).filter((key) => key.toLowerCase() === "cache-control"); + const cacheControlDirectives = cacheControlKeys + .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) + .filter((directive) => shouldRetainCacheControlDirective(directive, enforceNoStore)); + const cacheControlKey = cacheControlKeys[0] ?? "cache-control"; + for (const key of cacheControlKeys) delete transformedHeaders[key]; + transformedHeaders[cacheControlKey] = [ + ...cacheControlDirectives, + "no-cache", + ...(enforceNoStore ? ["no-store"] : []), + ].join(", "); + + const pragmaKeys = Object.keys(transformedHeaders).filter((key) => key.toLowerCase() === "pragma"); + const pragmaDirectives = pragmaKeys + .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) + .filter((directive) => getDirectiveName(directive) !== "no-cache"); + const pragmaKey = pragmaKeys[0] ?? "pragma"; + for (const key of pragmaKeys) delete transformedHeaders[key]; + transformedHeaders[pragmaKey] = [...pragmaDirectives, "no-cache"].join(", "); + } + return transformedHeaders; +} diff --git a/src/modules/services/nativeRequestHeaders.unit.spec.ts b/src/modules/services/nativeRequestHeaders.unit.spec.ts new file mode 100644 index 00000000..96455ef5 --- /dev/null +++ b/src/modules/services/nativeRequestHeaders.unit.spec.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { createNativeRequestHeaders, shouldRevalidateNativeRequest } from "./nativeRequestHeaders"; + +describe("native cache revalidation decision", () => { + it.each(["GET", "HEAD", "get", "head"])("revalidates explicit no-cache/no-store for %s", (method) => { + expect(shouldRevalidateNativeRequest(method, "no-cache")).toBe(true); + expect(shouldRevalidateNativeRequest(method, "no-store")).toBe(true); + }); + + it.each([undefined, "default", "reload", "force-cache", "only-if-cached"] as const)( + "does not strengthen cache mode %s", + (mode) => { + expect(shouldRevalidateNativeRequest("GET", mode)).toBe(false); + expect(shouldRevalidateNativeRequest("HEAD", mode)).toBe(false); + } + ); + + it.each(["POST", "PUT", "PATCH", "DELETE", "OPTIONS"])("does not revalidate %s", (method) => { + expect(shouldRevalidateNativeRequest(method, "no-cache")).toBe(false); + expect(shouldRevalidateNativeRequest(method, "no-store")).toBe(false); + }); +}); + +describe("native request header creation", () => { + it("copies input headers and removes the existing native-controlled header forms", () => { + const input = Object.freeze({ + host: "first.test", + Host: "second.test", + "content-length": "1", + "Content-Length": "2", + Authorization: "Bearer fixture", + "Cache-Control": "max-age=60", + }); + const result = createNativeRequestHeaders(input, "GET", "default"); + expect(result).toEqual({ Authorization: "Bearer fixture", "Cache-Control": "max-age=60" }); + expect(result).not.toBe(input); + expect(input.host).toBe("first.test"); + expect(input["Content-Length"]).toBe("2"); + }); + + it.each(["GET", "HEAD"])("creates no-store revalidation headers for %s", (method) => { + expect(createNativeRequestHeaders({}, method, "no-store")).toEqual({ + "cache-control": "no-cache, no-store", + pragma: "no-cache", + }); + }); + + it("preserves an existing no-store directive when enforcing no-cache", () => { + expect( + createNativeRequestHeaders( + { "Cache-Control": "no-store, max-age=3600", Pragma: "legacy" }, + "GET", + "no-cache" + ) + ).toEqual({ "Cache-Control": "no-store, no-cache", Pragma: "legacy, no-cache" }); + }); + + it("combines differently cased header fields and retains non-conflicting directives", () => { + const input = Object.freeze({ + "Cache-Control": 'Max-Age=99, no-transform, custom="a,b", No-Cache', + "cache-control": "max-stale=5, min-fresh=6, only-if-cached, NO-STORE, private", + Pragma: 'legacy="a,b", No-Cache', + pragma: "extension, no-cache", + }); + expect(createNativeRequestHeaders(input, "get", "no-store")).toEqual({ + "Cache-Control": 'no-transform, custom="a,b", private, no-cache, no-store', + Pragma: 'legacy="a,b", extension, no-cache', + }); + expect(input["cache-control"]).toContain("max-stale=5"); + }); + + it("does not split directive commas inside escaped quoted strings", () => { + const custom = String.raw`custom="a\"b,c"`; + expect(createNativeRequestHeaders({ "cache-control": `${custom}, max-age=1` }, "HEAD", "no-cache")).toEqual({ + "cache-control": `${custom}, no-cache`, + pragma: "no-cache", + }); + }); + + it("retains cache headers on a non-revalidated method", () => { + const headers = { "cache-control": "max-age=99", Pragma: "legacy" }; + expect(createNativeRequestHeaders(headers, "POST", "no-store")).toEqual(headers); + }); +}); From 988a10974b88c6859388aacf100a0001eaf69455 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 4 Nov 2026 03:02:32 +0000 Subject: [PATCH 5/8] Test empty cache directive lists --- .../services/nativeRequestHeaders.unit.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/modules/services/nativeRequestHeaders.unit.spec.ts b/src/modules/services/nativeRequestHeaders.unit.spec.ts index 96455ef5..0f74b21f 100644 --- a/src/modules/services/nativeRequestHeaders.unit.spec.ts +++ b/src/modules/services/nativeRequestHeaders.unit.spec.ts @@ -77,6 +77,20 @@ describe("native request header creation", () => { }); }); + it.each([ + ["", "no-cache"], + [", , ,", "no-cache"], + [" , no-transform , , ", "no-transform, no-cache"], + ])("ignores empty header-list members in %j", (value, expected) => { + const headers = Object.freeze({ "Cache-Control": value, Pragma: " , , " }); + + expect(createNativeRequestHeaders(headers, "GET", "no-cache")).toEqual({ + "Cache-Control": expected, + Pragma: "no-cache", + }); + expect(headers["Cache-Control"]).toBe(value); + }); + it("retains cache headers on a non-revalidated method", () => { const headers = { "cache-control": "max-age=99", Pragma: "legacy" }; expect(createNativeRequestHeaders(headers, "POST", "no-store")).toEqual(headers); From 6b21224746f4b608c5d7605299db4358ee0b4872 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 4 Nov 2026 05:06:56 +0000 Subject: [PATCH 6/8] Use Commonlib 0.3.1 for CouchDB cache revalidation --- package-lock.json | 32 +++++++++++++++++++++++++++----- package.json | 2 +- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 911e5037..3e44ee14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.3", + "@vrtmrz/livesync-commonlib": "0.3.1", "@vrtmrz/obsidian-plugin-kit": "0.1.7", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", @@ -1823,6 +1823,12 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz", + "integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==", + "license": "BlueOak-1.0.0" + }, "node_modules/@istanbuljs/schema": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", @@ -4121,9 +4127,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.3.tgz", - "integrity": "sha512-n1BaoegBVGDqerZ9EcNbgd4KFtRPJ/lPp5ADsAsHTkPzSwev4SAFxxyFgFKLpTLpPv7McFao4oGBHGvRvoXWEg==", + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.3.1.tgz", + "integrity": "sha512-/vm/5MEiFIS1mEfW/+kPaqbx/cpgXlEl+uQHTm5ClHxNZ/umARuxXZX33TlxFpVimLVjYPRQ97XSZAz5HKy8vw==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -4140,7 +4146,7 @@ "fflate": "^0.8.2", "idb": "^8.0.3", "markdown-it": "^14.3.3", - "minimatch": "^10.2.6", + "minimatch": "^11.0.3", "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-idb": "^9.0.0", @@ -4187,6 +4193,22 @@ "@trystero-p2p/core": "0.25.3" } }, + "node_modules/@vrtmrz/livesync-commonlib/node_modules/minimatch": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.4.tgz", + "integrity": "sha512-/UJ1dEoVQTjPNDh6SGlUo+ueLMmoHTas4q+8yaAR/5t6nLAIiBDJtoYUh20IYleix7ehiQkgczb84PE3uzHkxg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/nonrecursive-glob-match": "^1.0.3", + "brace-expansion": "^5.0.12" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@vrtmrz/obsidian-plugin-kit": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/@vrtmrz/obsidian-plugin-kit/-/obsidian-plugin-kit-0.1.7.tgz", diff --git a/package.json b/package.json index 182e7ee3..31595b2f 100644 --- a/package.json +++ b/package.json @@ -209,7 +209,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.3", + "@vrtmrz/livesync-commonlib": "0.3.1", "@vrtmrz/obsidian-plugin-kit": "0.1.7", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", From 5c759ef51ce66658b3817dc12d8a7b97cfac7b5b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 4 Nov 2026 07:15:42 +0000 Subject: [PATCH 7/8] Verify CouchDB cache behaviour across application restarts --- .../scripts/couchdb-http-cache.ts | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) diff --git a/test/e2e-obsidian/scripts/couchdb-http-cache.ts b/test/e2e-obsidian/scripts/couchdb-http-cache.ts index 03445fb9..83bbaf49 100644 --- a/test/e2e-obsidian/scripts/couchdb-http-cache.ts +++ b/test/e2e-obsidian/scripts/couchdb-http-cache.ts @@ -22,6 +22,7 @@ import { prepareRemote, pushLocalChanges, waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, } from "../runner/liveSyncWorkflow.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { createTemporaryVault } from "../runner/vault.ts"; @@ -31,11 +32,37 @@ process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; const useRequestAPI = process.argv.includes("--request-api"); type ObservedRequest = { transport: string; method: string; url: string; cache?: RequestCache }; +type RestartState = { + settings: { + useCouchDBCacheControlHeadersOnly: boolean; + couchDB_DBNAME: string; + deviceAndVaultName: string; + encrypt: boolean; + encryptInternalMetadata: boolean; + usePathObfuscation: boolean; + }; + localDatabaseName: string; + localMetadata: null | { id: string; revision: string; path: string; children: string[] }; + nodeId: string; +}; + +async function waitForNextUnixSecond(previousUnixSecond: number): Promise { + const timeoutMs = Number(process.env.E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS ?? 20000); + const deadline = Date.now() + timeoutMs; + while (Math.floor(Date.now() / 1000) <= previousUnixSecond) { + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for the next Unix second before restarting Obsidian."); + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} const customPath = ".obsidian/snippets/http-cache.css"; const nextCustomPath = ".obsidian/snippets/http-cache-new.css"; const ordinaryPath = "example.md"; const ordinaryContents = "/* HTTP cache fixture */\n"; +const restartNotePath = "restart-after-http-cache.md"; +const restartNoteContents = "Synchronised after restarting Obsidian.\n"; const settings = { encrypt: true, passphrase: "couchdb-http-cache-fixture-secret", @@ -181,6 +208,22 @@ async function main(): Promise { assert.equal(document.size, Buffer.byteLength(ordinaryContents)); assert.ok(Array.isArray(document.children) && document.children.length > 0); }; + const readRestartState = async (): Promise => + await evaluate([ + "const settings=core.services.setting.currentSettings();", + "const database=core.localDatabase.localDatabase;", + "const databaseInfo=await database.info();", + "const nodeInfo=await database.get('_local/obsydian_livesync_nodeinfo').catch(()=>null);", + "const localMetadata=await core.localDatabase.getDBEntry('example.md',undefined,false,true).catch(()=>null);", + "return JSON.stringify({settings:{", + "useCouchDBCacheControlHeadersOnly:settings.useCouchDBCacheControlHeadersOnly===true,", + "couchDB_DBNAME:settings.couchDB_DBNAME,deviceAndVaultName:core.services.setting.getDeviceAndVaultName(),", + "encrypt:settings.encrypt,encryptInternalMetadata:settings.encryptInternalMetadata,", + "usePathObfuscation:settings.usePathObfuscation},", + "localDatabaseName:databaseInfo.db_name,", + "localMetadata:localMetadata?{id:localMetadata._id,revision:localMetadata._rev,path:localMetadata.path,children:localMetadata.children??[]}:null,", + "nodeId:nodeInfo?.nodeid??''});", + ].join("")); try { await createCouchDbDatabase(couchDb, dbName); @@ -317,10 +360,180 @@ async function main(): Promise { ); assert.equal(await readFile(join(vault.path, ordinaryPath), "utf8"), ordinaryContents); await pushLocalChanges(cliBinary, session.cliEnv); + + const remoteBeforeRestart = await fetchCouchDbDocument(couchDb, dbName, original._id); + const seedBeforeRestart = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); + assert.equal(remoteBeforeRestart._rev, rebuilt._rev, "The Metadata revision changed before restart."); + assert.equal(remoteBeforeRestart.path, rebuilt.path, "The remote Metadata ciphertext changed before restart."); + assert.equal( + seedBeforeRestart.pbkdf2salt, + rebuiltParameters.pbkdf2salt, + "The Security Seed changed before restart." + ); + const stateBeforeRestart = await readRestartState(); + assert.equal(stateBeforeRestart.settings.useCouchDBCacheControlHeadersOnly, false); + assert.equal(stateBeforeRestart.settings.couchDB_DBNAME, dbName); + assert.equal(stateBeforeRestart.settings.deviceAndVaultName, "http-cache-source"); + assert.equal(stateBeforeRestart.settings.encrypt, true); + assert.equal(stateBeforeRestart.settings.encryptInternalMetadata, true); + assert.equal(stateBeforeRestart.settings.usePathObfuscation, true); + assert.equal(stateBeforeRestart.localMetadata?.path, ordinaryPath); + assert.ok(stateBeforeRestart.nodeId, "The device identity was not persisted before restart."); + + const previousUnixSecond = Math.floor(Date.now() / 1000); + await restoreCommonlibRequests(); + if (!session) throw new Error("No active Obsidian session to stop."); + await session.app.stop(); + session = undefined; + await waitForNextUnixSecond(previousUnixSecond); + + // Reuse the same Vault and isolated profile. Do not reseed settings or local device state. + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + artifactRoot: resolve(process.env.E2E_OBSIDIAN_ARTIFACT_ROOT ?? "."), + pluginStartup: "natural", + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + const stateAfterRestart = await readRestartState(); + assert.deepEqual(stateAfterRestart.settings, stateBeforeRestart.settings); + assert.equal(stateAfterRestart.localDatabaseName, stateBeforeRestart.localDatabaseName); + assert.deepEqual(stateAfterRestart.localMetadata, stateBeforeRestart.localMetadata); + assert.equal(stateAfterRestart.nodeId, stateBeforeRestart.nodeId, "Restart changed the persisted device identity."); + + const remoteAfterRestart = await fetchCouchDbDocument(couchDb, dbName, original._id); + const seedAfterRestart = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); + assert.equal(remoteAfterRestart._rev, remoteBeforeRestart._rev, "Restart changed the remote Metadata revision."); + assert.equal( + remoteAfterRestart.path, + remoteBeforeRestart.path, + "Restart changed the remote Metadata ciphertext." + ); + assert.equal(seedAfterRestart.pbkdf2salt, seedBeforeRestart.pbkdf2salt, "Restart changed the Security Seed."); + assert.equal(await readFile(join(vault.path, ordinaryPath), "utf8"), ordinaryContents); + if (!useRequestAPI) { + assert.equal( + (await readCachedDocument(original._id, "default")).path, + original.path, + "The isolated browser HTTP cache did not survive the application restart." + ); + } + + await observeCommonlibRequests(); + const restartedRead = await readCommonlibDocument(original._id); + assertLogicalDocument(restartedRead.document, original); + assert.deepEqual(restartedRead.document.children, initialRead.document.children); + const restartedMetadataRequest = metadataRequest(restartedRead, original._id); + const restartedEpoch = new URL(restartedMetadataRequest.url).searchParams.get("_livesync_epoch"); + assert.ok(restartedEpoch, "The restarted Metadata read did not include a cache namespace."); + assert.match(restartedEpoch, /^[0-9a-f]{64}$/, "The restarted URL must contain only the combined hash."); + assert.notEqual( + restartedEpoch, + rebuiltEpoch, + "A full application restart reused its earlier non-compatibility cache namespace." + ); + assert.equal(restartedMetadataRequest.cache, "no-cache"); + + await evaluate("await core.services.setting.applyPartial({readChunksOnline:false},true);return JSON.stringify(true);"); + const restartedFetchAttempt = await evaluate<{ succeeded: boolean; error?: string }>([ + "try{await core.rebuilder.fetchLocal(false,true);return JSON.stringify({succeeded:true});}", + "catch(error){return JSON.stringify({succeeded:false,error:String(error)});}", + ].join("")); + assert.ok( + restartedFetchAttempt.succeeded, + `Standard Fetch after restart failed: ${JSON.stringify(restartedFetchAttempt)}` + ); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + const afterRestartFetch = await readCommonlibDocument(original._id); + assertLogicalDocument(afterRestartFetch.document, original); + assert.deepEqual(afterRestartFetch.document.children, initialRead.document.children); + assert.equal( + new URL(metadataRequest(afterRestartFetch, original._id).url).searchParams.get("_livesync_epoch"), + restartedEpoch, + "Standard Fetch changed the restarted cache namespace." + ); + assert.equal(await readFile(join(vault.path, ordinaryPath), "utf8"), ordinaryContents); + const remoteAfterRestartFetch = await fetchCouchDbDocument(couchDb, dbName, original._id); + const seedAfterRestartFetch = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); + assert.equal(remoteAfterRestartFetch._rev, remoteBeforeRestart._rev); + assert.equal(remoteAfterRestartFetch.path, remoteBeforeRestart.path); + assert.equal(seedAfterRestartFetch.pbkdf2salt, seedBeforeRestart.pbkdf2salt); + + await writeFile(join(vault.path, restartNotePath), restartNoteContents); + const restartNoteEntry = await waitForLocalDatabaseEntry(cliBinary, session.cliEnv, restartNotePath); + assert.equal(restartNoteEntry.path, restartNotePath); + await pushLocalChanges(cliBinary, session.cliEnv); + const uploadedRestartNote = await readCommonlibDocument(restartNoteEntry.id); + assert.equal(uploadedRestartNote.document.path, restartNotePath); + assert.equal(uploadedRestartNote.document.size, Buffer.byteLength(restartNoteContents)); + assert.ok( + Array.isArray(uploadedRestartNote.document.children) && uploadedRestartNote.document.children.length > 0 + ); + assert.equal( + new URL(metadataRequest(uploadedRestartNote, restartNoteEntry.id).url).searchParams.get("_livesync_epoch"), + restartedEpoch, + "Subsequent synchronisation did not use the restarted cache namespace." + ); + assert.equal(await readFile(join(vault.path, restartNotePath), "utf8"), restartNoteContents); + const originalAfterRestartSync = await fetchCouchDbDocument(couchDb, dbName, original._id); + assert.equal(originalAfterRestartSync._rev, remoteBeforeRestart._rev); + assert.equal(originalAfterRestartSync.path, remoteBeforeRestart.path); + const uploadedRestartNoteRemote = await fetchCouchDbDocument(couchDb, dbName, restartNoteEntry.id); + + await evaluate("await core.services.setting.applyPartial({readChunksOnline:false},true);return JSON.stringify(true);"); + const uploadVerificationFetch = await evaluate<{ succeeded: boolean; error?: string }>([ + "try{await core.rebuilder.fetchLocal(false,true);return JSON.stringify({succeeded:true});}", + "catch(error){return JSON.stringify({succeeded:false,error:String(error)});}", + ].join("")); + assert.ok( + uploadVerificationFetch.succeeded, + `Standard Fetch after the upload failed: ${JSON.stringify(uploadVerificationFetch)}` + ); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + const fetchedRestartNoteEntry = await waitForLocalDatabaseEntry(cliBinary, session.cliEnv, restartNotePath); + assert.equal(fetchedRestartNoteEntry.id, restartNoteEntry.id); + assert.equal(fetchedRestartNoteEntry.rev, uploadedRestartNote.document._rev); + assert.deepEqual(fetchedRestartNoteEntry.children, uploadedRestartNote.document.children); + const fetchedRestartNoteBody = await evaluate([ + "const file=await core.serviceModules.databaseFileAccess.fetch(" + JSON.stringify(restartNotePath) + ");", + "if(!file||file.deleted||!file.body)throw new Error('The uploaded note was not readable from the replaced local database.');", + "return JSON.stringify(await file.body.text());", + ].join("")); + assert.equal( + fetchedRestartNoteBody, + restartNoteContents, + "The replaced local database did not return the exact uploaded Chunk content." + ); + const fetchedRestartNote = await readCommonlibDocument(restartNoteEntry.id); + assert.equal(fetchedRestartNote.document.path, restartNotePath); + assert.equal(fetchedRestartNote.document.size, Buffer.byteLength(restartNoteContents)); + assert.equal(fetchedRestartNote.document._rev, uploadedRestartNote.document._rev); + assert.deepEqual(fetchedRestartNote.document.children, uploadedRestartNote.document.children); + assert.equal( + new URL(metadataRequest(fetchedRestartNote, restartNoteEntry.id).url).searchParams.get("_livesync_epoch"), + restartedEpoch, + "Standard Fetch after the upload changed the restarted cache namespace." + ); + assert.equal(await readFile(join(vault.path, restartNotePath), "utf8"), restartNoteContents); + assert.equal(await readFile(join(vault.path, ordinaryPath), "utf8"), ordinaryContents); + const restartNoteAfterFetch = await fetchCouchDbDocument(couchDb, dbName, restartNoteEntry.id); + assert.equal(restartNoteAfterFetch._rev, uploadedRestartNoteRemote._rev); + assert.equal(restartNoteAfterFetch.path, uploadedRestartNoteRemote.path); + const originalAfterUploadFetch = await fetchCouchDbDocument(couchDb, dbName, original._id); + const seedAfterUploadFetch = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); + assert.equal(originalAfterUploadFetch._rev, remoteBeforeRestart._rev); + assert.equal(originalAfterUploadFetch.path, remoteBeforeRestart.path); + assert.equal(seedAfterUploadFetch.pbkdf2salt, seedBeforeRestart.pbkdf2salt); + await restoreCommonlibRequests(); console.log( `Cache compatibility switch persistence, Rebuild, Standard Fetch, and subsequent replication succeeded with the ${useRequestAPI ? "Internal API" : "browser"} HTTP cache.` ); + console.log( + `Full application restart on the same Vault and profile retained device identity, ` + + `changed the cache namespace, and passed post-restart Fetch and upload (${useRequestAPI ? "Internal API" : "browser"}).` + ); } finally { if (observingRequests && session) { try { From e7d77ca2323d2a531431bc8e1e5b4cad7516603f Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 4 Nov 2026 08:50:22 +0000 Subject: [PATCH 8/8] Report Journal recovery phase and CLI output on failure --- test/e2e-cli/test-journal-chunk-recovery.ts | 34 ++++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/test/e2e-cli/test-journal-chunk-recovery.ts b/test/e2e-cli/test-journal-chunk-recovery.ts index 345fbc88..6bf03389 100644 --- a/test/e2e-cli/test-journal-chunk-recovery.ts +++ b/test/e2e-cli/test-journal-chunk-recovery.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; -import { assertFilesEqual, runCliOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; +import { assertFilesEqual, runCli, runCliOrFail, sanitiseCatStdout, type CliResult } from "./helpers/cli.ts"; import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; import { startMinio, stopMinio } from "./helpers/docker.ts"; @@ -34,6 +34,20 @@ function createJournalFixture(): string { ].join(""); } +async function assertFetchedVaultMatches( + sourceFile: string, + receiverFile: string, + result: CliResult, + message: string +): Promise { + assert.equal(result.code, 0, `${message}\nCLI exit code: ${result.code}\n${result.combined}`); + try { + await assertFilesEqual(sourceFile, receiverFile, message); + } catch (error) { + throw new Error(`${message}\nCLI output:\n${result.combined}`, { cause: error }); + } +} + async function runScenario(encrypt: boolean): Promise { const suffix = `${encrypt ? "enc1" : "enc0"}-${Date.now()}-${Math.floor(Math.random() * 100000)}`; const minioEndpoint = requireEnv("MINIO_ENDPOINT", "minioEndpoint").replace(/\/$/, ""); @@ -99,7 +113,7 @@ async function runScenario(encrypt: boolean): Promise { "--force", "--write-settings" ); - await runCliOrFail( + const initialFetch = await runCli( receiverDatabase, "--settings", receiverSettings, @@ -110,7 +124,12 @@ async function runScenario(encrypt: boolean): Promise { "--write-settings" ); - await assertFilesEqual(sourceFile, receiverFile, "Fetched Vault file does not match the rebuild source"); + await assertFetchedVaultMatches( + sourceFile, + receiverFile, + initialFetch, + "Initial fetch did not reproduce the rebuild source in the receiver Vault" + ); const fetchedContent = sanitiseCatStdout( await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "cat", targetPath) ); @@ -163,7 +182,7 @@ async function runScenario(encrypt: boolean): Promise { "Fetch should retain both the intact remote and differing local contents without inferring their ancestry" ); await rename(receiverFile, workDir.join("damaged-vault-backup.md")); - await runCliOrFail( + const restoreFetch = await runCli( receiverDatabase, "--settings", receiverSettings, @@ -173,7 +192,12 @@ async function runScenario(encrypt: boolean): Promise { "--force", "--write-settings" ); - await assertFilesEqual(sourceFile, receiverFile, "Fetching into an empty Vault should restore the remote file"); + await assertFetchedVaultMatches( + sourceFile, + receiverFile, + restoreFetch, + "Fetch did not restore the remote file after the damaged Vault copy was moved" + ); assert.equal(await readFile(workDir.join("damaged-vault-backup.md"), "utf8"), damagedContent); const restoredContent = sanitiseCatStdout( await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "cat", targetPath)