From a7274aafd62c15c77df7728a46182172bfdb8921 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 1 Nov 2026 07:23:09 +0000 Subject: [PATCH 1/5] Honour no-store reads through Obsidian's Internal API --- src/modules/services/ObsidianAPIService.ts | 11 ++ .../services/ObsidianAPIService.unit.spec.ts | 105 +++++++++++++++++- test/e2e-obsidian/README.md | 2 + .../scripts/couchdb-http-cache.ts | 28 +++-- updates.md | 2 + 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/src/modules/services/ObsidianAPIService.ts b/src/modules/services/ObsidianAPIService.ts index c455d172..b319ac43 100644 --- a/src/modules/services/ObsidianAPIService.ts +++ b/src/modules/services/ObsidianAPIService.ts @@ -182,6 +182,17 @@ export class ObsidianAPIService extends InjectableAPIService ({ + requestUrl: vi.fn< + (request: RequestUrlParam) => Promise<{ + status: number; + headers: Record; + arrayBuffer: ArrayBuffer; + }> + >(), platform: { isMobile: false, }, @@ -8,12 +16,12 @@ const mocks = vi.hoisted(() => ({ vi.mock("@/deps.ts", () => ({ Platform: mocks.platform, - requestUrl: vi.fn(), + requestUrl: mocks.requestUrl, })); vi.mock("@/deps", () => ({ Platform: mocks.platform, - requestUrl: vi.fn(), + requestUrl: mocks.requestUrl, })); vi.mock("@/modules/essentialObsidian/APILib/ObsHttpHandler", () => ({ @@ -36,6 +44,8 @@ function createService(workspace: Record, isMobile = false): Ob beforeEach(() => { mocks.platform.isMobile = false; vi.clearAllMocks(); + mocks.requestUrl.mockReset(); + mocks.requestUrl.mockResolvedValue({ status: 200, headers: {}, arrayBuffer: new ArrayBuffer(0) }); }); describe("ObsidianAPIService.showWindowOnRight", () => { @@ -65,3 +75,94 @@ describe("ObsidianAPIService.showWindowOnRight", () => { expect(workspace.revealLeaf).toHaveBeenCalledWith(rightLeaf); }); }); + +describe("ObsidianAPIService.nativeFetch cache control", () => { + const url = "https://couchdb.example.test/vault/f%3Ametadata"; + + function sentHeaders(): Headers { + expect(mocks.requestUrl).toHaveBeenCalledOnce(); + return new Headers(mocks.requestUrl.mock.calls[0][0].headers); + } + + it.each(["GET", "HEAD", "get", "head"])("bypasses cached responses for an explicit no-store %s", async (method) => { + const headers = { + Authorization: "Basic fixture", + "Cache-Control": "public, max-age=3600", + "cAcHe-CoNtRoL": "max-stale", + Pragma: "cache", + }; + + await createService({}).nativeFetch(url, { method, cache: "no-store", headers }); + + expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store"); + expect(sentHeaders().get("pragma")).toBe("no-cache"); + expect(sentHeaders().get("authorization")).toBe("Basic fixture"); + expect(headers["Cache-Control"]).toBe("public, max-age=3600"); + expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method }); + }); + + it("uses the Request cache mode when the options do not override it", async () => { + const request = new Request(url, { + method: "HEAD", + cache: "no-store", + headers: { "X-Fixture": "request" }, + }); + const headers = new Headers({ "X-Other": "options" }); + + await createService({}).nativeFetch(request, { headers }); + + expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store"); + expect(sentHeaders().get("pragma")).toBe("no-cache"); + expect(sentHeaders().get("x-fixture")).toBe("request"); + expect(sentHeaders().get("x-other")).toBe("options"); + expect(headers.has("cache-control")).toBe(false); + expect(request.headers.has("cache-control")).toBe(false); + }); + + it("lets the options request no-store for an otherwise cacheable Request", async () => { + await createService({}).nativeFetch(new Request(url, { cache: "force-cache" }), { cache: "no-store" }); + + expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store"); + }); + + it("lets an explicit default cache mode override a no-store Request", async () => { + await createService({}).nativeFetch(new Request(url, { cache: "no-store" }), { cache: "default" }); + + expect(sentHeaders().has("cache-control")).toBe(false); + expect(sentHeaders().has("pragma")).toBe(false); + }); + + it.each([undefined, "default", "no-cache", "reload", "force-cache"] as const)( + "preserves caller headers for cache mode %s", + async (cache) => { + const headers = { "Cache-Control": "max-age=60", Pragma: "fixture" }; + + await createService({}).nativeFetch(url, { cache, headers }); + + expect(mocks.requestUrl.mock.calls[0][0].headers).toEqual(headers); + } + ); + + it("does not add cache directives to an ordinary request", async () => { + await createService({}).nativeFetch(url); + + expect(sentHeaders().has("cache-control")).toBe(false); + expect(sentHeaders().has("pragma")).toBe(false); + }); + + it.each(["POST", "PUT", "DELETE"])("preserves a no-store %s request", async (method) => { + const headers = { "Cache-Control": "no-transform" }; + + await createService({}).nativeFetch(url, { method, cache: "no-store", headers }); + + expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method, headers }); + expect(sentHeaders().has("pragma")).toBe(false); + }); + + it("uses the effective method from the options", async () => { + await createService({}).nativeFetch(new Request(url, { cache: "no-store" }), { method: "POST" }); + + expect(sentHeaders().has("cache-control")).toBe(false); + expect(sentHeaders().has("pragma")).toBe(false); + }); +}); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 361db22c..66cfa299 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -174,6 +174,8 @@ The same workflow checks the two remote-activity status boundaries. It first hol `test:e2e:obsidian:couchdb-http-cache` checks replication after rebuilding an encrypted CouchDB remote while an earlier HTTP response remains cached. It enables internal Metadata encryption with Hidden File Sync and Customisation Sync, verifies that the Rebuild replaces the Security Seed and ciphertext while retaining the document revision, and requires the replication pre-check and subsequent sends to succeed without clearing the browser cache. It uses the CouchDB fixture variables, an isolated Vault and profile, and a unique remote database. The raw command uses the current built plug-in by default; `E2E_OBSIDIAN_ARTIFACT_ROOT` can select an exact earlier build for regression comparison. +Run `npm run test:e2e:obsidian:focused -- couchdb-http-cache --request-api` to warm Obsidian's Internal API cache and perform the same workflow with **Use Internal API** enabled. Both modes require an explicit `no-store` read to return the rebuilt ciphertext. The default mode also retains the stale browser-cache assertion; the Internal API mode permits reads during Rebuild to refresh that cache. + `npm run test:e2e:obsidian:focused -- chunk-fetch-retry` checks delayed Chunk availability through a real CouchDB service and Obsidian. It creates a Metadata-only remote fixture, starts ordinary one-shot replication with `readChunksOnline`, and inserts the missing Chunk only after a real fetch has returned an empty result. A pass-through observer records the replicator's call times and results without substituting responses or adding waits. The fixture sets the existing minimum request interval to 500 ms to keep the real retry status observable even if finite completion expedites the final probe. The actual status bar must show zero initial requests (`🛄`) and one retry (`🔁`), and both counts must return to zero after delivery ends. On-demand replication excludes Chunk documents from the ordinary pull, so the delayed Chunk must arrive through the observed fetch and produce the exact Vault content. If finite replication was already inactive when the initial lookup began, the scenario requires a retry at least two seconds later and no physical request slot occupied during backoff. If finite replication ends during the initial lookup or the following backoff, the retry must instead be a post-completion final probe before the two-second delay would expire. This distinction is determined from the observed finite-count transitions, not an assumed ordering between replication and HTTP completion. diff --git a/test/e2e-obsidian/scripts/couchdb-http-cache.ts b/test/e2e-obsidian/scripts/couchdb-http-cache.ts index 19f7faf3..25a1ec81 100644 --- a/test/e2e-obsidian/scripts/couchdb-http-cache.ts +++ b/test/e2e-obsidian/scripts/couchdb-http-cache.ts @@ -28,6 +28,8 @@ import { createTemporaryVault } from "../runner/vault.ts"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; +const useRequestAPI = process.argv.includes("--request-api"); + const customPath = ".obsidian/snippets/http-cache.css"; const nextCustomPath = ".obsidian/snippets/http-cache-new.css"; const settings = { @@ -45,7 +47,7 @@ const settings = { syncInternalFilesInterval: 0, syncInternalFilesTargetPatterns: "^\\.http-cache(?:/|$)", watchInternalFileChanges: false, - useRequestAPI: false, + useRequestAPI, autoAcceptCompatibleTweak: false, doctorProcessedVersion: DoctorRegulation.version, deviceAndVaultName: "http-cache-source", @@ -74,10 +76,11 @@ async function main(): Promise { .storeCustomizationFiles(path,core.services.setting.getDeviceAndVaultName()); return JSON.stringify(true);`); }; - const browserRead = async (id: string, cache: RequestCache): Promise => { + const readCachedDocument = async (id: string, cache: RequestCache): Promise => { const url = `${couchDb.uri}/${encodeURIComponent(dbName)}/${encodeURIComponent(id)}`; const authorization = `Basic ${Buffer.from(`${couchDb.username}:${couchDb.password}`).toString("base64")}`; - return await evaluate(`const response=await fetch(${JSON.stringify(url)},{ + const fetchFunction = useRequestAPI ? "core.services.API.nativeFetch.bind(core.services.API)" : "fetch"; + return await evaluate(`const response=await ${fetchFunction}(${JSON.stringify(url)},{ headers:{authorization:${JSON.stringify(authorization)}},credentials:'include',cache:${JSON.stringify(cache)}}); if(!response.ok)throw new Error('Fixture read failed: '+response.status); return JSON.stringify(await response.json());`); @@ -110,7 +113,7 @@ async function main(): Promise { assert.ok(original?.path, "The encrypted ordinary Metadata fixture is missing."); const initialParameters = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); // Populate the HTTP cache as an earlier plug-in version would have done. - assert.deepEqual(await browserRead(original._id, "reload"), original); + assert.deepEqual(await readCachedDocument(original._id, "reload"), original); await evaluate(`await core.services.setting.applyPartial({encryptInternalMetadata:true},true); await core.services.control.applySettings(); @@ -128,11 +131,13 @@ async function main(): Promise { assert.notEqual(rebuiltParameters.pbkdf2salt, initialParameters.pbkdf2salt, "The Seed did not change."); assert.equal(rebuilt._rev, original._rev, "The document revision was not retained."); assert.notEqual(rebuilt.path, original.path, "The remote ciphertext did not change."); - assert.equal( - (await browserRead(original._id, "default")).path, - original.path, - "The stale browser HTTP cache fixture is no longer present." - ); + if (!useRequestAPI) { + assert.equal( + (await readCachedDocument(original._id, "default")).path, + original.path, + "The stale browser HTTP cache fixture is no longer present." + ); + } await evaluate(`await core.services.setting.applyPartial({usePluginSync:true,syncInternalFiles:true},true); await core.services.control.applySettings();return JSON.stringify(true);`); @@ -143,9 +148,12 @@ async function main(): Promise { if(opened)await opened.close();return JSON.stringify({admitted:!!opened}); }catch(error){return JSON.stringify({admitted:false,error:error.name});}`); assert.ok(admission.admitted, `Replication admission failed: ${JSON.stringify(admission)}`); + assert.equal((await readCachedDocument(original._id, "no-store")).path, rebuilt.path); await pushLocalChanges(cliBinary, session.cliEnv); await pushLocalChanges(cliBinary, session.cliEnv); - console.log("Rebuild and subsequent replication succeeded with stale ciphertext still in the HTTP cache."); + console.log( + `Rebuild and subsequent replication succeeded after warming the ${useRequestAPI ? "Internal API" : "browser"} HTTP cache.` + ); } finally { await session?.app.stop(); await vault.dispose(); diff --git a/updates.md b/updates.md index 3d01a759..8c863f30 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,8 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +- We can continue CouchDB synchronisation after rebuilding an encrypted remote when **Use Internal API** is enabled. (#1234) + - CouchDB read requests now carry cache-control headers to avoid reusing responses encrypted with the previous Security Seed. - The CLI mirror now starts correctly on Windows when the Vault path is omitted and the database directory is also used as the Vault. It checks logical Vault paths using `/` separators, correctly excluding internal database files and nested hidden directories even when the host uses Windows path separators. ## 1.0.43 From f2d364b73628fa0d641b3551b18cf8c311a52b9c Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 1 Nov 2026 08:37:14 +0000 Subject: [PATCH 2/5] Adopt published Commonlib 0.2.3 for Journal recovery --- docs/recovery.md | 2 +- package-lock.json | 16 ++++++++-------- package.json | 2 +- updates.md | 5 ++++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/recovery.md b/docs/recovery.md index 43012e16..8b0cb2ac 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -22,7 +22,7 @@ Do not switch transport, enable P2P, or run Garbage Collection as a substitute f This procedure addresses [issue #1303](https://github.com/vrtmrz/obsidian-livesync/issues/1303): an older Journal receiver can truncate a Chunk at a raw unit separator (`U+001F`) within its data. This can damage a received text file, including a plug-in's `main.js`, while the original batch remains intact in the bucket. -The receiver correction is currently unreleased. Use a build containing that correction on every participating client before recovery; versions 1.0.42 and 1.0.43 still contain the defect. Updating alone does not repair Chunks already stored locally. Resetting only the Journal received history is also insufficient because ordinary reception skips existing Chunk IDs. +The receiver correction is available in Commonlib 0.2.3 and is included in the unreleased Self-hosted LiveSync build. Use a build containing that correction on every participating client before recovery; versions 1.0.42 and 1.0.43 still contain the defect. Updating alone does not repair Chunks already stored locally. Resetting only the Journal received history is also insufficient because ordinary reception skips existing Chunk IDs. ### Preserve the intact source diff --git a/package-lock.json b/package-lock.json index bd8cd0f7..9f5b63e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.2", + "@vrtmrz/livesync-commonlib": "0.2.3", "@vrtmrz/obsidian-plugin-kit": "0.1.6", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", @@ -4121,9 +4121,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.2.tgz", - "integrity": "sha512-+nl4ntmRtUmSUuh9mOpicE0U/wpzhSPSQQXy9eg/2xdTBL58jkrQ6bQjbVKEhqUj0bUvg0ESjZm8YX/x+4zXiA==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.3.tgz", + "integrity": "sha512-n1BaoegBVGDqerZ9EcNbgd4KFtRPJ/lPp5ADsAsHTkPzSwev4SAFxxyFgFKLpTLpPv7McFao4oGBHGvRvoXWEg==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -4141,7 +4141,7 @@ "idb": "^8.0.3", "markdown-it": "^14.3.3", "minimatch": "^10.2.6", - "octagonal-wheels": "^0.1.55", + "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-idb": "^9.0.0", "pouchdb-adapter-indexeddb": "^9.0.0", @@ -8850,9 +8850,9 @@ "license": "MIT" }, "node_modules/octagonal-wheels": { - "version": "0.1.55", - "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.55.tgz", - "integrity": "sha512-mHsQv/h4Tw4ZGJCmTxXFlgeRvPuL9s47C0arD8Njqld+DKr3ppaWjVteEBP+9bWFSOpv/a1u2yY3LYNFGmuexg==", + "version": "0.1.56", + "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.56.tgz", + "integrity": "sha512-c4xQqdeEGE//ZExMJDEsqPsUHsRVHMNuxudux6scYI46eqA/54ao3WJmEZj3fbSsUhR7uCRmJ5ddd1cfRybIfg==", "license": "MIT", "dependencies": { "idb": "^8.0.3" diff --git a/package.json b/package.json index 29cd5e20..8287bb8f 100644 --- a/package.json +++ b/package.json @@ -206,7 +206,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.2", + "@vrtmrz/livesync-commonlib": "0.2.3", "@vrtmrz/obsidian-plugin-kit": "0.1.6", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", diff --git a/updates.md b/updates.md index 81a43562..9f4a22bf 100644 --- a/updates.md +++ b/updates.md @@ -13,9 +13,12 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased - Files containing a raw unit separator (`U+001F`) retain their complete contents when received through S3-compatible Object Storage. (#1303) - - The Journal receiver preserves the complete Chunk payload, including batches written by earlier versions, without changing the stored format. + - The Journal receiver in Commonlib 0.2.3 preserves the complete Chunk payload, including batches written by earlier versions, without changing the stored format. - Updating prevents further truncation during reception but does not repair existing damaged copies. Update every participating client, preserve an intact recovery source, and follow the [recovery procedure](docs/recovery.md#recover-files-truncated-during-object-storage-reception). Rebuild and Fetch leave Hidden File Sync and Customisation Sync disabled until they are deliberately initialised again. +- Files waiting for encrypted Chunks which arrive after their Metadata receive the decrypted contents once the local database is ready. + - Commonlib 0.2.3 restarts the database change feed after initialisation installs the decryption transforms. This corrects delivery of later Chunks; it does not replay earlier events. ([Commonlib #191](https://github.com/vrtmrz/livesync-commonlib/pull/191)) + - The CLI mirror now starts correctly on Windows when the Vault path is omitted and the database directory is also used as the Vault. It checks logical Vault paths using `/` separators, correctly excluding internal database files and nested hidden directories even when the host uses Windows path separators. ## 1.0.43 From e6fcbef69495a300c43265e6e9c3809f7e2ecc83 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 1 Nov 2026 10:42:27 +0000 Subject: [PATCH 3/5] Bound journal recovery fixture bucket names --- test/e2e-cli/test-journal-chunk-recovery.ts | 24 ++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/e2e-cli/test-journal-chunk-recovery.ts b/test/e2e-cli/test-journal-chunk-recovery.ts index cbddea52..345fbc88 100644 --- a/test/e2e-cli/test-journal-chunk-recovery.ts +++ b/test/e2e-cli/test-journal-chunk-recovery.ts @@ -15,6 +15,13 @@ function requireEnv(...keys: string[]): string { throw new Error(`Required env var is missing: ${keys.join(" or ")}`); } +function createJournalRecoveryBucketName(baseBucket: string, suffix: string): string { + const bucketSuffix = `journal-recovery-${suffix}`; + const maxBaseLength = 63 - bucketSuffix.length - 1; + const truncatedBase = baseBucket.slice(0, maxBaseLength).replace(/-+$/, ""); + return truncatedBase ? `${truncatedBase}-${bucketSuffix}` : bucketSuffix; +} + function createJournalFixture(): string { return [ "Journal chunk recovery fixture: UTF-8, newlines, and backslashes.\n", @@ -32,7 +39,7 @@ async function runScenario(encrypt: boolean): Promise { const minioEndpoint = requireEnv("MINIO_ENDPOINT", "minioEndpoint").replace(/\/$/, ""); const minioAccessKey = requireEnv("MINIO_ACCESS_KEY", "accessKey"); const minioSecretKey = requireEnv("MINIO_SECRET_KEY", "secretKey"); - const minioBucket = `${requireEnv("MINIO_BUCKET_NAME", "bucketName")}-journal-recovery-${suffix}`; + const minioBucket = createJournalRecoveryBucketName(requireEnv("MINIO_BUCKET_NAME", "bucketName"), suffix); const passphrase = "e2e-passphrase"; const fileContent = createJournalFixture(); const expectedBytes = Buffer.from(fileContent, "utf8"); @@ -184,6 +191,21 @@ async function runScenario(encrypt: boolean): Promise { } } +test("journal recovery bucket name construction stays within S3 limits", () => { + const ciBucket = createJournalRecoveryBucketName("livesync-test-bucket-ci", "enc0-1793522628122-62377"); + assert.ok(ciBucket.length <= 63, "The CI fixture must stay within the S3 bucket-name length limit"); + assert.match(ciBucket, /^[a-z0-9][a-z0-9-]*[a-z0-9]$/); + + const longBase = "a".repeat(63); + const unencryptedBucket = createJournalRecoveryBucketName(longBase, "enc0-1793522628122-62377"); + const encryptedBucket = createJournalRecoveryBucketName(longBase, "enc1-1793522628122-62377"); + assert.ok(unencryptedBucket.length <= 63); + assert.ok(encryptedBucket.length <= 63); + assert.match(unencryptedBucket, /^[a-z0-9][a-z0-9-]*[a-z0-9]$/); + assert.match(encryptedBucket, /^[a-z0-9][a-z0-9-]*[a-z0-9]$/); + assert.notEqual(unencryptedBucket, encryptedBucket, "Encryption scenarios should have distinct buckets"); +}); + test("journal chunk recovery through MINIO", async (t) => { await t.test("without encryption", async () => runScenario(false)); await t.test("with encryption", async () => runScenario(true)); From 74d9f80761c00abc6b410c9b58d2ae9eafda710a Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 1 Nov 2026 10:38:35 +0000 Subject: [PATCH 4/5] Preserve custom headers for no-store requests --- src/modules/services/ObsidianAPIService.ts | 64 +++++++++++++++++-- .../services/ObsidianAPIService.unit.spec.ts | 25 +++++++- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/src/modules/services/ObsidianAPIService.ts b/src/modules/services/ObsidianAPIService.ts index b319ac43..ebf4e011 100644 --- a/src/modules/services/ObsidianAPIService.ts +++ b/src/modules/services/ObsidianAPIService.ts @@ -16,6 +16,43 @@ declare module "obsidian" { } } +const cacheReuseDirectives = new Set(["max-age", "max-stale", "min-fresh", "only-if-cached"]); + +function splitHeaderDirectives(value: string): string[] { + const directives: string[] = []; + let start = 0; + let inQuotes = false; + let escaped = false; + + for (let index = 0; index < value.length; index++) { + const character = value[index]; + if (inQuotes && escaped) { + escaped = false; + continue; + } + if (inQuotes && character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + inQuotes = !inQuotes; + } else if (character === "," && !inQuotes) { + const directive = value.slice(start, index).trim(); + if (directive) directives.push(directive); + start = index + 1; + } + } + + const lastDirective = value.slice(start).trim(); + if (lastDirective) directives.push(lastDirective); + return directives; +} + +function getDirectiveName(directive: string): string { + const equalsIndex = directive.indexOf("="); + return (equalsIndex === -1 ? directive : directive.slice(0, equalsIndex)).trim().toLowerCase(); +} + export class ObsidianAPIService extends InjectableAPIService { _customHandler: ObsHttpHandler | undefined; _confirmInstance: Confirm; @@ -185,13 +222,26 @@ export class ObsidianAPIService extends InjectableAPIService key.toLowerCase() === "cache-control" + ); + const cacheControlDirectives = cacheControlKeys + .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) + .filter((directive) => { + const name = getDirectiveName(directive); + return !cacheReuseDirectives.has(name) && name !== "no-cache" && name !== "no-store"; + }); + const cacheControlKey = cacheControlKeys[0] ?? "cache-control"; + for (const key of cacheControlKeys) delete transformedHeaders[key]; + transformedHeaders[cacheControlKey] = [...cacheControlDirectives, "no-cache", "no-store"].join(", "); + + const pragmaKeys = Object.keys(transformedHeaders).filter((key) => key.toLowerCase() === "pragma"); + const pragmaDirectives = pragmaKeys + .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) + .filter((directive) => getDirectiveName(directive) !== "no-cache"); + const pragmaKey = pragmaKeys[0] ?? "pragma"; + for (const key of pragmaKeys) delete transformedHeaders[key]; + transformedHeaders[pragmaKey] = [...pragmaDirectives, "no-cache"].join(", "); } const contentType = transformedHeaders["content-type"] ?? transformedHeaders["Content-Type"] ?? "application/json"; diff --git a/src/modules/services/ObsidianAPIService.unit.spec.ts b/src/modules/services/ObsidianAPIService.unit.spec.ts index 7ad012c7..10f13081 100644 --- a/src/modules/services/ObsidianAPIService.unit.spec.ts +++ b/src/modules/services/ObsidianAPIService.unit.spec.ts @@ -94,13 +94,34 @@ describe("ObsidianAPIService.nativeFetch cache control", () => { await createService({}).nativeFetch(url, { method, cache: "no-store", headers }); - expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store"); - expect(sentHeaders().get("pragma")).toBe("no-cache"); + expect(sentHeaders().get("cache-control")).toBe("public, no-cache, no-store"); + expect(sentHeaders().get("pragma")).toBe("cache, no-cache"); expect(sentHeaders().get("authorization")).toBe("Basic fixture"); expect(headers["Cache-Control"]).toBe("public, max-age=3600"); expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method }); }); + it("preserves non-conflicting cache directives when enforcing no-store", async () => { + const headers = { + "Cache-Control": 'max-age=3600, no-transform, custom="a,b"', + "cAcHe-CoNtRoL": 'max-stale=30, min-fresh=5, only-if-cached, custom-two="max-age=1, only-if-cached"', + Pragma: 'legacy="a,b", no-cache', + "X-Custom-Header": "preserved", + }; + const originalHeaders = { ...headers }; + + await createService({}).nativeFetch(url, { cache: "no-store", headers }); + + const sent = mocks.requestUrl.mock.calls[0][0].headers ?? {}; + expect(sent["Cache-Control"]).toBe( + 'no-transform, custom="a,b", custom-two="max-age=1, only-if-cached", no-cache, no-store' + ); + expect(sent["cAcHe-CoNtRoL"]).toBeUndefined(); + expect(sent.Pragma).toBe('legacy="a,b", no-cache'); + expect(sent["X-Custom-Header"]).toBe("preserved"); + expect(headers).toEqual(originalHeaders); + }); + it("uses the Request cache mode when the options do not override it", async () => { const request = new Request(url, { method: "HEAD", From fb196cac3a35570490babf4825617f98cc9aff02 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 1 Nov 2026 10:51:46 +0000 Subject: [PATCH 5/5] Run CLI database continuity regression in standard CI --- test/README.md | 32 ++++-- test/e2e-cli/ci-tasks.json | 3 +- test/e2e-cli/helpers/docker.ts | 54 ++++++++++ test/e2e-cli/tasks.json | 3 +- test/e2e-cli/test-ci-task-contract.ts | 18 ++++ .../e2e-cli/test-fetch-database-continuity.ts | 101 +++++++++++++++--- 6 files changed, 185 insertions(+), 26 deletions(-) create mode 100644 test/e2e-cli/test-ci-task-contract.ts diff --git a/test/README.md b/test/README.md index 12d1c5eb..4930753f 100644 --- a/test/README.md +++ b/test/README.md @@ -79,21 +79,37 @@ The expected result is `PASS=7 FAIL=0`. These mirror suites use temporary local ## CLI Fetch database continuity -After building the CLI, run `node test/e2e-cli/run-task.ts test:fetch-database-continuity` -with Node.js 26 and a disposable S3-compatible service. Set +The standard CLI CI suite and direct task runner execute +`test:fetch-database-continuity` against a dedicated RustFS container. Each +scenario creates its own bucket. The test binds a uniquely named container to a +dynamic loopback port, then stops only that container. Redacted evidence is +written under the operating system's temporary directory and is not uploaded as +an artefact. + +For a local run, build the CLI and run +`node test/e2e-cli/run-task.ts test:fetch-database-continuity` with Node.js 26 +and Docker available. The managed container uses `accessKey` and `secretKey` +from the CLI `.test.env` file when present, and defaults to `minioadmin` for +both. No endpoint, bucket, or evidence-directory variables are needed. Set +`LIVESYNC_ISSUE1304_EVIDENCE_DIR` to choose another evidence directory outside +the worktree. + +To use an external S3-compatible fixture without Docker, set all four `LIVESYNC_ISSUE1304_MINIO_ENDPOINT`, `LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY`, `LIVESYNC_ISSUE1304_MINIO_SECRET_KEY`, and `LIVESYNC_ISSUE1304_MINIO_BUCKET`. -The bucket must be fresh and empty for every run. Set -`LIVESYNC_ISSUE1304_EVIDENCE_DIR` to an evidence directory outside the worktree. -This test writes remote data; never point it at a production bucket. +The bucket must be fresh and empty for every run. Partial external fixture +configuration is rejected. This test writes remote data; never point it at a +production bucket. The test seeds an encrypted, path-obfuscated file with a second-generation revision, imports one Setup URI on two clients, fetches on A without `--write-settings`, and restarts A while B remains online. It checks the saved database suffix, file bytes, original revision, and absence of a duplicate root. -Set `LIVESYNC_ISSUE1304_POSITIVE_CONTROL=1` and -`LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET` to a second fresh bucket to exercise -the explicit `--write-settings` variant as well. +Managed runs execute the explicit `--write-settings` control automatically in +a separate scenario bucket. For an external fixture, set +`LIVESYNC_ISSUE1304_POSITIVE_CONTROL=1` and +`LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET` to a second fresh bucket to run +that control. For historical reproduction against the unmodified implementation, set `LIVESYNC_ISSUE1304_VERIFY_LEGACY_FAILURE=1`. This preserves the original diff --git a/test/e2e-cli/ci-tasks.json b/test/e2e-cli/ci-tasks.json index 08450098..078d3740 100644 --- a/test/e2e-cli/ci-tasks.json +++ b/test/e2e-cli/ci-tasks.json @@ -14,5 +14,6 @@ "test:e2e-matrix:couchdb-enc0", "test:e2e-matrix:couchdb-enc1", "test:e2e-matrix:minio-enc0", - "test:e2e-matrix:minio-enc1" + "test:e2e-matrix:minio-enc1", + "test:fetch-database-continuity" ] diff --git a/test/e2e-cli/helpers/docker.ts b/test/e2e-cli/helpers/docker.ts index 018dc226..961ba957 100644 --- a/test/e2e-cli/helpers/docker.ts +++ b/test/e2e-cli/helpers/docker.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { writeSync } from "node:fs"; import { spawnProcess } from "./process.ts"; /** @@ -577,6 +578,59 @@ export async function startMinio( await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket); } +export type IsolatedMinioFixture = { + endpoint: string; + accessKey: string; + secretKey: string; + stop: () => Promise; +}; + +/** Start an S3 fixture owned by this caller without touching the shared test container. */ +export async function startIsolatedMinio(accessKey: string, secretKey: string): Promise { + const container = `livesync-cli-continuity-${randomUUID()}`; + console.log(`[INFO] starting isolated S3 test container: ${container}`); + let tracked = false; + const stop = async () => { + await stopAndRemoveContainer(container); + if (tracked) untrackContainer(container); + }; + + try { + await dockerOrFail( + "run", + "-d", + "--name", + container, + "-p", + "127.0.0.1::9000", + "-e", + `RUSTFS_ACCESS_KEY=${accessKey}`, + "-e", + `RUSTFS_SECRET_KEY=${secretKey}`, + "-e", + "RUSTFS_CONSOLE_ENABLE=true", + "-e", + "RUSTFS_CORS_ALLOWED_ORIGINS=*", + S3_IMAGE, + "/data" + ); + trackContainer(container); + tracked = true; + + const mapping = await dockerOrFail("port", container, "9000/tcp"); + const port = Number(mapping.trim().match(/:(\d+)$/)?.[1]); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error(`Could not determine the isolated S3 fixture port from: ${mapping.trim()}`); + } + const endpoint = `http://127.0.0.1:${port}`; + await fetchRetry(endpoint, { method: "GET" }, 30, 1000, [403, 404]); + return { endpoint, accessKey, secretKey, stop }; + } catch (error) { + await stop(); + throw error; + } +} + // --------------------------------------------------------------------------- // P2P relay (strfry) // --------------------------------------------------------------------------- diff --git a/test/e2e-cli/tasks.json b/test/e2e-cli/tasks.json index f20da103..a0e5f460 100644 --- a/test/e2e-cli/tasks.json +++ b/test/e2e-cli/tasks.json @@ -1,7 +1,8 @@ { "test:helpers": { "test": [ - "helpers/process.test.ts" + "helpers/process.test.ts", + "test-ci-task-contract.ts" ] }, "test": { diff --git a/test/e2e-cli/test-ci-task-contract.ts b/test/e2e-cli/test-ci-task-contract.ts new file mode 100644 index 00000000..22a94565 --- /dev/null +++ b/test/e2e-cli/test-ci-task-contract.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { test } from "node:test"; + +test("fetch database continuity is exercised by standard CLI CI", async () => { + const ciTasks = JSON.parse(await readFile(new URL("./ci-tasks.json", import.meta.url), "utf8")) as string[]; + const tasks = JSON.parse(await readFile(new URL("./tasks.json", import.meta.url), "utf8")) as Record< + string, + { test?: string[] } + >; + const workflow = await readFile(new URL("../../.github/workflows/cli-node-tests.yml", import.meta.url), "utf8"); + + assert(ciTasks.includes("test:helpers"), "the standard CI suite must run the task registration contract"); + assert(tasks["test:helpers"]?.test?.includes("test-ci-task-contract.ts")); + assert(ciTasks.includes("test:fetch-database-continuity"), "the continuity task must be in the standard CI suite"); + assert.deepEqual(tasks["test:fetch-database-continuity"]?.test, ["test-fetch-database-continuity.ts"]); + assert.match(workflow, /test\/e2e-cli\/ci-tasks\.json/, "the automatic CI matrix must read the authoritative task list"); +}); diff --git a/test/e2e-cli/test-fetch-database-continuity.ts b/test/e2e-cli/test-fetch-database-continuity.ts index bf66c0aa..8f38548f 100644 --- a/test/e2e-cli/test-fetch-database-continuity.ts +++ b/test/e2e-cli/test-fetch-database-continuity.ts @@ -1,9 +1,12 @@ import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { test } from "node:test"; +import { after, before, test } from "node:test"; +import { CreateBucketCommand, S3Client } from "@aws-sdk/client-s3"; import { CLI_DIR, CLI_DIST } from "./helpers/cli.ts"; +import { startIsolatedMinio, type IsolatedMinioFixture } from "./helpers/docker.ts"; import { applyRemoteSyncSettings, generateSetupUriFromSettings, initSettingsFile } from "./helpers/settings.ts"; import { TempDir } from "./helpers/temp.ts"; import { spawnProcess, type RunningProcess } from "./helpers/process.ts"; @@ -24,20 +27,89 @@ const RESOLUTION_MARKER = "Resolving [same]: Conflicted revision has been delete const INITIAL_REPLICATION_COMPLETE = "[Daemon] Initial replication complete"; const INITIAL_REPLICATION_FAILED = "[Daemon] Initial replication failed"; const VERIFY_LEGACY_FAILURE = process.env["LIVESYNC_ISSUE1304_VERIFY_LEGACY_FAILURE"] === "1"; +const EXPLICIT_FIXTURE_VARIABLES = [ + "LIVESYNC_ISSUE1304_MINIO_ENDPOINT", + "LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY", + "LIVESYNC_ISSUE1304_MINIO_SECRET_KEY", + "LIVESYNC_ISSUE1304_MINIO_BUCKET", +]; +const EXPLICIT_FIXTURE = EXPLICIT_FIXTURE_VARIABLES.some((name) => process.env[name]?.trim()); -function requiredEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`Required environment variable is missing: ${name}`); - return value; +if (EXPLICIT_FIXTURE && EXPLICIT_FIXTURE_VARIABLES.some((name) => !process.env[name]?.trim())) { + throw new Error(`External fixture mode requires all of: ${EXPLICIT_FIXTURE_VARIABLES.join(", ")}`); } -function remoteSettings(bucketOverride?: string): RemoteSettings { +let managedFixture: IsolatedMinioFixture | undefined; +if (!EXPLICIT_FIXTURE) { + before(async () => { + managedFixture = await startIsolatedMinio( + process.env["accessKey"]?.trim() ?? "minioadmin", + process.env["secretKey"]?.trim() ?? "minioadmin" + ); + }); + after(async () => { + await managedFixture?.stop(); + }); +} + +function requiredEnv(...names: string[]): string { + for (const name of names) { + const value = process.env[name]?.trim(); + if (value) return value; + } + throw new Error(`Required environment variable is missing: ${names.join(" or ")}`); +} + +function evidenceDirectory(): string { + return resolve( + process.env["LIVESYNC_ISSUE1304_EVIDENCE_DIR"]?.trim() || join(tmpdir(), "livesync-cli-issue1304-evidence") + ); +} + +function remoteSettings(): RemoteSettings { return { endpoint: requiredEnv("LIVESYNC_ISSUE1304_MINIO_ENDPOINT").replace(/\/$/, ""), accessKey: requiredEnv("LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY"), secretKey: requiredEnv("LIVESYNC_ISSUE1304_MINIO_SECRET_KEY"), - bucket: bucketOverride ?? requiredEnv("LIVESYNC_ISSUE1304_MINIO_BUCKET"), - evidenceDir: resolve(requiredEnv("LIVESYNC_ISSUE1304_EVIDENCE_DIR")), + bucket: requiredEnv("LIVESYNC_ISSUE1304_MINIO_BUCKET"), + evidenceDir: evidenceDirectory(), + }; +} + +async function createManagedBucket(remote: Pick, mode: ScenarioMode): Promise { + const shortMode = mode === "reproduction" ? "repro" : "control"; + const bucket = `livesync-ci-1304-${shortMode}-${randomUUID().replaceAll("-", "")}`; + const client = new S3Client({ + endpoint: remote.endpoint, + region: "us-east-1", + forcePathStyle: true, + credentials: { accessKeyId: remote.accessKey, secretAccessKey: remote.secretKey }, + }); + try { + await client.send(new CreateBucketCommand({ Bucket: bucket })); + } finally { + client.destroy(); + } + return bucket; +} + +async function scenarioRemoteSettings(mode: ScenarioMode): Promise { + if (EXPLICIT_FIXTURE) { + const remote = remoteSettings(); + if (mode === "write-settings-control") remote.bucket = requiredEnv("LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET"); + return remote; + } + + if (!managedFixture) throw new Error("The managed S3 fixture was not started"); + const remote = { + endpoint: managedFixture.endpoint, + accessKey: managedFixture.accessKey, + secretKey: managedFixture.secretKey, + }; + return { + ...remote, + bucket: await createManagedBucket(remote, mode), + evidenceDir: evidenceDirectory(), }; } @@ -265,8 +337,7 @@ async function settingsSummary(settingsPath: string): Promise { - const remote = remoteSettings(bucket); +async function runScenario(mode: ScenarioMode, remote: RemoteSettings): Promise { const evidenceRoot = remote.evidenceDir; const evidenceName = `${mode}-${Date.now()}-${process.pid}`; const evidenceDir = join(evidenceRoot, evidenceName); @@ -556,13 +627,11 @@ async function runScenario(mode: ScenarioMode, bucket: string): Promise { } test("CLI fetch: files remain after same-content duplicate root resolution", async () => { - const remote = remoteSettings(); - await runScenario("reproduction", remote.bucket); + await runScenario("reproduction", await scenarioRemoteSettings("reproduction")); }); -if (process.env["LIVESYNC_ISSUE1304_POSITIVE_CONTROL"] === "1") { +if (!EXPLICIT_FIXTURE || process.env["LIVESYNC_ISSUE1304_POSITIVE_CONTROL"] === "1") { test("CLI fetch: --write-settings keeps the selected database across daemon startup", async () => { - const controlBucket = requiredEnv("LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET"); - await runScenario("write-settings-control", controlBucket); + await runScenario("write-settings-control", await scenarioRemoteSettings("write-settings-control")); }); }