diff --git a/package-lock.json b/package-lock.json index 084852b1..07748b78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.35", + "@vrtmrz/livesync-commonlib": "0.1.36", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4567,9 +4567,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.35", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.35.tgz", - "integrity": "sha512-8aM2dPtspEeQ1NyG0uPU0yeuM9LtTS6EquM032sfMMl11t+tLR/UEuHIvXI3XtpAqlGUpU02ZZI8waZvQ8i4SQ==", + "version": "0.1.36", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.36.tgz", + "integrity": "sha512-YEVEM3Uq0YnqMP70hOFhmfkyS2X3Ptp/t43EH/6yc/NirQhpTYfYV6hLEuXok5LEw2gkWV7G6wMiYGlEjFCTig==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 4e52c521..35c643db 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "test:e2e:obsidian:p2p-pane": "tsx test/e2e-obsidian/scripts/p2p-pane.ts", "test:e2e:obsidian:vault-reflection": "tsx test/e2e-obsidian/scripts/vault-reflection.ts", "test:e2e:obsidian:couchdb-upload": "tsx test/e2e-obsidian/scripts/couchdb-upload.ts", + "test:e2e:obsidian:couchdb-http-cache": "tsx test/e2e-obsidian/scripts/couchdb-http-cache.ts", "test:e2e:obsidian:chunk-fetch-retry": "tsx test/e2e-obsidian/scripts/chunk-fetch-retry.ts", "test:e2e:obsidian:tweak-compatibility": "tsx test/e2e-obsidian/scripts/tweak-compatibility.ts", "test:e2e:obsidian:couchdb-manual-setup-workflow": "tsx test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts", @@ -191,7 +192,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.35", + "@vrtmrz/livesync-commonlib": "0.1.36", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index f389ab5c..daf382c5 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -78,6 +78,7 @@ npm run test:e2e:obsidian:focused -- two-vault-sync npm run test:e2e:obsidian:focused -- stale-file-restart npm run test:e2e:obsidian:focused -- folder-batch npm run test:e2e:obsidian:focused -- security-seed-reconnect +npm run test:e2e:obsidian:focused -- couchdb-http-cache ``` The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. @@ -139,6 +140,8 @@ The Harness also measures ID generation with fixed in-memory data on desktop and The same workflow checks the two remote-activity status boundaries. It first holds a real CouchDB request at the selected fetch implementation and confirms that `🌐N` is visible while `📲` is absent. It then holds the real one-shot replication immediately before its replicator call, confirms that `📲` is visible while no physical request is active, releases it, and requires the finite and bounded activity counts to return to zero, the request and response counts to balance, and both indicators to disappear. Finally, it creates a remote-only chunk, holds the real on-demand fetch immediately before its remote call, makes the same logical active and idle assertions, and verifies that the fetched chunk is written into the local database. These gates make the active states deterministic without replacing the remote request or operation. +`test:e2e:obsidian:couchdb-http-cache` checks replication after rebuilding an encrypted CouchDB remote while an earlier HTTP response remains cached. It enables internal Metadata encryption with Hidden File Sync and Customisation Sync, verifies that the Rebuild replaces the Security Seed and ciphertext while retaining the document revision, and requires the replication pre-check and subsequent sends to succeed without clearing the browser cache. It uses the CouchDB fixture variables, an isolated Vault and profile, and a unique remote database. The raw command uses the current built plug-in by default; `E2E_OBSIDIAN_ARTIFACT_ROOT` can select an exact earlier build for regression comparison. + `npm run test:e2e:obsidian:focused -- chunk-fetch-retry` checks delayed Chunk availability through a real CouchDB service and Obsidian. It creates a Metadata-only remote fixture, starts ordinary one-shot replication with `readChunksOnline`, and inserts the missing Chunk only after a real fetch has returned an empty result. A pass-through observer records the replicator's call times and results without substituting responses or adding waits. The fixture sets the existing minimum request interval to 500 ms to keep the real retry status observable even if finite completion expedites the final probe. The actual status bar must show zero initial requests (`🛄`) and one retry (`🔁`), and both counts must return to zero after delivery ends. On-demand replication excludes Chunk documents from the ordinary pull, so the delayed Chunk must arrive through the observed fetch and produce the exact Vault content. If finite replication was already inactive when the initial lookup began, the scenario requires a retry at least two seconds later and no physical request slot occupied during backoff. If finite replication ends during the initial lookup or the following backoff, the retry must instead be a post-completion final probe before the two-second delay would expire. This distinction is determined from the observed finite-count transitions, not an assumed ordering between replication and HTTP completion. diff --git a/test/e2e-obsidian/scripts/couchdb-http-cache.ts b/test/e2e-obsidian/scripts/couchdb-http-cache.ts new file mode 100644 index 00000000..19f7faf3 --- /dev/null +++ b/test/e2e-obsidian/scripts/couchdb-http-cache.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc"; +import { DOCID_SYNC_PARAMETERS } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + fetchAllCouchDbDocs, + fetchCouchDbDocument, + loadCouchDbConfig, + makeUniqueDatabaseName, + type CouchDbDocument, +} from "../runner/couchdb.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + configureCouchDb, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + pushLocalChanges, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +const customPath = ".obsidian/snippets/http-cache.css"; +const nextCustomPath = ".obsidian/snippets/http-cache-new.css"; +const settings = { + encrypt: true, + passphrase: "couchdb-http-cache-fixture-secret", + idDerivationVersion: 0, + usePathObfuscation: true, + encryptInternalMetadata: false, + usePluginSync: true, + usePluginSyncV2: true, + autoSweepPlugins: false, + autoSweepPluginsPeriodic: false, + syncInternalFiles: true, + syncInternalFilesBeforeReplication: false, + syncInternalFilesInterval: 0, + syncInternalFilesTargetPatterns: "^\\.http-cache(?:/|$)", + watchInternalFileChanges: false, + useRequestAPI: false, + autoAcceptCompatibleTweak: false, + doctorProcessedVersion: DoctorRegulation.version, + deviceAndVaultName: "http-cache-source", +}; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const couchDb = await loadCouchDbConfig(); + await assertCouchDbReachable(couchDb); + const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "http-cache"); + const connection = { ...couchDb, dbName }; + const vault = await createTemporaryVault(); + let session: ObsidianLiveSyncSession | undefined; + const evaluate = async (body: string): Promise => { + if (!session) throw new Error("No active Obsidian session."); + return await evalObsidianJson( + cliBinary, + `(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;${body}})()`, + session.cliEnv + ); + }; + const store = async (paths: string[]): Promise => { + await evaluate(`await core.getAddOn('HiddenFileSync').scanAllStorageChanges(true); + for(const path of ${JSON.stringify(paths)}) await core.getAddOn('ConfigSync') + .storeCustomizationFiles(path,core.services.setting.getDeviceAndVaultName()); + return JSON.stringify(true);`); + }; + const browserRead = async (id: string, cache: RequestCache): Promise => { + const url = `${couchDb.uri}/${encodeURIComponent(dbName)}/${encodeURIComponent(id)}`; + const authorization = `Basic ${Buffer.from(`${couchDb.username}:${couchDb.password}`).toString("base64")}`; + return await evaluate(`const response=await fetch(${JSON.stringify(url)},{ + headers:{authorization:${JSON.stringify(authorization)}},credentials:'include',cache:${JSON.stringify(cache)}}); + if(!response.ok)throw new Error('Fixture read failed: '+response.status); + return JSON.stringify(await response.json());`); + }; + + try { + await createCouchDbDatabase(couchDb, dbName); + for (const path of ["example.md", ".http-cache/example.json", customPath]) { + await mkdir(dirname(join(vault.path, path)), { recursive: true }); + await writeFile(join(vault.path, path), "/* HTTP cache fixture */\n"); + } + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + artifactRoot: resolve(process.env.E2E_OBSIDIAN_ARTIFACT_ROOT ?? "."), + pluginData: createE2eCouchDbPluginData(connection, settings), + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + }); + await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); + await configureCouchDb(cliBinary, session.cliEnv, connection, settings); + await evaluate(`core.services.setting.setDeviceAndVaultName('http-cache-source'); + await core.services.setting.saveSettingData();return JSON.stringify(true);`); + await prepareRemote(cliBinary, session.cliEnv); + await store([customPath]); + await pushLocalChanges(cliBinary, session.cliEnv); + + const initialRows = await fetchAllCouchDbDocs(couchDb, dbName); + const original = initialRows.rows.find((row) => row.id.startsWith("f:"))?.doc; + assert.ok(original?.path, "The encrypted ordinary Metadata fixture is missing."); + const initialParameters = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); + // Populate the HTTP cache as an earlier plug-in version would have done. + assert.deepEqual(await browserRead(original._id, "reload"), original); + + await evaluate(`await core.services.setting.applyPartial({encryptInternalMetadata:true},true); + await core.services.control.applySettings(); + await core.services.replicator.getActiveReplicator() + .setPreferredRemoteTweakSettings(core.services.setting.currentSettings()); + return JSON.stringify(true);`); + await writeFile(join(vault.path, nextCustomPath), "/* Protected internal Metadata */\n"); + await store([customPath, nextCustomPath]); + await pushLocalChanges(cliBinary, session.cliEnv); + + // Maintenance Send rebuilds the remote while retaining local document revisions. + await evaluate(`await core.rebuilder.performRemoteRebuild();return JSON.stringify(true);`); + const rebuilt = await fetchCouchDbDocument(couchDb, dbName, original._id); + const rebuiltParameters = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); + assert.notEqual(rebuiltParameters.pbkdf2salt, initialParameters.pbkdf2salt, "The Seed did not change."); + assert.equal(rebuilt._rev, original._rev, "The document revision was not retained."); + assert.notEqual(rebuilt.path, original.path, "The remote ciphertext did not change."); + assert.equal( + (await browserRead(original._id, "default")).path, + original.path, + "The stale browser HTTP cache fixture is no longer present." + ); + + await evaluate(`await core.services.setting.applyPartial({usePluginSync:true,syncInternalFiles:true},true); + await core.services.control.applySettings();return JSON.stringify(true);`); + await store([customPath, nextCustomPath]); + const admission = await evaluate<{ admitted: boolean; error?: string }>(`try { + const opened=await core.services.replicator.getActiveReplicator() + .checkReplicationConnectivity(core.services.setting.currentSettings(),false,false,true); + if(opened)await opened.close();return JSON.stringify({admitted:!!opened}); + }catch(error){return JSON.stringify({admitted:false,error:error.name});}`); + assert.ok(admission.admitted, `Replication admission failed: ${JSON.stringify(admission)}`); + await pushLocalChanges(cliBinary, session.cliEnv); + await pushLocalChanges(cliBinary, session.cliEnv); + console.log("Rebuild and subsequent replication succeeded with stale ciphertext still in the HTTP cache."); + } finally { + await session?.app.stop(); + await vault.dispose(); + await deleteCouchDbDatabase(couchDb, dbName); + } +} + +await main(); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index f9caa36f..84e3b733 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -17,6 +17,7 @@ const focusedScenarios = new Set([ "p2p-pane", "vault-reflection", "couchdb-upload", + "couchdb-http-cache", "chunk-fetch-retry", "couchdb-manual-setup-workflow", "cli-to-obsidian-sync", diff --git a/updates.md b/updates.md index 7db9f4f8..2dc166a2 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation + +#### Fixed + +- We can continue synchronising after rebuilding an encrypted CouchDB remote, including when Hidden File Sync and Customisation Sync are enabled. (#1234) + - This plug-in reads the rebuilt data without reusing HTTP responses encrypted with the previous Security Seed. + ### Customisation Sync #### Fixed