Fix concurrent Customisation Sync catalogue updates

test/e2e-obsidian/README.md

@@ -250,6 +250,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms
 
 `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy.
 
+`test:e2e:obsidian:customisation-catalogue-reload` verifies that the first Reload lists all three files of a sample plug-in and that Apply restores their contents. It also checks a second enumeration and Apply in one temporary Vault with a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_CUSTOMISATION_CATALOGUE_OUTPUT` to choose the JSON report path.
+
 The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow.
 
 `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`.

test/e2e-obsidian/scripts/customisation-catalogue-reload.ts

@@ -0,0 +1,220 @@
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { fileURLToPath } from "node:url";
+import { evalObsidianJson } from "../runner/cli.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    createE2eCouchDbPluginData,
+    createE2eObsidianDeviceLocalState,
+    waitForLiveSyncCoreReady,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+const pluginName = "livesync-catalogue-example";
+const sourceTerm = "catalogue-source";
+const pluginRoot = `.obsidian/plugins/${pluginName}`;
+const pluginFiles = {
+    [`${pluginRoot}/main.js`]: "module.exports = class ExamplePlugin {};\n",
+    [`${pluginRoot}/manifest.json`]: JSON.stringify({
+        id: pluginName,
+        name: "Catalogue Example",
+        version: "1.0.0",
+        minAppVersion: "1.0.0",
+        description: "An isolated test fixture",
+        author: "Self-hosted LiveSync",
+    }),
+    [`${pluginRoot}/styles.css`]: ".livesync-catalogue-example { color: red; }\n",
+};
+const allowList = "\\.(?:md|png|pdf)$";
+const coreCode = "const core=app.plugins.plugins['obsidian-livesync'].core;";
+const configCode = `${coreCode}const addOn=core.getAddOn('ConfigSync');`;
+const settleCode = `
+    const settle=async()=>{
+        await new Promise(resolve=>setTimeout(resolve,100));
+        for(let attempt=0;attempt<200;attempt++){
+            if(addOn.updatingV2Count===0)return;
+            await new Promise(resolve=>setTimeout(resolve,25));
+        }
+        throw new Error('Customisation Sync did not settle');
+    };
+`;
+const sourceRowCode = `
+    const row=addOn.pluginList.find(entry=>entry.category==='PLUGIN_MAIN'&&entry.name===${JSON.stringify(pluginName)}&&entry.term===${JSON.stringify(sourceTerm)});
+    if(!row)throw new Error('The source catalogue row is missing');
+`;
+
+type Observation = { name: string; expected: unknown; actual: unknown; passed: boolean };
+
+async function writeFixtures(vaultPath: string, files: Record<string, string>) {
+    for (const [path, content] of Object.entries(files)) {
+        await mkdir(dirname(join(vaultPath, path)), { recursive: true });
+        await writeFile(join(vaultPath, path), content);
+    }
+}
+
+async function removeFixtures(vaultPath: string, files: Record<string, string>) {
+    for (const path of Object.keys(files)) await rm(join(vaultPath, path), { force: true });
+}
+
+async function matchingFiles(vaultPath: string, files: Record<string, string>) {
+    const matches: string[] = [];
+    for (const [path, content] of Object.entries(files)) {
+        try {
+            if ((await readFile(join(vaultPath, path), "utf8")) === content) matches.push(path);
+        } catch (error) {
+            if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error;
+        }
+    }
+    return matches.sort();
+}
+
+async function main() {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error("The Obsidian CLI is unavailable");
+    const cliBinary = cli.binary;
+    const output =
+        process.env.E2E_CUSTOMISATION_CATALOGUE_OUTPUT ?? join(tmpdir(), "livesync-customisation-catalogue.json");
+    const observations: Observation[] = [];
+    const report: Record<string, unknown> = {
+        revision: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(),
+        sourceDiffSha256: createHash("sha256")
+            .update(new Uint8Array(execFileSync("git", ["diff", "--binary", "HEAD"])))
+            .digest("hex"),
+        scenarioSha256: createHash("sha256")
+            .update(new Uint8Array(await readFile(fileURLToPath(import.meta.url))))
+            .digest("hex"),
+        commonlibEntrySha256: createHash("sha256")
+            .update(
+                new Uint8Array(
+                    await readFile(
+                        "node_modules/@vrtmrz/livesync-commonlib/dist/managers/EntryManager/EntryManagerImpls.js"
+                    )
+                )
+            )
+            .digest("hex"),
+        bundleSha256: createHash("sha256")
+            .update(new Uint8Array(await readFile("main.js")))
+            .digest("hex"),
+        commonlibVersion: JSON.parse(await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8"))
+            .version,
+        observations,
+    };
+    const vault = await createTemporaryVault("livesync-customisation-catalogue-");
+    let session: ObsidianLiveSyncSession | undefined;
+    const record = (name: string, actual: unknown, expected: unknown) => {
+        const observation = { name, expected, actual, passed: isDeepStrictEqual(actual, expected) };
+        observations.push(observation);
+        console.log(JSON.stringify(observation));
+    };
+    const evaluate = async <T>(code: string): Promise<T> => {
+        if (!session) throw new Error("The isolated Obsidian session has not started");
+        return await evalObsidianJson<T>(cliBinary, `(async()=>{${code}})()`, session.cliEnv, 60_000);
+    };
+    try {
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary,
+            vault,
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+            pluginData: createE2eCouchDbPluginData(
+                { uri: "http://127.0.0.1:9", username: "", password: "", dbName: "catalogue-local-only" },
+                { periodicReplication: false, syncOnFileOpen: false, useOnlyLocalChunk: true }
+            ),
+        });
+        report.pluginVersion = session.readiness.pluginVersion;
+        await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
+        report.runtime = await evaluate(`${coreCode}
+            core.services.setting.setDeviceAndVaultName('catalogue-local');
+            Object.assign(core.settings,{
+                isConfigured:true,usePluginSync:true,usePluginSyncV2:true,syncInternalFiles:false,
+                autoSweepPlugins:false,autoSweepPluginsPeriodic:false,watchInternalFileChanges:false,
+                syncOnlyRegEx:${JSON.stringify(allowList)},syncIgnoreRegEx:''
+            });
+            return JSON.stringify({userAgent:navigator.userAgent,databaseReady:core.localDatabase.isReady});
+        `);
+
+        await writeFixtures(vault.path, pluginFiles);
+        await evaluate(`${configCode}${settleCode}
+            for(const path of ${JSON.stringify(Object.keys(pluginFiles))}){
+                const stored=await addOn.storeCustomisationFileV2(path,${JSON.stringify(sourceTerm)});
+                if(stored===false)throw new Error('Could not seed a Customisation Sync document');
+            }
+            await settle();
+            return JSON.stringify({seeded:true});
+        `);
+        const catalogue = await evaluate<{ files: string[]; documents: number }>(`${configCode}${settleCode}
+            await addOn.reloadPluginList(false);
+            await settle();
+            ${sourceRowCode}
+            let documents=0;
+            for await(const doc of core.localDatabase.findEntries('ix:','ix;')){
+                if(doc.path.startsWith(${JSON.stringify(`ix:${sourceTerm}/PLUGIN_MAIN/${pluginName}%`)})&&!doc.deleted&&!doc._deleted)documents++;
+            }
+            return JSON.stringify({files:row.files.map(file=>file.filename).sort(),documents});
+        `);
+        record("#1271: all source documents remain in the database", catalogue.documents, 3);
+        record(
+            "#1271: one reload lists all files",
+            catalogue.files,
+            Object.keys(pluginFiles)
+                .map((path) => path.substring(".obsidian/".length))
+                .sort()
+        );
+        await removeFixtures(vault.path, pluginFiles);
+        const firstApply = await evaluate(
+            `${configCode}${sourceRowCode}return JSON.stringify({applied:await addOn.applyDataV2(row)});`
+        );
+        report.firstApply = firstApply;
+        record(
+            "#1271: first Apply restores all file contents",
+            await matchingFiles(vault.path, pluginFiles),
+            Object.keys(pluginFiles).sort()
+        );
+
+        const secondPass = await evaluate<string[]>(`${configCode}${settleCode}
+            await addOn.updatePluginList(false);await settle();${sourceRowCode}
+            return JSON.stringify(row.files.map(file=>file.filename).sort());
+        `);
+        record(
+            "#1271 control: a second pass lists all files",
+            secondPass,
+            Object.keys(pluginFiles)
+                .map((path) => path.substring(".obsidian/".length))
+                .sort()
+        );
+        await removeFixtures(vault.path, pluginFiles);
+        await evaluate(`${configCode}${sourceRowCode}return JSON.stringify({applied:await addOn.applyDataV2(row)});`);
+        record(
+            "#1271 control: Apply after a second pass restores all contents",
+            await matchingFiles(vault.path, pluginFiles),
+            Object.keys(pluginFiles).sort()
+        );
+    } catch (error) {
+        report.runtimeError = error instanceof Error ? error.stack : String(error);
+        if (session) report.processOutput = JSON.stringify(session.app.output()).slice(-4_000);
+        throw error;
+    } finally {
+        try {
+            await writeFile(output, JSON.stringify(report, null, 2) + "\n");
+            console.log(`Verification report: ${output}`);
+        } finally {
+            try {
+                if (session) await session.app.stop();
+            } finally {
+                await vault.dispose();
+            }
+        }
+    }
+    if (observations.some((observation) => !observation.passed)) process.exitCode = 1;
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exitCode = 1;
+});

test/e2e-obsidian/scripts/run-focused.ts

@@ -40,6 +40,7 @@ const focusedScenarios = new Set([
     "security-seed-reconnect",
     "hidden-file-snippet-sync",
     "customisation-sync",
+    "customisation-catalogue-reload",
     "received-change-readiness",
     "internal-metadata-doctor",
     "setting-markdown-export",

test/fixtures/optionalFileSyncDatabase.ts

@@ -0,0 +1,78 @@
+import { randomUUID } from "node:crypto";
+import PouchDB from "pouchdb-core";
+import MemoryAdapter from "pouchdb-adapter-memory";
+import { vi } from "vitest";
+import {
+    DEFAULT_SETTINGS,
+    type EntryDoc,
+    type SavingEntry,
+    type FilePathWithPrefix,
+    type DocumentID,
+} from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { createTextBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
+import { createLiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context";
+import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB";
+
+PouchDB.plugin(MemoryAdapter);
+
+export function optionalFileEntry(path: string, content: string): SavingEntry {
+    return {
+        _id: path as DocumentID,
+        path: path as FilePathWithPrefix,
+        ctime: 1_000,
+        mtime: 2_000,
+        size: content.length,
+        type: "plain",
+        datatype: "plain",
+        data: createTextBlob(content),
+        children: [],
+        eden: {},
+    };
+}
+
+export async function createOptionalFileSyncDatabase() {
+    const name = `optional-file-sync-${randomUUID()}`;
+    const database = new PouchDB<EntryDoc>(name, { adapter: "memory" });
+    const settings = {
+        ...DEFAULT_SETTINGS,
+        syncInternalFiles: true,
+        useOnlyLocalChunk: true,
+        usePluginSync: true,
+        usePluginSyncV2: true,
+        pluginSyncExtendedSetting: {},
+    };
+    const services = {
+        API: { addLog: vi.fn(), getSystemConfigDir: () => ".obsidian" },
+        setting: { currentSettings: () => settings, getDeviceAndVaultName: () => "device" },
+        path: {
+            path2id: async (path: string) => path,
+            id2path: (id: string, entry?: { path?: string }) => entry?.path ?? id,
+            getPath: (entry: { path: FilePathWithPrefix }) => entry.path,
+        },
+        context: { events: createLiveSyncEventHub() },
+        database: { createPouchDBInstance: () => database },
+        databaseEvents: {
+            onDatabaseInitialisation: async () => true,
+            onDatabaseHasReady: async () => true,
+            onCloseDatabase: async () => true,
+            onUnloadDatabase: async () => true,
+        },
+        replicator: { onCloseActiveReplication: async () => true },
+    };
+    const localDatabase = new LiveSyncLocalDB(name, { services } as unknown as LiveSyncLocalDBEnv);
+    if (!(await localDatabase.initializeDatabase())) {
+        await database.destroy();
+        throw new Error("The optional-file test database could not initialise");
+    }
+    return {
+        database,
+        localDatabase,
+        services,
+        settings,
+        async dispose() {
+            localDatabase.offRemoteChunkFetchedHandler?.();
+            await localDatabase.managers.teardownManagers();
+            await database.destroy();
+        },
+    };
+}