Clarify Hidden File Sync selectors and test normal allow-lists

test/e2e-obsidian/README.md

@@ -248,6 +248,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms
 
 `test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives.
 
+`test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path.
+
 `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy.
 
 The workflow also opens the Customisation Sync dialogue. `--case=visibility` checks that **Hide not applicable items** hides an identical snippet, preserves a different snippet and its source selector, and restores the identical snippet when cleared. `--case=mtime` gives the multi-file plug-in fixture modern millisecond timestamps and requires an older remote copy to remain labelled **Older** and unselected by **Select All Shiny**. With no case argument, both checks run before the existing apply, update, and deletion workflow.

test/e2e-obsidian/scripts/hidden-file-selectors.ts

@@ -0,0 +1,202 @@
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { fileURLToPath } from "node:url";
+import { evalObsidianJson } from "../runner/cli.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    createE2eCouchDbPluginData,
+    createE2eObsidianDeviceLocalState,
+    waitForLiveSyncCoreReady,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+const hiddenRoot = ".obsidian/livesync-selector-example";
+const hiddenFiles = {
+    [`${hiddenRoot}/main.js`]: "console.log('hidden fixture');\n",
+    [`${hiddenRoot}/styles.css`]: ".hidden-fixture { color: blue; }\n",
+    [`${hiddenRoot}/settings.json`]: '{"fixture":true}\n',
+};
+const controlRoot = ".obsidian/livesync-selector-control";
+const controlFiles = {
+    [`${controlRoot}/main.js`]: "console.log('control fixture');\n",
+    [`${controlRoot}/styles.css`]: ".control-fixture { color: red; }\n",
+    [`${controlRoot}/settings.json`]: '{"control":true}\n',
+};
+const allowList = "\\.(?:md|png|pdf)$";
+const coreCode = "const core=app.plugins.plugins['obsidian-livesync'].core;";
+const hiddenCode = `${coreCode}const addOn=core.getAddOn('HiddenFileSync');`;
+
+type Observation = { name: string; expected: unknown; actual: unknown; passed: boolean };
+
+async function writeFixtures(vaultPath: string, files: Record<string, string>) {
+    for (const [path, content] of Object.entries(files)) {
+        await mkdir(dirname(join(vaultPath, path)), { recursive: true });
+        await writeFile(join(vaultPath, path), content);
+    }
+}
+
+async function removeFixtures(vaultPath: string, files: Record<string, string>) {
+    for (const path of Object.keys(files)) await rm(join(vaultPath, path), { force: true });
+}
+
+async function matchingFiles(vaultPath: string, files: Record<string, string>) {
+    const matches: string[] = [];
+    for (const [path, content] of Object.entries(files)) {
+        try {
+            if ((await readFile(join(vaultPath, path), "utf8")) === content) matches.push(path);
+        } catch (error) {
+            if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error;
+        }
+    }
+    return matches.sort();
+}
+
+async function main() {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error("The Obsidian CLI is unavailable");
+    const cliBinary = cli.binary;
+    const output =
+        process.env.E2E_HIDDEN_FILE_SELECTORS_OUTPUT ?? join(tmpdir(), "livesync-hidden-file-selectors.json");
+    const observations: Observation[] = [];
+    const report: Record<string, unknown> = {
+        revision: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(),
+        sourceDiffSha256: createHash("sha256")
+            .update(new Uint8Array(execFileSync("git", ["diff", "--binary", "HEAD"])))
+            .digest("hex"),
+        scenarioSha256: createHash("sha256")
+            .update(new Uint8Array(await readFile(fileURLToPath(import.meta.url))))
+            .digest("hex"),
+        commonlibEntrySha256: createHash("sha256")
+            .update(
+                new Uint8Array(
+                    await readFile(
+                        "node_modules/@vrtmrz/livesync-commonlib/dist/managers/EntryManager/EntryManagerImpls.js"
+                    )
+                )
+            )
+            .digest("hex"),
+        bundleSha256: createHash("sha256")
+            .update(new Uint8Array(await readFile("main.js")))
+            .digest("hex"),
+        commonlibVersion: JSON.parse(await readFile("node_modules/@vrtmrz/livesync-commonlib/package.json", "utf8"))
+            .version,
+        observations,
+    };
+    const vault = await createTemporaryVault("livesync-hidden-file-selectors-");
+    let session: ObsidianLiveSyncSession | undefined;
+    const record = (name: string, actual: unknown, expected: unknown) => {
+        const observation = { name, expected, actual, passed: isDeepStrictEqual(actual, expected) };
+        observations.push(observation);
+        console.log(JSON.stringify(observation));
+    };
+    const evaluate = async <T>(code: string): Promise<T> => {
+        if (!session) throw new Error("The isolated Obsidian session has not started");
+        return await evalObsidianJson<T>(cliBinary, `(async()=>{${code}})()`, session.cliEnv, 60_000);
+    };
+    try {
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary,
+            vault,
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+            pluginData: createE2eCouchDbPluginData(
+                { uri: "http://127.0.0.1:9", username: "", password: "", dbName: "selector-local-only" },
+                { periodicReplication: false, syncOnFileOpen: false, useOnlyLocalChunk: true }
+            ),
+        });
+        report.pluginVersion = session.readiness.pluginVersion;
+        await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
+        report.runtime = await evaluate(`${coreCode}
+            core.services.setting.setDeviceAndVaultName('selector-local');
+            Object.assign(core.settings,{
+                isConfigured:true,usePluginSync:false,usePluginSyncV2:true,syncInternalFiles:false,
+                autoSweepPlugins:false,autoSweepPluginsPeriodic:false,watchInternalFileChanges:false,
+                syncOnlyRegEx:${JSON.stringify(allowList)},syncIgnoreRegEx:''
+            });
+            return JSON.stringify({userAgent:navigator.userAgent,databaseReady:core.localDatabase.isReady});
+        `);
+
+        await writeFixtures(vault.path, hiddenFiles);
+        const uploadCode = (normalAllowList: string, files: Record<string, string>) => `${hiddenCode}
+            Object.assign(core.settings,{
+                syncInternalFiles:true,syncOnlyRegEx:${JSON.stringify(normalAllowList)},
+                syncInternalFilesTargetPatterns:'^\\\\.obsidian(?:$|/livesync-selector-(?:example|control)(?:/|$))',syncInternalFilesIgnorePatterns:''
+            });
+            const candidates=await addOn.scanInternalFileNames();
+            const selected=candidates.filter(path=>${JSON.stringify(Object.keys(files))}.includes(path));
+            if(selected.length!==3)throw new Error('Hidden-file fixture selection failed: '+JSON.stringify({selected,target:core.settings.syncInternalFilesTargetPatterns}));
+            for(const path of selected)await addOn.updateLastProcessedAsActualFile(path);
+            await addOn.rebuildFromStorage(false,${JSON.stringify(Object.keys(files))},false);
+            const paths=[];
+            for await(const doc of core.localDatabase.findEntries('i:','i;')){
+                if(${JSON.stringify(Object.keys(files).map((path) => `i:${path}`))}.includes(doc.path)&&!doc.deleted&&!doc._deleted)paths.push(doc.path.substring(2));
+            }
+            return JSON.stringify(paths.sort());
+        `;
+        const hiddenUpload = await evaluate<string[]>(uploadCode(allowList, hiddenFiles));
+        record(
+            "#1272: a normal allow-list permits selected hidden uploads",
+            hiddenUpload,
+            Object.keys(hiddenFiles).sort()
+        );
+        await writeFixtures(vault.path, controlFiles);
+        const uploadControl = await evaluate<string[]>(uploadCode("", controlFiles));
+        record(
+            "#1272 control: clearing the normal allow-list permits hidden uploads",
+            uploadControl,
+            Object.keys(controlFiles).sort()
+        );
+
+        const pullCode = (normalAllowList: string, files: Record<string, string>) => `${hiddenCode}
+            core.settings.syncOnlyRegEx=${JSON.stringify(normalAllowList)};
+            const results=[];
+            for await(const doc of core.localDatabase.findEntries('i:','i;')){
+                if(${JSON.stringify(Object.keys(files).map((path) => `i:${path}`))}.includes(doc.path)&&!doc.deleted&&!doc._deleted){
+                    results.push(await addOn.extractInternalFileFromDatabase(doc.path.substring(2),true,doc,false));
+                }
+            }
+            return JSON.stringify(results);
+        `;
+        await removeFixtures(vault.path, hiddenFiles);
+        report.hiddenPullWithAllowList = await evaluate(pullCode(allowList, hiddenFiles));
+        record(
+            "#1272: a normal allow-list permits selected hidden downloads",
+            await matchingFiles(vault.path, hiddenFiles),
+            Object.keys(hiddenFiles).sort()
+        );
+        await removeFixtures(vault.path, controlFiles);
+        report.hiddenPullWithoutAllowList = await evaluate(pullCode("", controlFiles));
+        record(
+            "#1272 control: clearing the normal allow-list permits hidden downloads",
+            await matchingFiles(vault.path, controlFiles),
+            Object.keys(controlFiles).sort()
+        );
+    } catch (error) {
+        report.runtimeError = error instanceof Error ? error.stack : String(error);
+        if (session) report.processOutput = JSON.stringify(session.app.output()).slice(-4_000);
+        throw error;
+    } finally {
+        try {
+            await writeFile(output, JSON.stringify(report, null, 2) + "\n");
+            console.log(`Verification report: ${output}`);
+        } finally {
+            try {
+                if (session) await session.app.stop();
+            } finally {
+                await vault.dispose();
+            }
+        }
+    }
+    if (observations.some((observation) => !observation.passed)) process.exitCode = 1;
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exitCode = 1;
+});

test/e2e-obsidian/scripts/run-focused.ts

@@ -39,6 +39,7 @@ const focusedScenarios = new Set([
     "two-vault-sync",
     "security-seed-reconnect",
     "hidden-file-snippet-sync",
+    "hidden-file-selectors",
     "customisation-sync",
     "received-change-readiness",
     "internal-metadata-doctor",

test/fixtures/optionalFileSyncDatabase.ts

@@ -0,0 +1,78 @@
+import { randomUUID } from "node:crypto";
+import PouchDB from "pouchdb-core";
+import MemoryAdapter from "pouchdb-adapter-memory";
+import { vi } from "vitest";
+import {
+    DEFAULT_SETTINGS,
+    type EntryDoc,
+    type SavingEntry,
+    type FilePathWithPrefix,
+    type DocumentID,
+} from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { createTextBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
+import { createLiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context";
+import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB";
+
+PouchDB.plugin(MemoryAdapter);
+
+export function optionalFileEntry(path: string, content: string): SavingEntry {
+    return {
+        _id: path as DocumentID,
+        path: path as FilePathWithPrefix,
+        ctime: 1_000,
+        mtime: 2_000,
+        size: content.length,
+        type: "plain",
+        datatype: "plain",
+        data: createTextBlob(content),
+        children: [],
+        eden: {},
+    };
+}
+
+export async function createOptionalFileSyncDatabase() {
+    const name = `optional-file-sync-${randomUUID()}`;
+    const database = new PouchDB<EntryDoc>(name, { adapter: "memory" });
+    const settings = {
+        ...DEFAULT_SETTINGS,
+        syncInternalFiles: true,
+        useOnlyLocalChunk: true,
+        usePluginSync: true,
+        usePluginSyncV2: true,
+        pluginSyncExtendedSetting: {},
+    };
+    const services = {
+        API: { addLog: vi.fn(), getSystemConfigDir: () => ".obsidian" },
+        setting: { currentSettings: () => settings, getDeviceAndVaultName: () => "device" },
+        path: {
+            path2id: async (path: string) => path,
+            id2path: (id: string, entry?: { path?: string }) => entry?.path ?? id,
+            getPath: (entry: { path: FilePathWithPrefix }) => entry.path,
+        },
+        context: { events: createLiveSyncEventHub() },
+        database: { createPouchDBInstance: () => database },
+        databaseEvents: {
+            onDatabaseInitialisation: async () => true,
+            onDatabaseHasReady: async () => true,
+            onCloseDatabase: async () => true,
+            onUnloadDatabase: async () => true,
+        },
+        replicator: { onCloseActiveReplication: async () => true },
+    };
+    const localDatabase = new LiveSyncLocalDB(name, { services } as unknown as LiveSyncLocalDBEnv);
+    if (!(await localDatabase.initializeDatabase())) {
+        await database.destroy();
+        throw new Error("The optional-file test database could not initialise");
+    }
+    return {
+        database,
+        localDatabase,
+        services,
+        settings,
+        async dispose() {
+            localDatabase.offRemoteChunkFetchedHandler?.();
+            await localDatabase.managers.teardownManagers();
+            await database.destroy();
+        },
+    };
+}