Reuse scoped content and progress operations

src/features/ConfigSync/CmdConfigSync.applyContent.unit.spec.ts

@@ -0,0 +1,275 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("@/deps.ts", () => ({
+    addIcon: vi.fn(),
+    diff_match_patch: class DiffMatchPatch {},
+    normalizePath: vi.fn((path: string) => path),
+    parseYaml: vi.fn(),
+    Platform: {},
+}));
+vi.mock("./PluginDialogModal.ts", () => ({
+    PluginDialogModal: class PluginDialogModal {},
+}));
+vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({
+    JsonResolveModal: class JsonResolveModal {},
+}));
+vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({
+    ConflictResolveModal: class ConflictResolveModal {},
+}));
+vi.mock("@/features/LiveSyncCommands.ts", () => ({
+    LiveSyncCommands: class LiveSyncCommands {
+        core!: { services: unknown };
+        get services() {
+            return this.core.services;
+        }
+    },
+}));
+vi.mock("@/common/types.ts", () => ({
+    ICXHeader: "ix:",
+    PERIODIC_PLUGIN_SWEEP: 60,
+}));
+vi.mock("@/common/utils.ts", () => ({
+    cancelTask: vi.fn(),
+    EVEN: Symbol("even"),
+    isCustomisationSyncMetadata: vi.fn(),
+    isPluginMetadata: vi.fn(),
+    scheduleTask: vi.fn(),
+}));
+vi.mock("@/common/PeriodicProcessor.ts", () => ({
+    PeriodicProcessor: class PeriodicProcessor {},
+}));
+vi.mock("@/common/events.ts", () => ({
+    EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync",
+    eventHub: {
+        onEvent: vi.fn(),
+    },
+}));
+vi.mock("@/common/translation", () => ({
+    $msg: vi.fn((message: string) => message),
+}));
+vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({
+    getObsidianCommunityPluginManager: vi.fn(),
+}));
+
+import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { ConfigSync, type PluginDataExDisplayV2 } from "./CmdConfigSync";
+
+const BASE_DIR = ".obsidian";
+const DEVICE_NAME = "test-device";
+const STAT = { mtime: 123456, ctime: 123000 };
+
+type ApplyFile = {
+    filename: string;
+    datatype: "newnote" | "plain";
+    data: string[];
+    mtime: number;
+    ctime: number;
+};
+
+function makeApplyFile(datatype: ApplyFile["datatype"], data: string[]): ApplyFile {
+    return {
+        filename: "plugins/example/data.json",
+        datatype,
+        data,
+        mtime: STAT.mtime,
+        ctime: STAT.ctime,
+    };
+}
+
+function setup(file: ApplyFile, currentText = "before", currentBinary = new ArrayBuffer(0)) {
+    const path = `${BASE_DIR}/${file.filename}`;
+    const events: string[] = [];
+    const storageAccess = {
+        ensureDir: vi.fn(async (_path: string) => {
+            events.push("ensure");
+        }),
+        readHiddenFileText: vi.fn(async (_path: string) => {
+            events.push("read-text");
+            return currentText;
+        }),
+        readHiddenFileBinary: vi.fn(async (_path: string) => {
+            events.push("read-binary");
+            return currentBinary;
+        }),
+        writeHiddenFileAuto: vi.fn(
+            async (_path: string, _content: string | ArrayBuffer, _stat?: { mtime: number; ctime: number }) => {
+                events.push("write");
+            }
+        ),
+    };
+    const storeCustomisationFileV2 = vi.fn(async (_path: string, _term: string) => {
+        events.push("store");
+        return true;
+    });
+    const log = vi.fn();
+    const configSync = Object.create(ConfigSync.prototype) as ConfigSync;
+    Object.assign(configSync, {
+        core: {
+            services: {
+                API: { getSystemConfigDir: () => BASE_DIR },
+                setting: { getDeviceAndVaultName: () => DEVICE_NAME },
+            },
+            storageAccess,
+        },
+        _log: log,
+        storeCustomisationFileV2,
+    });
+    const data = {
+        displayName: "Example plug-in",
+        name: "example",
+        files: [file],
+    } as unknown as PluginDataExDisplayV2;
+    return { configSync, data, events, log, path, storageAccess, storeCustomisationFileV2 };
+}
+
+const changedInputs = [
+    {
+        kind: "text",
+        file: makeApplyFile("plain", ["replacement text"]),
+        currentText: "previous text",
+        expectedText: "replacement text",
+    },
+    {
+        kind: "binary",
+        file: makeApplyFile("newnote", ["AQID"]),
+        currentBinary: new Uint8Array([9]).buffer,
+        expectedBytes: [1, 2, 3],
+    },
+] as const;
+
+const unchangedInputs = [
+    {
+        kind: "text",
+        file: makeApplyFile("plain", ["unchanged text"]),
+        currentText: "unchanged text",
+    },
+    {
+        kind: "binary",
+        file: makeApplyFile("newnote", ["AQID"]),
+        currentBinary: new Uint8Array([1, 2, 3]).buffer,
+    },
+] as const;
+
+describe("ConfigSync V2 content application", () => {
+    it.each(changedInputs)("writes changed $kind content, then stores it", async (input) => {
+        const fixture = setup(
+            input.file,
+            input.kind === "text" ? input.currentText : undefined,
+            input.kind === "binary" ? input.currentBinary : undefined
+        );
+
+        await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true);
+
+        expect(fixture.storageAccess.ensureDir).toHaveBeenCalledWith(fixture.path);
+        expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledOnce();
+        expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledWith(
+            fixture.path,
+            input.kind === "text" ? input.expectedText : expect.any(ArrayBuffer),
+            STAT
+        );
+        if (input.kind === "binary") {
+            const written = fixture.storageAccess.writeHiddenFileAuto.mock.calls[0]?.[1];
+            expect(written).toBeInstanceOf(ArrayBuffer);
+            expect(Array.from(new Uint8Array(written as ArrayBuffer))).toEqual(input.expectedBytes);
+        }
+        expect(fixture.events).toEqual([
+            "ensure",
+            input.kind === "text" ? "read-text" : "read-binary",
+            "write",
+            "store",
+        ]);
+        expect(fixture.storeCustomisationFileV2).toHaveBeenCalledWith(fixture.path, DEVICE_NAME);
+    });
+
+    it("writes the file timestamps captured before reading existing content", async () => {
+        const file = makeApplyFile("plain", ["replacement text"]);
+        const fixture = setup(file, "previous text");
+        fixture.storageAccess.readHiddenFileText.mockImplementation(async () => {
+            fixture.events.push("read-text");
+            file.mtime = STAT.mtime + 1;
+            file.ctime = STAT.ctime + 1;
+            return "previous text";
+        });
+
+        await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true);
+
+        expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledWith(fixture.path, "replacement text", STAT);
+    });
+
+    it.each(unchangedInputs)("skips writing and storing unchanged $kind content", async (input) => {
+        const fixture = setup(
+            input.file,
+            input.kind === "text" ? input.currentText : undefined,
+            input.kind === "binary" ? input.currentBinary : undefined
+        );
+
+        await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true);
+
+        expect(fixture.storageAccess.writeHiddenFileAuto).not.toHaveBeenCalled();
+        expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled();
+        expect(fixture.events).toEqual(["ensure", input.kind === "text" ? "read-text" : "read-binary"]);
+        expect(fixture.log).toHaveBeenCalledWith(
+            `The file ${input.file.filename} is already up-to-date`,
+            LOG_LEVEL_VERBOSE
+        );
+    });
+
+    it.each([
+        { kind: "text", file: makeApplyFile("plain", ["replacement text"]) },
+        { kind: "binary", file: makeApplyFile("newnote", ["AQID"]) },
+    ])("uses empty $kind content when reading the existing file fails", async ({ kind, file }) => {
+        const fixture = setup(file);
+        const readError = new Error("Could not read existing content");
+        if (kind === "text") {
+            fixture.storageAccess.readHiddenFileText.mockRejectedValueOnce(readError);
+        } else {
+            fixture.storageAccess.readHiddenFileBinary.mockRejectedValueOnce(readError);
+        }
+
+        await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true);
+
+        expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledOnce();
+        expect(fixture.storeCustomisationFileV2).toHaveBeenCalledWith(fixture.path, DEVICE_NAME);
+        expect(fixture.log).toHaveBeenCalledWith(`Could not read the file ${file.filename}`, LOG_LEVEL_VERBOSE);
+        expect(fixture.log).toHaveBeenCalledWith(readError, LOG_LEVEL_VERBOSE);
+    });
+
+    it("awaits the write before storing the applied content", async () => {
+        const file = makeApplyFile("plain", ["replacement text"]);
+        const fixture = setup(file, "previous text");
+        let finishWrite!: () => void;
+        let signalWriteStarted!: () => void;
+        const writeGate = new Promise<void>((resolve) => {
+            finishWrite = resolve;
+        });
+        const writeStarted = new Promise<void>((resolve) => {
+            signalWriteStarted = resolve;
+        });
+        fixture.storageAccess.writeHiddenFileAuto.mockImplementation(async () => {
+            fixture.events.push("write-start");
+            signalWriteStarted();
+            await writeGate;
+            fixture.events.push("write-finished");
+        });
+
+        const applying = fixture.configSync.applyDataV2(fixture.data);
+        await writeStarted;
+        expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled();
+
+        finishWrite();
+        await expect(applying).resolves.toBe(true);
+        expect(fixture.events.slice(-3)).toEqual(["write-start", "write-finished", "store"]);
+    });
+
+    it("returns false after a write failure and does not store the file", async () => {
+        const fixture = setup(makeApplyFile("plain", ["replacement text"]), "previous text");
+        const writeError = new Error("Write failed");
+        fixture.storageAccess.writeHiddenFileAuto.mockRejectedValueOnce(writeError);
+
+        await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(false);
+
+        expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled();
+        expect(fixture.log).toHaveBeenCalledWith("Applying Example plug-in.. Failed", LOG_LEVEL_NOTICE);
+        expect(fixture.log).toHaveBeenCalledWith(writeError, LOG_LEVEL_VERBOSE);
+    });
+});

src/features/ConfigSync/CmdConfigSync.scanAllConfigFiles.unit.spec.ts

@@ -0,0 +1,279 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@/deps.ts", () => ({
+    addIcon: vi.fn(),
+    diff_match_patch: class DiffMatchPatch {},
+    normalizePath: vi.fn((path: string) => path),
+    parseYaml: vi.fn(),
+    Platform: {},
+}));
+vi.mock("./PluginDialogModal.ts", () => ({
+    PluginDialogModal: class PluginDialogModal {},
+}));
+vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({
+    JsonResolveModal: class JsonResolveModal {},
+}));
+vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({
+    ConflictResolveModal: class ConflictResolveModal {},
+}));
+vi.mock("@/features/LiveSyncCommands.ts", () => ({
+    LiveSyncCommands: class LiveSyncCommands {
+        core!: { services: unknown; localDatabase: unknown };
+        get localDatabase() {
+            return this.core.localDatabase;
+        }
+        get services() {
+            return this.core.services;
+        }
+    },
+}));
+vi.mock("@/common/types.ts", () => ({
+    ICXHeader: "ix:",
+    PERIODIC_PLUGIN_SWEEP: 60,
+}));
+vi.mock("@/common/utils.ts", () => ({
+    cancelTask: vi.fn(),
+    EVEN: Symbol("even"),
+    isCustomisationSyncMetadata: vi.fn(),
+    isPluginMetadata: vi.fn(),
+    scheduleTask: vi.fn(),
+}));
+vi.mock("@/common/PeriodicProcessor.ts", () => ({
+    PeriodicProcessor: class PeriodicProcessor {},
+}));
+vi.mock("@/common/events.ts", () => ({
+    EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync",
+    eventHub: {
+        onEvent: vi.fn(),
+    },
+}));
+vi.mock("@/common/translation", () => ({
+    $msg: vi.fn((message: string) => message),
+}));
+vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({
+    getObsidianCommunityPluginManager: vi.fn(),
+}));
+
+vi.mock("octagonal-wheels/concurrency/scopedIteration", { spy: true });
+
+import { LOG_LEVEL_VERBOSE, type FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import * as iterationModule from "octagonal-wheels/concurrency/scopedIteration";
+import { promiseWithResolvers } from "octagonal-wheels/promises";
+import { ConfigSync } from "./CmdConfigSync";
+
+const term = "device";
+const prefix = `ix:${term}/`;
+const keyOf = (path: string) => prefix + path;
+type Row = { _id: string; path: string };
+const row = (name: string, path = name): Row => ({ _id: keyOf(name), path });
+
+function setup(files: string[] = [], rows: Row[] = []) {
+    const events: string[] = [];
+    const findEntries = vi.fn(async function* () {
+        yield* rows;
+    });
+    const scanInternalFiles = vi.fn(async () => files as FilePath[]);
+    const storeCustomisationFileV2 = vi.fn(async (path: string, _term: string) => {
+        events.push(`store:${path}`);
+    });
+    const deleteConfigOnDatabase = vi.fn(async (key: string) => {
+        events.push(`delete:${key}`);
+    });
+    const updatePluginList = vi.fn(async (_showMessage: boolean) => {});
+    const log = vi.fn();
+    const configSync = Object.create(ConfigSync.prototype) as ConfigSync;
+    Object.assign(configSync, {
+        core: {
+            settings: { usePluginSyncV2: true },
+            localDatabase: { findEntries },
+            services: { setting: { getDeviceAndVaultName: () => term } },
+        },
+        scanInternalFiles,
+        isTargetPath: (path: string) => path !== "ignored",
+        filenameWithUnifiedKey: (path: string, _term: string) => keyOf(path),
+        unifiedKeyPrefixOfTerminal: (_term: string) => prefix,
+        storeCustomisationFileV2,
+        deleteConfigOnDatabase,
+        updatePluginList,
+        _log: log,
+    });
+    return {
+        configSync,
+        events,
+        findEntries,
+        scanInternalFiles,
+        storeCustomisationFileV2,
+        deleteConfigOnDatabase,
+        updatePluginList,
+        log,
+    };
+}
+
+beforeEach(() => {
+    vi.clearAllMocks();
+});
+
+describe("ConfigSync V2 scan behaviour", () => {
+    it("finishes existing database entries before storing extra files and retains target filtering", async () => {
+        const fixture = setup(
+            ["matched", "extra", "ignored"],
+            [row("matched"), row("stale"), row("encoded", "encoded%path")]
+        );
+        await fixture.configSync.scanAllConfigFiles(false);
+
+        expect(fixture.events).toEqual(["store:matched", `delete:${keyOf("stale")}`, "store:extra"]);
+        expect(fixture.storeCustomisationFileV2.mock.calls).toEqual([
+            ["matched", term],
+            ["extra", term],
+        ]);
+        expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledExactlyOnceWith(keyOf("stale"));
+        expect(fixture.findEntries).toHaveBeenCalledExactlyOnceWith(prefix, `${prefix}\u{10ffff}`, {
+            include_docs: true,
+        });
+        expect(fixture.updatePluginList).toHaveBeenCalledExactlyOnceWith(false);
+    });
+
+    it("finishes database enumeration before dispatching database work", async () => {
+        const fixture = setup();
+        const enumerationPaused = promiseWithResolvers<void>();
+        const continueEnumeration = promiseWithResolvers<void>();
+        fixture.findEntries.mockImplementation(async function* () {
+            yield row("stale");
+            enumerationPaused.resolve();
+            await continueEnumeration.promise;
+        });
+
+        const scan = fixture.configSync.scanAllConfigFiles(false);
+        try {
+            await enumerationPaused.promise;
+            expect(fixture.deleteConfigOnDatabase).not.toHaveBeenCalled();
+            expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled();
+        } finally {
+            continueEnumeration.resolve();
+            await scan;
+        }
+
+        expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledExactlyOnceWith(keyOf("stale"));
+    });
+
+    it("limits actual database operations to ten and completes that batch before the extra batch", async () => {
+        const fixture = setup(
+            ["extra"],
+            Array.from({ length: 11 }, (_, index) => row(String(index)))
+        );
+        const gates = Array.from({ length: 11 }, () => promiseWithResolvers<void>());
+        const tenStarted = promiseWithResolvers<void>();
+        const lastStarted = promiseWithResolvers<void>();
+        let active = 0;
+        let maximum = 0;
+        let started = 0;
+        fixture.deleteConfigOnDatabase.mockImplementation(async (key) => {
+            const index = Number(key.slice(prefix.length));
+            active++;
+            maximum = Math.max(maximum, active);
+            started++;
+            if (started === 10) tenStarted.resolve();
+            if (index === 10) lastStarted.resolve();
+            try {
+                await gates[index].promise;
+            } finally {
+                active--;
+            }
+        });
+        const scan = fixture.configSync.scanAllConfigFiles(false);
+        try {
+            await tenStarted.promise;
+            expect(active).toBe(10);
+            expect(started).toBe(10);
+            expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled();
+            gates[0].resolve();
+            await lastStarted.promise;
+            expect(active).toBe(10);
+            expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled();
+        } finally {
+            gates.forEach((gate) => gate.resolve());
+            await scan;
+        }
+        expect(maximum).toBe(10);
+        expect(active).toBe(0);
+        expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledTimes(11);
+        expect(fixture.storeCustomisationFileV2).toHaveBeenCalledExactlyOnceWith("extra", term);
+    });
+
+    it("logs a database processing error and continues through the extra batch", async () => {
+        const fixture = setup(["extra"], [row("bad"), row("other")]);
+        const error = new Error("delete");
+        fixture.deleteConfigOnDatabase.mockRejectedValueOnce(error);
+        await fixture.configSync.scanAllConfigFiles(false);
+
+        expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledTimes(2);
+        expect(fixture.log).toHaveBeenCalledWith(`scanAllConfigFiles - Error: ${keyOf("bad")}`, LOG_LEVEL_VERBOSE);
+        expect(fixture.log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE);
+        expect(fixture.storeCustomisationFileV2).toHaveBeenCalledExactlyOnceWith("extra", term);
+        expect(fixture.updatePluginList).toHaveBeenCalledOnce();
+    });
+
+    it("retains a matched local file for the extra batch when its first store fails", async () => {
+        const fixture = setup(["matched"], [row("matched")]);
+        const error = new Error("first store");
+        fixture.storeCustomisationFileV2.mockRejectedValueOnce(error);
+        await fixture.configSync.scanAllConfigFiles(false);
+
+        expect(fixture.storeCustomisationFileV2.mock.calls).toEqual([
+            ["matched", term],
+            ["matched", term],
+        ]);
+        expect(fixture.log).toHaveBeenCalledWith(`scanAllConfigFiles - Error: ${keyOf("matched")}`, LOG_LEVEL_VERBOSE);
+        expect(fixture.log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE);
+        expect(fixture.events).toEqual(["store:matched"]);
+        expect(fixture.updatePluginList).toHaveBeenCalledOnce();
+    });
+
+    it("logs an extra-file processing error and continues with the remaining files", async () => {
+        const fixture = setup(["bad", "other"]);
+        const error = new Error("store");
+        fixture.storeCustomisationFileV2.mockRejectedValueOnce(error);
+        await fixture.configSync.scanAllConfigFiles(false);
+
+        expect(fixture.storeCustomisationFileV2.mock.calls).toEqual([
+            ["bad", term],
+            ["other", term],
+        ]);
+        expect(fixture.log).toHaveBeenCalledWith("scanAllConfigFiles - Error: bad", LOG_LEVEL_VERBOSE);
+        expect(fixture.log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE);
+        expect(fixture.updatePluginList).toHaveBeenCalledOnce();
+    });
+});
+
+describe("ConfigSync scoped iteration composition", () => {
+    it("uses bounded child scopes for both batches with the same shared values", async () => {
+        const fixture = setup(["matched", "extra"], [row("matched"), row("stale")]);
+        await fixture.configSync.scanAllConfigFiles(false);
+
+        expect(iterationModule.forEachInScope).toHaveBeenCalledTimes(2);
+        const [database, extra] = vi.mocked(iterationModule.forEachInScope).mock.calls;
+        expect(database[0]).toBe(extra[0]);
+        expect(database[0].value).toEqual({ term, localFileMap: new Map([[keyOf("extra"), "extra"]]) });
+        expect(database[2]).toEqual({ concurrency: 10 });
+        expect(extra[2]).toEqual({ concurrency: 10 });
+        expect(fixture.events).toEqual(["store:matched", `delete:${keyOf("stale")}`, "store:extra"]);
+    });
+
+    it.each(["database", "extra"])("propagates a %s runner failure outside per-item recovery", async (batch) => {
+        const fixture = setup(["extra"], [row("stale")]);
+        const error = new Error("iteration");
+        const actual = await vi.importActual<typeof import("octagonal-wheels/concurrency/scopedIteration")>(
+            "octagonal-wheels/concurrency/scopedIteration"
+        );
+        if (batch === "extra") {
+            vi.mocked(iterationModule.forEachInScope).mockImplementationOnce(actual.forEachInScope);
+        }
+        vi.mocked(iterationModule.forEachInScope).mockRejectedValueOnce(error);
+        await expect(fixture.configSync.scanAllConfigFiles(false)).rejects.toBe(error);
+
+        expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledTimes(batch === "extra" ? 1 : 0);
+        expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled();
+        expect(fixture.log).not.toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE);
+        expect(fixture.updatePluginList).not.toHaveBeenCalled();
+    });
+});

src/features/ConfigSync/CmdConfigSync.ts

@@ -59,7 +59,8 @@ import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
 import { pluginScanningCount } from "@vrtmrz/livesync-commonlib/compat/mock_and_interop/stores";
 import { base64ToArrayBuffer, base64ToString } from "octagonal-wheels/binary/base64";
 import { ConflictResolveModal } from "@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts";
-import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
+import { forEachInScope } from "octagonal-wheels/concurrency/scopedIteration";
+import { withScope } from "octagonal-wheels/resource/scope";
 import { EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG, eventHub } from "@/common/events.ts";
 import { PluginDialogModal } from "./PluginDialogModal.ts";
 import { $msg } from "@/common/translation";
@@ -68,6 +69,7 @@ import type { LiveSyncCore } from "@/main.ts";
 import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError";
 import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts";
 import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts";
+import { createContentApplier } from "octagonal-wheels/content";
 
 import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils";
 import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
@@ -285,6 +287,10 @@ export type PluginDataExDisplay = {
     mtime: number;
 };
 type LoadedEntryPluginDataExFile = LoadedEntry & PluginDataExFile;
+type ConfigFileApplicationTarget = {
+    readonly path: FilePath;
+    readonly stat: { readonly mtime: number; readonly ctime: number };
+};
 
 function categoryToFolder(category: string, configDir: string = ""): string {
     switch (category) {
@@ -1027,44 +1033,63 @@ export class ConfigSync extends LiveSyncCommands {
                 await this.storeCustomisationFileV2(path, this.services.setting.getDeviceAndVaultName());
             } else {
                 const files = data.files;
+                const applyBinaryContent = createContentApplier<
+                    ConfigFileApplicationTarget,
+                    LoadedEntryPluginDataExFile,
+                    ArrayBuffer,
+                    ArrayBuffer
+                >({
+                    read: (target) => this.core.storageAccess.readHiddenFileBinary(target.path),
+                    onReadError: (ex, _path, file) => {
+                        this._log(`Could not read the file ${file.filename}`, LOG_LEVEL_VERBOSE);
+                        this._log(ex, LOG_LEVEL_VERBOSE);
+                        return new ArrayBuffer(0);
+                    },
+                    decode: (file) => base64ToArrayBuffer(file.data),
+                    isSame: (oldData, content) => isDocContentSame(oldData, content),
+                    write: (target, content) =>
+                        this.core.storageAccess.writeHiddenFileAuto(target.path, content, target.stat),
+                });
+                const applyTextContent = createContentApplier<
+                    ConfigFileApplicationTarget,
+                    LoadedEntryPluginDataExFile,
+                    string,
+                    string
+                >({
+                    read: (target) => this.core.storageAccess.readHiddenFileText(target.path),
+                    onReadError: (ex, _path, file) => {
+                        this._log(`Could not read the file ${file.filename}`, LOG_LEVEL_VERBOSE);
+                        this._log(ex, LOG_LEVEL_VERBOSE);
+                        return "";
+                    },
+                    decode: (file) => getDocData(file.data),
+                    isSame: (oldData, content) => isDocContentSame(oldData, content),
+                    write: (target, content) =>
+                        this.core.storageAccess.writeHiddenFileAuto(target.path, content, target.stat),
+                });
                 for (const f of files) {
                     // If files have applied, modified time will be updated to the current time.
-                    const stat = { mtime: f.mtime, ctime: f.ctime };
                     const path = `${baseDir}/${f.filename}` as FilePath;
+                    const target: ConfigFileApplicationTarget = {
+                        path,
+                        stat: { mtime: f.mtime, ctime: f.ctime },
+                    };
                     this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`);
                     // const contentEach = createBlob(f.data);
                     await this.core.storageAccess.ensureDir(path);
 
                     if (f.datatype == "newnote") {
-                        let oldData;
-                        try {
-                            oldData = await this.core.storageAccess.readHiddenFileBinary(path);
-                        } catch (ex) {
-                            this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE);
-                            this._log(ex, LOG_LEVEL_VERBOSE);
-                            oldData = new ArrayBuffer(0);
-                        }
-                        const content = base64ToArrayBuffer(f.data);
-                        if (await isDocContentSame(oldData, content)) {
+                        const result = await applyBinaryContent(target, f);
+                        if (result == "unchanged") {
                             this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE);
                             continue;
                         }
-                        await this.core.storageAccess.writeHiddenFileAuto(path, content, stat);
                     } else {
-                        let oldData;
-                        try {
-                            oldData = await this.core.storageAccess.readHiddenFileText(path);
-                        } catch (ex) {
-                            this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE);
-                            this._log(ex, LOG_LEVEL_VERBOSE);
-                            oldData = "";
-                        }
-                        const content = getDocData(f.data);
-                        if (await isDocContentSame(oldData, content)) {
+                        const result = await applyTextContent(target, f);
+                        if (result == "unchanged") {
                             this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE);
                             continue;
                         }
-                        await this.core.storageAccess.writeHiddenFileAuto(path, content, stat);
                     }
                     this._log(`Applied ${f.filename} of ${data.displayName || data.name}..`);
                     await this.storeCustomisationFileV2(path, this.services.setting.getDeviceAndVaultName());
@@ -1613,49 +1638,35 @@ export class ConfigSync extends LiveSyncCommands {
                 const entries = this.localDatabase.findEntries(prefix + "", `${prefix}\u{10ffff}`, {
                     include_docs: true,
                 });
-                const tasks = [] as (() => Promise<void>)[];
-                const concurrency = 10;
-                const semaphore = Semaphore(concurrency);
+                const databaseEntries: { _id: string; path: FilePathWithPrefix }[] = [];
                 for await (const item of entries) {
-                    if (item.path.indexOf("%") !== -1) {
-                        continue;
-                    }
-                    tasks.push(async () => {
-                        const releaser = await semaphore.acquire();
+                    if (item.path.indexOf("%") === -1) databaseEntries.push(item);
+                }
+                await withScope({ term, localFileMap }, async (scope) => {
+                    await forEachInScope(scope, databaseEntries, { concurrency: 10 }, async (item, ctx) => {
                         try {
                             const unifiedFilenameWithKey = `${item._id}` as FilePathWithPrefix;
-                            const localPath = localFileMap.get(unifiedFilenameWithKey);
+                            const localPath = ctx.value.localFileMap.get(unifiedFilenameWithKey);
                             if (localPath) {
-                                await this.storeCustomisationFileV2(localPath, term);
-                                localFileMap.delete(unifiedFilenameWithKey);
+                                await this.storeCustomisationFileV2(localPath, ctx.value.term);
+                                ctx.value.localFileMap.delete(unifiedFilenameWithKey);
                             } else {
                                 await this.deleteConfigOnDatabase(unifiedFilenameWithKey);
                             }
                         } catch (ex) {
                             this._log(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE);
                             this._log(ex, LOG_LEVEL_VERBOSE);
-                        } finally {
-                            releaser();
                         }
                     });
-                }
-                await Promise.all(tasks.map((e) => e()));
-                // Extra files
-                const taskExtra = [] as (() => Promise<void>)[];
-                for (const [, filePath] of localFileMap) {
-                    taskExtra.push(async () => {
-                        const releaser = await semaphore.acquire();
+                    await forEachInScope(scope, localFileMap.values(), { concurrency: 10 }, async (filePath, ctx) => {
                         try {
-                            await this.storeCustomisationFileV2(filePath, term);
+                            await this.storeCustomisationFileV2(filePath, ctx.value.term);
                         } catch (ex) {
                             this._log(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE);
                             this._log(ex, LOG_LEVEL_VERBOSE);
-                        } finally {
-                            releaser();
                         }
                     });
-                }
-                await Promise.all(taskExtra.map((e) => e()));
+                });
                 fireAndForget(() => this.updatePluginList(false));
             } else {
                 const files = filesAll