Preparing v0.24.0

src/modules/core/ModuleDatabaseFileAccess.ts

@@ -0,0 +1,262 @@
+import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
+import { EVENT_FILE_SAVED, eventHub } from "../../common/events";
+import { getPathFromUXFileInfo, isInternalMetadata, markChangesAreSame } from "../../common/utils";
+import type { UXFileInfoStub, FilePathWithPrefix, UXFileInfo, MetaEntry, LoadedEntry, FilePath, SavingEntry } from "../../lib/src/common/types";
+import type { DatabaseFileAccess } from "../interfaces/DatabaseFileAccess";
+import { type IObsidianModule } from "../AbstractObsidianModule.ts";
+import { isPlainText, shouldBeIgnored } from "../../lib/src/string_and_binary/path";
+import { createBlob, createTextBlob, delay, determineTypeFromBlob, isDocContentSame, readContent } from "../../lib/src/common/utils";
+import { serialized } from "octagonal-wheels/concurrency/lock";
+import { AbstractModule } from "../AbstractModule.ts";
+
+
+export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidianModule, DatabaseFileAccess {
+    $everyOnload(): Promise<boolean> {
+        this.core.databaseFileAccess = this;
+        return Promise.resolve(true);
+    }
+
+    async $everyModuleTest(): Promise<boolean> {
+        if (!this.settings.enableDebugTools) return Promise.resolve(true);
+        const testString = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam nec purus nec nunc"
+        // Before test, we need to delete completely.
+        const conflicts = await this.getConflictedRevs("autoTest.md" as FilePathWithPrefix);
+        for (const rev of conflicts) {
+            await this.delete("autoTest.md" as FilePathWithPrefix, rev);
+        }
+        await this.delete("autoTest.md" as FilePathWithPrefix);
+        // OK, begin!
+
+        await this._test("storeContent", async () => (await this.storeContent("autoTest.md" as FilePathWithPrefix, testString)));
+        // For test, we need to clear the caches.
+        await this.localDatabase.hashCaches.clear();
+        await this._test("readContent", async () => {
+            const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
+            if (!content) return "File not found";
+            if (content.deleted) return "File is deleted";
+            return (await content.body.text() == testString) ? true : `Content is not same ${await content.body.text()}`;
+        });
+        await this._test("delete", async () => (await this.delete("autoTest.md" as FilePathWithPrefix)));
+        await this._test("read deleted content", async () => {
+            const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
+            if (!content) return true;
+            if (content.deleted) return true;
+            return `Still exist !:${await content.body.text()},${JSON.stringify(content, undefined, 2)}`;
+        });
+        await delay(100);
+        return this.testDone();
+    }
+
+    async checkIsTargetFile(file: UXFileInfoStub | FilePathWithPrefix): Promise<boolean> {
+        const path = getPathFromUXFileInfo(file);
+        if (!await this.core.$$isTargetFile(path)) {
+            this._log(`File is not target`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        if (shouldBeIgnored(path)) {
+            this._log(`File should be ignored`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        return true;
+    }
+
+    async delete(file: UXFileInfoStub | FilePathWithPrefix, rev?: string): Promise<boolean> {
+        if (!await this.checkIsTargetFile(file)) {
+            return true;
+        }
+        const fullPath = getPathFromUXFileInfo(file);
+        try {
+            this._log(`deleteDB By path:${fullPath}`);
+            return await this.deleteFromDBbyPath(fullPath, rev);
+        } catch (ex) {
+            this._log(`Failed to delete ${fullPath}`);
+            this._log(ex, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+    }
+
+    async createChunks(file: UXFileInfo, force: boolean = false, skipCheck?: boolean): Promise<boolean> {
+        return await this._store(file, force, skipCheck, true);
+    }
+
+    async store(file: UXFileInfo, force: boolean = false, skipCheck?: boolean): Promise<boolean> {
+        return await this._store(file, force, skipCheck, false);
+    }
+    async storeContent(path: FilePathWithPrefix, content: string): Promise<boolean> {
+        const blob = createTextBlob(content);
+        const bytes = (await blob.arrayBuffer()).byteLength;
+        const dummyUXFileInfo: UXFileInfo = {
+            name: path.split("/").pop() as string,
+            path: path,
+            stat: {
+                size: bytes,
+                ctime: Date.now(),
+                mtime: Date.now(),
+                type: "file",
+            },
+            body: blob,
+        }
+        return await this._store(dummyUXFileInfo, true, false, false);
+    }
+
+    async _store(file: UXFileInfo, force: boolean = false, skipCheck?: boolean, onlyChunks?: boolean): Promise<boolean> {
+        if (!skipCheck) {
+            if (!await this.checkIsTargetFile(file)) {
+                return true;
+            }
+        }
+        if (!file) {
+            this._log("File seems bad", LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        const path = getPathFromUXFileInfo(file);
+
+        const isPlain = isPlainText(file.name);
+        const possiblyLarge = !isPlain;
+        const content = file.body;
+        if (possiblyLarge) this._log(`Processing: ${path}`, LOG_LEVEL_VERBOSE);
+        const datatype = determineTypeFromBlob(content);
+        const fullPath = file.path;
+        const id = await this.core.$$path2id(fullPath);
+        const d: SavingEntry = {
+            _id: id,
+            path: file.path,
+            data: content,
+            ctime: file.stat.ctime,
+            mtime: file.stat.mtime,
+            size: file.stat.size,
+            children: [],
+            datatype: datatype,
+            type: datatype,
+            eden: {},
+        };
+        //upsert should locked
+        const msg = `STORAGE -> DB (${datatype}) `;
+        const isNotChanged = await serialized("file-" + fullPath, async () => {
+            // Commented out temporarily: this checks that the file was made ourself.
+            // if (this.core.storageAccess.recentlyTouched(file)) {
+            //     return true;
+            // }
+            try {
+                const old = await this.localDatabase.getDBEntry(fullPath, undefined, false, true, false);
+                if (old !== false) {
+                    const oldData = { data: old.data, deleted: old._deleted || old.deleted };
+                    const newData = { data: d.data, deleted: d._deleted || d.deleted };
+                    if (oldData.deleted != newData.deleted) return false;
+                    if (!await isDocContentSame(old.data, newData.data)) return false;
+                    this._log(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
+                    markChangesAreSame(old, d.mtime, old.mtime);
+                    return true;
+                    // d._rev = old._rev;
+                }
+            } catch (ex) {
+                if (force) {
+                    this._log(msg + "Error, Could not check the diff for the old one." + (force ? "force writing." : "") + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
+                } else {
+                    this._log(msg + "Error, Could not check the diff for the old one." + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
+                }
+                this._log(ex, LOG_LEVEL_VERBOSE);
+                return !force;
+            }
+            return false;
+        });
+        if (isNotChanged) {
+            this._log(msg + " Skip " + fullPath, LOG_LEVEL_VERBOSE);
+            return true;
+        }
+        const ret = await this.localDatabase.putDBEntry(d, onlyChunks);
+        if (ret !== false) {
+            this._log(msg + fullPath);
+            eventHub.emitEvent(EVENT_FILE_SAVED);
+        }
+        return ret != false;
+    }
+
+    async getConflictedRevs(file: UXFileInfoStub | FilePathWithPrefix): Promise<string[]> {
+        if (!await this.checkIsTargetFile(file)) {
+            return [];
+        }
+        const filename = getPathFromUXFileInfo(file);
+        const doc = await this.localDatabase.getDBEntryMeta(filename, { conflicts: true }, true);
+        if (doc === false) {
+            return [];
+        }
+        return doc._conflicts || [];
+    }
+
+    async fetch(file: UXFileInfoStub | FilePathWithPrefix,
+        rev?: string, waitForReady?: boolean, skipCheck = false): Promise<UXFileInfo | false> {
+        if (skipCheck && !await this.checkIsTargetFile(file)) {
+            return false;
+        }
+
+        const entry = await this.fetchEntry(file, rev, waitForReady, true);
+        if (entry === false) {
+            return false;
+        }
+        const data = createBlob(readContent(entry));
+        const fileInfo: UXFileInfo = {
+            name: entry.path.split("/").pop() as string,
+            path: entry.path,
+            stat: {
+                size: entry.size,
+                ctime: entry.ctime,
+                mtime: entry.mtime,
+                type: "file",
+            },
+            body: data,
+            deleted: entry.deleted || entry._deleted,
+        }
+        if (isInternalMetadata(entry.path)) {
+            fileInfo.isInternal = true;
+        }
+        return fileInfo;
+    }
+    async fetchEntryMeta(file: UXFileInfoStub | FilePathWithPrefix,
+        rev?: string, skipCheck = false): Promise<MetaEntry | false> {
+        const filename = getPathFromUXFileInfo(file);
+        if (skipCheck && !await this.checkIsTargetFile(file)) {
+            return false;
+        }
+
+        const doc = await this.localDatabase.getDBEntryMeta(
+            filename, rev ? { rev: rev } : undefined, true);
+        if (doc === false) {
+            return false;
+        }
+        return doc as MetaEntry;
+    }
+    async fetchEntryFromMeta(meta: MetaEntry, waitForReady: boolean = true, skipCheck = false): Promise<LoadedEntry | false> {
+        if (skipCheck && !await this.checkIsTargetFile(meta.path)) {
+            return false;
+        }
+        const doc = await this.localDatabase.getDBEntryFromMeta(meta as LoadedEntry, undefined, false, waitForReady, true);
+        if (doc === false) {
+            return false;
+        }
+        return doc;
+    }
+    async fetchEntry(file: UXFileInfoStub | FilePathWithPrefix,
+        rev?: string, waitForReady: boolean = true, skipCheck = false): Promise<LoadedEntry | false> {
+        if (skipCheck && !await this.checkIsTargetFile(file)) {
+            return false;
+        }
+        const entry = await this.fetchEntryMeta(file, rev, true);
+        if (entry === false) {
+            return false;
+        }
+        const doc = await this.fetchEntryFromMeta(entry, waitForReady, true);
+        return doc;
+    }
+    async deleteFromDBbyPath(fullPath: FilePath | FilePathWithPrefix, rev?: string): Promise<boolean> {
+        if (!await this.checkIsTargetFile(fullPath)) {
+            this._log(`storeFromStorage: File is not target: ${fullPath}`);
+            return true;
+        }
+        const opt = rev ? { rev: rev } : undefined;
+        const ret = await this.localDatabase.deleteDBEntry(fullPath, opt);
+        eventHub.emitEvent(EVENT_FILE_SAVED);
+        return ret;
+    }
+
+}

src/modules/core/ModuleFileHandler.ts

@@ -0,0 +1,370 @@
+import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
+import { serialized } from "octagonal-wheels/concurrency/lock";
+import type { FileEventItem } from "../../common/types";
+import type { FilePath, FilePathWithPrefix, MetaEntry, UXFileInfo, UXFileInfoStub, UXInternalFileInfoStub } from "../../lib/src/common/types";
+import { AbstractModule } from "../AbstractModule.ts";
+import { compareFileFreshness, EVEN, getPath, getPathFromUXFileInfo, getPathWithoutPrefix, markChangesAreSame } from "../../common/utils";
+import { getDocDataAsArray, isDocContentSame, readContent } from "../../lib/src/common/utils";
+import { shouldBeIgnored } from "../../lib/src/string_and_binary/path";
+import type { ICoreModule } from "../ModuleTypes";
+import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
+
+export class ModuleFileHandler extends AbstractModule implements ICoreModule {
+
+    get db() {
+        return this.core.databaseFileAccess;
+    }
+    get storage() {
+        return this.core.storageAccess;
+    }
+
+    $everyOnloadStart(): Promise<boolean> {
+        this.core.fileHandler = this;
+        return Promise.resolve(true);
+    }
+
+    async readFileFromStub(file: UXFileInfoStub | UXFileInfo) {
+        if ("body" in file && file.body) {
+            return file;
+        }
+        const readFile = await this.storage.readStubContent(file);
+        if (!readFile) {
+            throw new Error(`File ${file.path} is not exist on the storage`);
+        }
+        return readFile
+    }
+
+    async storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force: boolean = false, onlyChunks: boolean = false): Promise<boolean | undefined> {
+        const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
+        if (file == null) {
+            this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        // const file = item.args.file;
+        if (file.isInternal) {
+            this._log(`Internal file ${file.path} is not allowed to be processed on processFileEvent`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        // First, check the file on the database 
+        const entry = await this.db.fetchEntry(file, undefined, true, true);
+
+        if (!entry || entry.deleted || entry._deleted) {
+            // If the file is not exist on the database, then it should be created.
+            const readFile = await this.readFileFromStub(file);
+            if (!onlyChunks) {
+                return await this.db.store(readFile);
+            } else {
+                return true;
+            }
+        }
+        // I remember that it should be processed naturally. --> 
+
+        // // If the file is exist on the database, then it should be updated.
+        // // Check the file is already conflicted or not.
+        // const conflictedRevs = await this.db.getConflictedRevs(file);
+        // if (conflictedRevs.length > 0) {
+        //     // If conflicted, then it should be stored as new conflicted file.
+        //     const readFile = await this.readFileFromStub(file);
+        //     this.db.store(readFile, true);
+        //     return false;
+        // }
+        //< --
+
+        // entry is exist on the database, check the difference between the file and the entry.
+
+        let shouldApplied = false;
+        if (!force && !onlyChunks) {
+            // 1. if the time stamp is far different, then it should be updated.
+            // Note: This checks only the mtime with the resolution reduced to 2 seconds.
+            //       2 seconds it for the ZIP file's mtime. If not, we cannot backup the vault as the ZIP file.
+            //       This is hardcoded on `compareMtime` of `src/common/utils.ts`.
+            if (compareFileFreshness(file, entry) !== EVEN) {
+                shouldApplied = true;
+            }
+            // 2. if not, the content should be checked.
+            let readFile: UXFileInfo | undefined = undefined;
+            if (!shouldApplied) {
+                readFile = await this.readFileFromStub(file);
+                if (await isDocContentSame(getDocDataAsArray(entry.data), readFile.body)) {
+                    if (shouldApplied) {
+                        // Timestamp is different but the content is same. therefore, two timestamps should be handled as same.
+                        // So, mark the changes are same.
+                        markChangesAreSame(file, file.stat.mtime, entry.mtime);
+                    }
+                } else {
+                    shouldApplied = true;
+                }
+            }
+
+            if (!shouldApplied) {
+                this._log(`File ${file.path} is not changed`, LOG_LEVEL_VERBOSE);
+                return true;
+            }
+            if (!readFile) readFile = await this.readFileFromStub(file);
+            // If the file is changed, then the file should be stored.
+            if (onlyChunks) {
+                return await this.db.createChunks(readFile, false, true);
+            } else {
+                return await this.db.store(readFile, false, true);
+            }
+        } else {
+            // If force is true, then it should be updated.
+            const readFile = await this.readFileFromStub(file);
+            if (onlyChunks) {
+                return await this.db.createChunks(readFile, true, true);
+            } else {
+                return await this.db.store(readFile, true, true);
+            }
+        }
+    }
+
+    async deleteFileFromDB(info: UXFileInfoStub | UXInternalFileInfoStub | FilePath): Promise<boolean | undefined> {
+        const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
+        if (file == null) {
+            this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        // const file = item.args.file;
+        if (file.isInternal) {
+            this._log(`Internal file ${file.path} is not allowed to be processed on processFileEvent`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        // First, check the file on the database 
+        const entry = await this.db.fetchEntry(file, undefined, true, true);
+        if (!entry || entry.deleted || entry._deleted) {
+            this._log(`File ${file.path} is not exist or already deleted on the database`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        // Check the file is already conflicted. if so, only the conflicted one should be deleted.
+        const conflictedRevs = await this.db.getConflictedRevs(file);
+        if (conflictedRevs.length > 0) {
+            // If conflicted, then it should be deleted. entry._rev should be own file's rev.
+            // TODO: I BELIEVED SO. BUT I NOTICED THAT I AN NOT SURE. I SHOULD CHECK THIS.
+            //       ANYWAY, I SHOULD DELETE THE FILE. ACTUALLY WE SIMPLY DELETED THE FILE UNTIL PREVIOUS VERSIONS.
+            return await this.db.delete(file, entry._rev);
+        }
+        // Otherwise, the file should be deleted simply. This is the previous behaviour.
+        return await this.db.delete(file);
+    }
+
+    async deleteRevisionFromDB(info: UXFileInfoStub | FilePath | FilePathWithPrefix, rev: string): Promise<boolean | undefined> {
+        //TODO: Possibly check the conflicting.
+        return await this.db.delete(info, rev);
+    }
+
+
+    async resolveConflictedByDeletingRevision(info: UXFileInfoStub | FilePath, rev: string): Promise<boolean | undefined> {
+        if (!await this.deleteRevisionFromDB(info, rev)) {
+            this._log(`Failed to delete the conflicted revision ${rev} of ${getPathFromUXFileInfo(info)}`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        if (!await this.dbToStorageWithSpecificRev(info, rev, true)) {
+            this._log(`Failed to apply the resolved revision ${rev} of ${getPathFromUXFileInfo(info)} to the storage`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+    }
+
+    async dbToStorageWithSpecificRev(info: UXFileInfoStub | UXFileInfo | FilePath | null, rev: string, force?: boolean): Promise<boolean> {
+        const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
+        if (file == null) {
+            this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        const docEntry = await this.db.fetchEntryMeta(file, rev, true);
+        if (!docEntry) {
+            this._log(`File ${file.path} is not exist on the database`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        return await this.dbToStorage(docEntry, file, force);
+    }
+
+    async dbToStorage(entryInfo: MetaEntry | FilePathWithPrefix, info: UXFileInfoStub | UXFileInfo | FilePath | null, force?: boolean): Promise<boolean> {
+        const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
+        const mode = file == null ? "create" : "modify";
+
+        const docEntry = typeof entryInfo === "string" ?
+            await this.db.fetchEntryMeta(entryInfo, undefined, true) : await this.db.fetchEntryMeta(entryInfo.path, undefined, true);
+        if (!docEntry) {
+            this._log(`File ${entryInfo} is not exist on the database`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        const path = getPath(docEntry);
+
+        // 1. Check if it already conflicted.
+        const revs = await this.db.getConflictedRevs(path);
+        if (revs.length > 0) {
+            // Some conflicts are exist.
+            if (this.settings.writeDocumentsIfConflicted) {
+                // If configured to write the document even if conflicted, then it should be written.
+                // NO OP
+            } else {
+                // If not, then it should be checked. and will be processed later (i.e., after the conflict is resolved).
+                await this.core.$$queueConflictCheckIfOpen(path);
+                return true;
+            }
+        }
+
+        // 2. Check if the file is already exist on the storage.
+        const existDoc = this.storage.getStub(path);
+        if (existDoc && existDoc.isFolder) {
+            this._log(`Folder ${path} is already exist on the storage as a folder`, LOG_LEVEL_VERBOSE);
+            // We can do nothing, and other modules should also nothing to do.
+            return true;
+        }
+
+        // Check existence of both file and docEntry.
+        const existOnDB = !(docEntry._deleted || docEntry.deleted || false);
+        const existOnStorage = existDoc != null;
+        if (!existOnDB && !existOnStorage) {
+            this._log(`File ${path} seems to be deleted, but already not on storage`, LOG_LEVEL_VERBOSE);
+            return true;
+        }
+        if (!existOnDB && existOnStorage) {
+            // Deletion has been Transferred. Storage files will be deleted.
+            // Note: If the folder becomes empty, the folder will be deleted if not configured to keep it.
+            // This behaviour is implemented on the `ModuleFileAccessObsidian`.
+            // And it does not care actually deleted.
+            await this.storage.deleteVaultItem(path);
+            return true;
+        }
+        // Okay, the file is exist on the database. Let's check the file is exist on the storage.
+        const docRead = await this.db.fetchEntryFromMeta(docEntry);
+        if (!docRead) {
+            this._log(`File ${path} is not exist on the database`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        const docData = readContent(docRead);
+
+        if (!existOnStorage) {
+            // The file is not exist on the storage. We do not care about the differences.
+            await this.storage.ensureDir(path);
+            return await this.storage.writeFileAuto(path, docData, { ctime: docRead.ctime, mtime: docRead.mtime });
+        }
+        if (!force) {
+            // Ok, we have to compare.
+            let shouldApplied = false;
+            // 1. if the time stamp is far different, then it should be updated.
+            // Note: This checks only the mtime with the resolution reduced to 2 seconds.
+            //       2 seconds it for the ZIP file's mtime. If not, we cannot backup the vault as the ZIP file.
+            //       This is hardcoded on `compareMtime` of `src/common/utils.ts`.
+            if (compareFileFreshness(existDoc, docEntry) !== EVEN) {
+                shouldApplied = true;
+            }
+            // 2. if not, the content should be checked.
+
+            if (shouldApplied) {
+                const readFile = await this.readFileFromStub(existDoc);
+                if (await isDocContentSame(docData, readFile.body)) {
+                    // The content is same. So, we do not need to update the file.
+                    shouldApplied = false;
+                    // Timestamp is different but the content is same. therefore, two timestamps should be handled as same.
+                    // So, mark the changes are same.
+                    markChangesAreSame(docRead, docRead.mtime, existDoc.stat.mtime);
+                } else {
+                    shouldApplied = true;
+                }
+            }
+            if (!shouldApplied) {
+                this._log(`File ${docRead.path} is not changed`, LOG_LEVEL_VERBOSE);
+                return true;
+            }
+            // Let's apply the changes.
+        }
+        await this.storage.ensureDir(path);
+        const ret = await this.storage.writeFileAuto(path, docData, { ctime: docRead.ctime, mtime: docRead.mtime });
+        this.storage.touched(path);
+        this.storage.triggerFileEvent(mode, path);
+        return ret;
+    }
+
+    async $anyHandlerProcessesFileEvent(item: FileEventItem): Promise<boolean | undefined> {
+        const eventItem = item.args;
+        const type = item.type;
+        const path = eventItem.file.path;
+        if (!await this.core.$$isTargetFile(path)) {
+            this._log(`File ${path} is not the target file`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        if (shouldBeIgnored(path)) {
+            this._log(`File ${path} should be ignored`, LOG_LEVEL_VERBOSE);
+            return false;
+        }
+        const lockKey = `processFileEvent-${path}`;
+        return await serialized(lockKey, async () => {
+            switch (type) {
+                case "CREATE":
+                case "CHANGED":
+                    return await this.storeFileToDB(item.args.file);
+                case "DELETE":
+                    return await this.deleteFileFromDB(item.args.file);
+                case "INTERNAL":
+                    // this should be handled on the other module.
+                    return false;
+                default:
+                    this._log(`Unsupported event type: ${type}`, LOG_LEVEL_VERBOSE);
+                    return false;
+            }
+        });
+    }
+
+    async $anyProcessReplicatedDoc(entry: MetaEntry): Promise<boolean | undefined> {
+        return await serialized(entry.path, async () => {
+            if (!await this.core.$$isTargetFile(entry.path)) {
+                this._log(`File ${entry.path} is not the target file`, LOG_LEVEL_VERBOSE);
+                return false;
+            }
+            if (shouldBeIgnored(entry.path)) {
+                this._log(`File ${entry.path} should be ignored`, LOG_LEVEL_VERBOSE);
+                return false;
+            }
+            const path = getPath(entry);
+
+            const targetFile = this.storage.getStub(getPathWithoutPrefix(entry));
+            if (targetFile && targetFile.isFolder) {
+                this._log(`${getPath(entry)} is already exist as the folder`);
+                // Nothing to do and other modules should also nothing to do.
+                return true;
+            } else {
+                this._log(`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`, LOG_LEVEL_VERBOSE);
+                const ret = await this.dbToStorage(entry, targetFile);
+                this._log(`Processing ${path} (${entry._id.substring(0, 8)} :${entry._rev?.substring(0, 5)}) : Done`);
+                return ret;
+            }
+        });
+
+    }
+
+    async createAllChunks(showingNotice?: boolean): Promise<void> {
+        this._log("Collecting local files on the storage", LOG_LEVEL_VERBOSE);
+        const semaphore = Semaphore(10);
+
+        let processed = 0;
+        const filesStorageSrc = this.storage.getFiles();
+        const incProcessed = () => {
+            processed++;
+            if (processed % 25 == 0) this._log(`Creating missing chunks: ${processed} of ${total} files`, showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO, "chunkCreation");
+        }
+        const total = filesStorageSrc.length;
+        const procAllChunks = filesStorageSrc.map(async (file) => {
+            if (!await this.core.$$isTargetFile(file)) {
+                incProcessed();
+                return true;
+            }
+            if (shouldBeIgnored(file.path)) {
+                incProcessed();
+                return true;
+            }
+            const release = await semaphore.acquire();
+            incProcessed();
+            try {
+                await this.storeFileToDB(file, false, true);
+            } catch (ex) {
+                this._log(ex, LOG_LEVEL_VERBOSE);
+            } finally {
+                release();
+            }
+        });
+        await Promise.all(procAllChunks);
+        this._log(`Creating chunks Done: ${processed} of ${total} files`, showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO, "chunkCreation");
+    }
+}

src/modules/core/ModuleLocalDatabaseObsidian.ts

@@ -0,0 +1,23 @@
+import { $f } from "../../lib/src/common/i18n";
+import { LiveSyncLocalDB } from "../../lib/src/pouchdb/LiveSyncLocalDB.ts";
+import { initializeStores } from "../../common/stores.ts";
+import { AbstractModule } from "../AbstractModule.ts";
+import type { ICoreModule } from "../ModuleTypes.ts";
+
+export class ModuleLocalDatabaseObsidian extends AbstractModule implements ICoreModule {
+
+    $everyOnloadStart(): Promise<boolean> {
+        return Promise.resolve(true);
+    }
+    async $$openDatabase(): Promise<boolean> {
+        if (this.localDatabase != null) {
+            await this.localDatabase.close();
+        }
+        const vaultName = this.core.$$getVaultName();
+        this._log($f`Waiting for ready...`);
+        this.core.localDatabase = new LiveSyncLocalDB(vaultName, this.core);
+        initializeStores(vaultName);
+        return await this.localDatabase.initializeDatabase();
+    }
+
+}

src/modules/core/ModulePeriodicProcess.ts

@@ -0,0 +1,35 @@
+import { PeriodicProcessor } from "../../common/utils";
+import { AbstractModule } from "../AbstractModule";
+import type { ICoreModule } from "../ModuleTypes";
+
+export class ModulePeriodicProcess extends AbstractModule implements ICoreModule {
+
+    periodicSyncProcessor = new PeriodicProcessor(this.core, async () => await this.core.$$replicate());
+
+    _disablePeriodic() {
+        this.periodicSyncProcessor?.disable();
+        return Promise.resolve(true);
+    }
+    _resumePeriodic() {
+        this.periodicSyncProcessor.enable(this.settings.periodicReplication ? this.settings.periodicReplicationInterval * 1000 : 0);
+        return Promise.resolve(true);
+    }
+    $allOnUnload() {
+        return this._disablePeriodic();
+
+    }
+    $everyBeforeRealizeSetting(): Promise<boolean> {
+        return this._disablePeriodic();
+
+    }
+    $everyBeforeSuspendProcess(): Promise<boolean> {
+        return this._disablePeriodic();
+
+    }
+    $everyAfterResumeProcess(): Promise<boolean> {
+        return this._resumePeriodic();
+    }
+    $everyAfterRealizeSetting(): Promise<boolean> {
+        return this._resumePeriodic();
+    }
+}

src/modules/core/ModulePouchDB.ts

@@ -0,0 +1,16 @@
+import { AbstractModule } from "../AbstractModule";
+import type { ICoreModule } from "../ModuleTypes";
+import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser";
+
+export class ModulePouchDB extends AbstractModule implements ICoreModule {
+    $$createPouchDBInstance<T extends object>(name?: string, options?: PouchDB.Configuration.DatabaseConfiguration): PouchDB.Database<T> {
+        const optionPass = options ?? {};
+        if (this.settings.useIndexedDBAdapter) {
+            optionPass.adapter = "indexeddb";
+            //@ts-ignore :missing def
+            optionPass.purged_infos_limit = 1;
+            return new PouchDB(name + "-indexeddb", optionPass);
+        }
+        return new PouchDB(name, optionPass);
+    }
+}

src/modules/core/ModuleRebuilder.ts

@@ -0,0 +1,216 @@
+import { delay } from "octagonal-wheels/promises";
+import { FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, REMOTE_COUCHDB, REMOTE_MINIO } from "../../lib/src/common/types.ts";
+import { AbstractModule } from "../AbstractModule.ts";
+import type { Rebuilder } from "../interfaces/DatabaseRebuilder.ts";
+import type { ICoreModule } from "../ModuleTypes.ts";
+import type { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator.ts";
+import { fetchAllUsedChunks } from "../../lib/src/pouchdb/utils_couchdb.ts";
+
+export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebuilder {
+
+    $everyOnload(): Promise<boolean> {
+        this.core.rebuilder = this;
+        return Promise.resolve(true);
+    }
+    async $performRebuildDB(method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks"): Promise<void> {
+        if (method == "localOnly") {
+            await this.$fetchLocal();
+        }
+        if (method == "localOnlyWithChunks") {
+            await this.$fetchLocal(true);
+        }
+        if (method == "remoteOnly") {
+            await this.$rebuildRemote();
+        }
+        if (method == "rebuildBothByThisDevice") {
+            await this.$rebuildEverything();
+        }
+    }
+
+    async rebuildRemote() {
+        await this.core.$allSuspendExtraSync();
+        this.core.settings.isConfigured = true;
+
+        await this.core.realizeSettingSyncMode();
+        await this.core.$$markRemoteLocked();
+        await this.core.$$tryResetRemoteDatabase();
+        await this.core.$$markRemoteLocked();
+        await delay(500);
+        await this.core.$allAskUsingOptionalSyncFeature({ enableOverwrite: true });
+        await delay(1000);
+        await this.core.$$replicateAllToServer(true);
+        await delay(1000);
+        await this.core.$$replicateAllToServer(true);
+    }
+    $rebuildRemote(): Promise<void> {
+        return this.rebuildRemote();
+    }
+
+
+    async rebuildEverything() {
+        await this.core.$allSuspendExtraSync();
+        await this.askUseNewAdapter();
+        this.core.settings.isConfigured = true;
+        await this.core.realizeSettingSyncMode();
+        await this.resetLocalDatabase();
+        await delay(1000);
+        await this.core.$$initializeDatabase(true);
+        await this.core.$$markRemoteLocked();
+        await this.core.$$tryResetRemoteDatabase();
+        await this.core.$$markRemoteLocked();
+        await delay(500);
+        await this.core.$allAskUsingOptionalSyncFeature({ enableOverwrite: true });
+        await delay(1000);
+        await this.core.$$replicateAllToServer(true);
+        await delay(1000);
+        await this.core.$$replicateAllToServer(true);
+
+    }
+
+    $rebuildEverything(): Promise<void> {
+        return this.rebuildEverything();
+    }
+
+    $fetchLocal(makeLocalChunkBeforeSync?: boolean): Promise<void> {
+        return this.fetchLocal(makeLocalChunkBeforeSync);
+    }
+
+    async scheduleRebuild(): Promise<void> {
+        try {
+            await this.core.storageAccess.writeFileAuto(FLAGMD_REDFLAG2_HR, "");
+        } catch (ex) {
+            this._log("Could not create red_flag_rebuild.md", LOG_LEVEL_NOTICE);
+            this._log(ex, LOG_LEVEL_VERBOSE);
+        }
+        this.core.$$performRestart();
+    }
+    async scheduleFetch(): Promise<void> {
+        try {
+            await this.core.storageAccess.writeFileAuto(FLAGMD_REDFLAG3_HR, "");
+        } catch (ex) {
+            this._log("Could not create red_flag_fetch.md", LOG_LEVEL_NOTICE);
+            this._log(ex, LOG_LEVEL_VERBOSE);
+        }
+        this.core.$$performRestart();
+    }
+
+    async $$tryResetRemoteDatabase(): Promise<void> {
+        await this.core.replicator.tryResetRemoteDatabase(this.settings);
+    }
+
+    async $$tryCreateRemoteDatabase(): Promise<void> {
+        await this.core.replicator.tryCreateRemoteDatabase(this.settings);
+    }
+
+    async $$resetLocalDatabase(): Promise<void> {
+        this.core.storageAccess.clearTouched();
+        await this.localDatabase.resetDatabase();
+    }
+
+
+    async suspendAllSync() {
+        this.core.settings.liveSync = false;
+        this.core.settings.periodicReplication = false;
+        this.core.settings.syncOnSave = false;
+        this.core.settings.syncOnEditorSave = false;
+        this.core.settings.syncOnStart = false;
+        this.core.settings.syncOnFileOpen = false;
+        this.core.settings.syncAfterMerge = false;
+        await this.core.$allSuspendExtraSync();
+    }
+    async suspendReflectingDatabase() {
+        if (this.core.settings.doNotSuspendOnFetching) return;
+        if (this.core.settings.remoteType == REMOTE_MINIO) return;
+        this._log(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE);
+        this.core.settings.suspendParseReplicationResult = true;
+        this.core.settings.suspendFileWatching = true;
+        await this.core.saveSettings();
+    }
+    async resumeReflectingDatabase() {
+        if (this.core.settings.doNotSuspendOnFetching) return;
+        if (this.core.settings.remoteType == REMOTE_MINIO) return;
+        this._log(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE);
+        this.core.settings.suspendParseReplicationResult = false;
+        this.core.settings.suspendFileWatching = false;
+        await this.core.$$performFullScan(true);
+        await this.core.$everyBeforeReplicate(false); //TODO: Check actual need of this.
+        await this.core.saveSettings();
+
+    }
+    async askUseNewAdapter() {
+        if (!this.core.settings.useIndexedDBAdapter) {
+            const message = `Now this core has been configured to use the old database adapter for keeping compatibility. Do you want to deactivate it?`;
+            const CHOICE_YES = "Yes, disable and use latest";
+            const CHOICE_NO = "No, keep compatibility";
+            const choices = [CHOICE_YES, CHOICE_NO];
+
+            const ret = await this.core.confirm.confirmWithMessage("Database adapter", message, choices, CHOICE_YES, 10);
+            if (ret == CHOICE_YES) {
+                this.core.settings.useIndexedDBAdapter = true;
+            }
+        }
+    }
+    async fetchLocal(makeLocalChunkBeforeSync?: boolean) {
+        await this.core.$allSuspendExtraSync();
+        await this.askUseNewAdapter();
+        this.core.settings.isConfigured = true;
+        await this.suspendReflectingDatabase();
+        await this.core.realizeSettingSyncMode();
+        await this.resetLocalDatabase();
+        await delay(1000);
+        await this.core.$$openDatabase();
+        this.core.isReady = true;
+        if (makeLocalChunkBeforeSync) {
+            await this.core.fileHandler.createAllChunks(true);
+        }
+        await this.core.$$markRemoteResolved();
+        await delay(500);
+        await this.core.$$replicateAllFromServer(true);
+        await delay(1000);
+        await this.core.$$replicateAllFromServer(true);
+        await this.resumeReflectingDatabase();
+        await this.core.$allAskUsingOptionalSyncFeature({ enableFetch: true });
+    }
+    async fetchLocalWithRebuild() {
+        return await this.fetchLocal(true);
+    }
+
+    async $allSuspendAllSync(): Promise<boolean> {
+        await this.suspendAllSync();
+        return true;
+    }
+
+    async resetLocalDatabase() {
+        if (this.core.settings.isConfigured && this.core.settings.additionalSuffixOfDatabaseName == "") {
+            // Discard the non-suffixed database
+            await this.core.$$resetLocalDatabase();
+        }
+        const suffix = (await this.core.$anyGetAppId()) || "";
+        this.core.settings.additionalSuffixOfDatabaseName = suffix;
+        await this.core.$$resetLocalDatabase();
+    }
+    async fetchRemoteChunks() {
+        if (!this.core.settings.doNotSuspendOnFetching && this.core.settings.readChunksOnline && this.core.settings.remoteType == REMOTE_COUCHDB) {
+            this._log(`Fetching chunks`, LOG_LEVEL_NOTICE);
+            const replicator = this.core.getReplicator() as LiveSyncCouchDBReplicator;
+            const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.core.getIsMobile(), true);
+            if (typeof remoteDB == "string") {
+                this._log(remoteDB, LOG_LEVEL_NOTICE);
+            } else {
+                await fetchAllUsedChunks(this.localDatabase.localDatabase, remoteDB.db);
+            }
+            this._log(`Fetching chunks done`, LOG_LEVEL_NOTICE);
+        }
+    }
+    async resolveAllConflictedFilesByNewerOnes() {
+        this._log(`Resolving conflicts by newer ones`, LOG_LEVEL_NOTICE);
+        const files = this.core.storageAccess.getFileNames();
+
+        let i = 0;
+        for (const file of files) {
+            if (i++ % 10) this._log(`Check and Processing ${i} / ${files.length}`, LOG_LEVEL_NOTICE, "resolveAllConflictedFilesByNewerOnes");
+            await this.core.$anyResolveConflictByNewest(file);
+        }
+        this._log(`Done!`, LOG_LEVEL_NOTICE, "resolveAllConflictedFilesByNewerOnes");
+    }
+}

src/modules/core/ModuleReplicator.ts

@@ -0,0 +1,343 @@
+import { fireAndForget, yieldMicrotask } from "octagonal-wheels/promises";
+import type { LiveSyncLocalDB } from "../../lib/src/pouchdb/LiveSyncLocalDB";
+import { AbstractModule } from "../AbstractModule";
+import type { ICoreModule } from "../ModuleTypes";
+import { Logger, LOG_LEVEL_NOTICE, LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
+import { isLockAcquired, shareRunningResult, skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
+import { purgeUnreferencedChunks, balanceChunkPurgedDBs } from "../../lib/src/pouchdb/utils_couchdb";
+import { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator";
+import { throttle } from "octagonal-wheels/function";
+import { arrayToChunkedArray } from "octagonal-wheels/collection";
+import { SYNCINFO_ID, VER, type EntryBody, type EntryDoc, type LoadedEntry, type MetaEntry } from "../../lib/src/common/types";
+import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
+import { getPath, isChunk, isValidPath, scheduleTask } from "../../common/utils";
+import { sendValue } from "octagonal-wheels/messagepassing/signal";
+import { isAnyNote } from "../../lib/src/common/utils";
+import { EVENT_FILE_SAVED, eventHub } from "../../common/events";
+
+export class ModuleReplicator extends AbstractModule implements ICoreModule {
+
+    $everyOnloadAfterLoadSettings(): Promise<boolean> {
+        eventHub.onEvent(EVENT_FILE_SAVED, () => {
+            if (this.settings.syncOnSave && !this.core.suspended) {
+                scheduleTask("perform-replicate-after-save", 250, () => this.core.$$waitForReplicationOnce());
+            }
+        })
+        return Promise.resolve(true);
+    }
+
+    async setReplicator() {
+        const replicator = await this.core.$anyNewReplicator();
+        if (!replicator) {
+            this._log("No replicator is available, this is the fatal error.", LOG_LEVEL_NOTICE);
+            return false
+        }
+        this.core.replicator = replicator;
+        await yieldMicrotask();
+        return true;
+    }
+    $everyOnInitializeDatabase(db: LiveSyncLocalDB): Promise<boolean> {
+        return this.setReplicator();
+    }
+
+    $everyOnResetDatabase(db: LiveSyncLocalDB): Promise<boolean> {
+        return this.setReplicator();
+
+    }
+
+    async $everyBeforeReplicate(showMessage: boolean): Promise<boolean> {
+        await this.loadQueuedFiles();
+        return true;
+    }
+    async $$replicate(showMessage: boolean = false): Promise<boolean | void> {
+        //--? 
+        if (!this.core.isReady) return;
+        if (isLockAcquired("cleanup")) {
+            Logger("Database cleaning up is in process. replication has been cancelled", LOG_LEVEL_NOTICE);
+            return;
+        }
+        if (this.settings.versionUpFlash != "") {
+            Logger("Open settings and check message, please. replication has been cancelled.", LOG_LEVEL_NOTICE);
+            return;
+        }
+        if (!await this.core.$everyCommitPendingFileEvent()) {
+            Logger("Some file events are pending. Replication has been cancelled.", LOG_LEVEL_NOTICE);
+            return false;
+        }
+        if (!await this.core.$everyBeforeReplicate(showMessage)) {
+            Logger(`Replication has been cancelled by some module failure`, LOG_LEVEL_NOTICE);
+            return false;
+        }
+
+        //<-- Here could be an module.
+        const ret = await this.core.replicator.openReplication(this.settings, false, showMessage, false);
+        if (!ret) {
+            if (this.core.replicator.tweakSettingsMismatched) {
+                await this.core.$$askResolvingMismatchedTweaks();
+
+            } else {
+                if (this.core.replicator?.remoteLockedAndDeviceNotAccepted) {
+                    if (this.core.replicator.remoteCleaned && this.settings.useIndexedDBAdapter) {
+                        Logger(`The remote database has been cleaned.`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
+                        await skipIfDuplicated("cleanup", async () => {
+                            const count = await purgeUnreferencedChunks(this.localDatabase.localDatabase, true);
+                            const message = `The remote database has been cleaned up.
+To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device.
+However, If there are many chunks to be deleted, maybe fetching again is faster.
+We will lose the history of this device if we fetch the remote database again.
+Even if you choose to clean up, you will see this option again if you exit Obsidian and then synchronise again.`
+                            const CHOICE_FETCH = "Fetch again";
+                            const CHOICE_CLEAN = "Cleanup";
+                            const CHOICE_DISMISS = "Dismiss";
+                            const ret = await this.core.confirm.confirmWithMessage("Cleaned", message, [
+                                CHOICE_FETCH,
+                                CHOICE_CLEAN,
+                                CHOICE_DISMISS], CHOICE_DISMISS, 30);
+                            if (ret == CHOICE_FETCH) {
+                                await this.core.rebuilder.$performRebuildDB("localOnly");
+                            }
+                            if (ret == CHOICE_CLEAN) {
+                                const replicator = this.core.getReplicator();
+                                if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
+                                const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.core.getIsMobile(), true);
+                                if (typeof remoteDB == "string") {
+                                    Logger(remoteDB, LOG_LEVEL_NOTICE);
+                                    return false;
+                                }
+
+                                await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
+                                this.localDatabase.hashCaches.clear();
+                                // Perform the synchronisation once.
+                                if (await this.core.replicator.openReplication(this.settings, false, showMessage, true)) {
+                                    await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
+                                    await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
+                                    this.localDatabase.hashCaches.clear();
+                                    await this.core.getReplicator().markRemoteResolved(this.settings);
+                                    Logger("The local database has been cleaned up.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO)
+                                } else {
+                                    Logger("Replication has been cancelled. Please try it again.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO)
+                                }
+
+                            }
+                        });
+                    } else {
+                        const message = `
+The remote database has been rebuilt.
+To synchronize, this device must fetch everything again once.
+Or if you are sure know what had been happened, we can unlock the database from the setting dialog.
+                    `
+                        const CHOICE_FETCH = "Fetch again";
+                        const CHOICE_DISMISS = "Dismiss";
+                        const ret = await this.core.confirm.confirmWithMessage("Locked", message, [
+                            CHOICE_FETCH,
+                            CHOICE_DISMISS], CHOICE_DISMISS, 10);
+                        if (ret == CHOICE_FETCH) {
+                            const CHOICE_RESTART = "Restart";
+                            const CHOICE_WITHOUT_RESTART = "Without restart";
+                            if (await this.core.confirm.askSelectStringDialogue(
+                                "Self-hosted LiveSync restarts Obsidian to fetch everything safely. However, you can do it without restarting. Please choose one.",
+                                [CHOICE_RESTART, CHOICE_WITHOUT_RESTART], {
+                                title: "Fetch again",
+                                defaultAction: CHOICE_RESTART,
+                                timeout: 30,
+                            }
+                            ) == CHOICE_RESTART) {
+                                await this.core.rebuilder.scheduleFetch();
+                                // await this.core.$$scheduleAppReload();
+                                return;
+                            } else {
+                                await this.core.rebuilder.$performRebuildDB("localOnly");
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return ret;
+    }
+
+    $$parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): void {
+        if (this.settings.suspendParseReplicationResult && !this.replicationResultProcessor.isSuspended) {
+            this.replicationResultProcessor.suspend()
+        }
+        this.replicationResultProcessor.enqueueAll(docs);
+        if (!this.settings.suspendParseReplicationResult && this.replicationResultProcessor.isSuspended) {
+            this.replicationResultProcessor.resume()
+        }
+    }
+    _saveQueuedFiles = throttle(() => {
+        const saveData = this.replicationResultProcessor._queue.filter(e => e !== undefined && e !== null).map((e) => e?._id ?? "" as string) as string[];
+        const kvDBKey = "queued-files"
+        // localStorage.setItem(lsKey, saveData);
+        fireAndForget(() => this.core.kvDB.set(kvDBKey, saveData));
+    }, 100);
+    saveQueuedFiles() {
+        this._saveQueuedFiles();
+    }
+    async loadQueuedFiles() {
+        if (this.settings.suspendParseReplicationResult) return;
+        if (!this.settings.isConfigured) return;
+        const kvDBKey = "queued-files"
+        // const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[];
+        const ids = [...new Set(await this.core.kvDB.get<string[]>(kvDBKey) ?? [])];
+        const batchSize = 100;
+        const chunkedIds = arrayToChunkedArray(ids, batchSize);
+        for await (const idsBatch of chunkedIds) {
+            const ret = await this.localDatabase.allDocsRaw<EntryDoc>({
+                keys: idsBatch,
+                include_docs: true,
+                limit: 100
+            });
+            const docs = ret.rows.filter(e => e.doc).map(e => e.doc) as PouchDB.Core.ExistingDocument<EntryDoc>[];
+            const errors = ret.rows.filter(e => !e.doc && !e.value.deleted);
+            if (errors.length > 0) {
+                Logger("Some queued processes were not resurrected");
+                Logger(JSON.stringify(errors), LOG_LEVEL_VERBOSE);
+            }
+            this.replicationResultProcessor.enqueueAll(docs);
+            await this.replicationResultProcessor.waitForAllProcessed();
+        }
+
+    }
+
+    replicationResultProcessor = new QueueProcessor(async (docs: PouchDB.Core.ExistingDocument<EntryDoc>[]) => {
+        if (this.settings.suspendParseReplicationResult) return;
+        const change = docs[0];
+        if (!change) return;
+        if (isChunk(change._id)) {
+            // SendSignal?
+            // this.parseIncomingChunk(change);
+            sendValue(`leaf-${change._id}`, change);
+            return;
+        }
+        if (await this.core.$anyModuleParsedReplicationResultItem(change)) return;
+        // any addon needs this item?
+        // for (const proc of this.core.addOns) {
+        //     if (await proc.parseReplicationResultItem(change)) {
+        //         return;
+        //     }
+        // }
+        if (change.type == "versioninfo") {
+            if (change.version > VER) {
+                this.core.replicator.closeReplication();
+                Logger(`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`, LOG_LEVEL_NOTICE);
+            }
+            return;
+        }
+        if (change._id == SYNCINFO_ID || // Synchronisation information data
+            change._id.startsWith("_design") //design document
+        ) {
+            return;
+        }
+        if (isAnyNote(change)) {
+            if (this.databaseQueuedProcessor._isSuspended) {
+                Logger(`Processing scheduled: ${change.path}`, LOG_LEVEL_INFO);
+            }
+            const size = change.size;
+            if (this.core.$$isFileSizeExceeded(size)) {
+                Logger(`Processing ${change.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
+                return;
+            }
+            this.databaseQueuedProcessor.enqueue(change);
+        }
+        return;
+    }, {
+        batchSize: 1,
+        suspended: true,
+        concurrentLimit: 100,
+        delay: 0,
+        totalRemainingReactiveSource: this.core.replicationResultCount
+    }).replaceEnqueueProcessor((queue, newItem) => {
+        const q = queue.filter(e => e._id != newItem._id);
+        return [...q, newItem];
+    }).startPipeline().onUpdateProgress(() => {
+        this.saveQueuedFiles();
+    });
+
+    databaseQueuedProcessor = new QueueProcessor(async (docs: EntryBody[]) => {
+        const dbDoc = docs[0] as LoadedEntry; // It has no `data`
+        const path = getPath(dbDoc);
+        // If `Read chunks online` is disabled, chunks should be transferred before here.
+        // However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
+        const doc = await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, {}, false, true, true);
+        if (!doc) {
+            Logger(`Something went wrong while gathering content of ${path} (${dbDoc._id.substring(0, 8)}, ${dbDoc._rev?.substring(0, 10)}) `, LOG_LEVEL_NOTICE)
+            return;
+        }
+
+        if (await this.core.$anyProcessOptionalSyncFiles(dbDoc)) {
+            // Already processed
+        } else if (isValidPath(getPath(doc))) {
+            this.storageApplyingProcessor.enqueue(doc as MetaEntry);
+        } else {
+            Logger(`Skipped: ${doc._id.substring(0, 8)}`, LOG_LEVEL_VERBOSE);
+        }
+        return;
+    }, {
+        suspended: true,
+        batchSize: 1,
+        concurrentLimit: 10,
+        yieldThreshold: 1,
+        delay: 0,
+        totalRemainingReactiveSource: this.core.databaseQueueCount
+    }).replaceEnqueueProcessor((queue, newItem) => {
+        const q = queue.filter(e => e._id != newItem._id);
+        return [...q, newItem];
+    }).startPipeline();
+
+
+    storageApplyingProcessor = new QueueProcessor(async (docs: MetaEntry[]) => {
+        const entry = docs[0];
+        await this.core.$anyProcessReplicatedDoc(entry);
+        return;
+    }, {
+        suspended: true,
+        batchSize: 1,
+        concurrentLimit: 6,
+        yieldThreshold: 1,
+        delay: 0,
+        totalRemainingReactiveSource: this.core.storageApplyingCount
+    }).replaceEnqueueProcessor((queue, newItem) => {
+        const q = queue.filter(e => e._id != newItem._id);
+        return [...q, newItem];
+    }).startPipeline()
+
+    $everyBeforeSuspendProcess(): Promise<boolean> {
+        this.core.replicator.closeReplication();
+        return Promise.resolve(true);
+    }
+
+    async $$replicateAllToServer(showingNotice: boolean = false): Promise<boolean> {
+        if (!this.core.isReady) return false;
+        if (!await this.core.$everyBeforeReplicate(showingNotice)) {
+            Logger(`Replication has been cancelled by some module failure`, LOG_LEVEL_NOTICE);
+            return false;
+        }
+        if (this.core.replicator instanceof LiveSyncCouchDBReplicator) {
+            if (await this.core.confirm.askYesNoDialog("Do you want to send all chunks before replication?", { defaultOption: "No", timeout: 20 }) == "yes") {
+                await this.core.replicator.sendChunks(this.core.settings, undefined, true, 0);
+            }
+        }
+        const ret = await this.core.replicator.replicateAllToServer(this.settings, showingNotice);
+        if (ret) return true;
+        const checkResult = await this.core.$anyAfterConnectCheckFailed();
+        if (checkResult == "CHECKAGAIN") return await this.core.$$replicateAllToServer(showingNotice);
+        return !checkResult;
+    }
+    async $$replicateAllFromServer(showingNotice: boolean = false): Promise<boolean> {
+        if (!this.core.isReady) return false;
+        const ret = await this.core.replicator.replicateAllFromServer(this.settings, showingNotice);
+        if (ret) return true;
+        const checkResult = await this.core.$anyAfterConnectCheckFailed();
+        if (checkResult == "CHECKAGAIN") return await this.core.$$replicateAllFromServer(showingNotice);
+        return !checkResult;
+    }
+
+    async $$waitForReplicationOnce(): Promise<boolean | void> {
+        return await shareRunningResult(`replication`, () => this.core.$$replicate());
+    }
+
+
+
+}

src/modules/core/ModuleReplicatorCouchDB.ts

@@ -0,0 +1,33 @@
+import { fireAndForget } from "octagonal-wheels/promises";
+import { REMOTE_MINIO, type RemoteDBSettings } from "../../lib/src/common/types";
+import { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator";
+import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
+import { AbstractModule } from "../AbstractModule";
+import type { ICoreModule } from "../ModuleTypes";
+
+export class ModuleReplicatorCouchDB extends AbstractModule implements ICoreModule {
+    $anyNewReplicator(settingOverride: Partial<RemoteDBSettings> = {}): Promise<LiveSyncAbstractReplicator> {
+        const settings = { ...this.settings, ...settingOverride };
+        // If new remote types were added, add them here. Do not use `REMOTE_COUCHDB` directly for the safety valve.
+        if (settings.remoteType == REMOTE_MINIO) {
+            return undefined!
+        }
+        return Promise.resolve(new LiveSyncCouchDBReplicator(this.core));
+
+    }
+    $everyAfterResumeProcess(): Promise<boolean> {
+        if (this.settings.remoteType != REMOTE_MINIO) {
+            // If LiveSync enabled, open replication
+            if (this.settings.liveSync) {
+                fireAndForget(() => this.core.replicator.openReplication(this.settings, true, false, false));
+            }
+            // If sync on start enabled, open replication
+            if (!this.settings.liveSync && this.settings.syncOnStart) {
+                // Possibly ok as if only share the result
+                fireAndForget(() => this.core.replicator.openReplication(this.settings, false, false, false));
+            }
+        }
+
+        return Promise.resolve(true);
+    }
+}

src/modules/core/ModuleReplicatorMinIO.ts

@@ -0,0 +1,17 @@
+import { REMOTE_MINIO, type RemoteDBSettings } from "../../lib/src/common/types";
+import { LiveSyncJournalReplicator } from "../../lib/src/replication/journal/LiveSyncJournalReplicator";
+import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
+import { AbstractModule } from "../AbstractModule";
+import type { ICoreModule } from "../ModuleTypes";
+
+export class ModuleReplicatorMinIO extends AbstractModule implements ICoreModule {
+    $anyNewReplicator(settingOverride: Partial<RemoteDBSettings> = {}): Promise<LiveSyncAbstractReplicator> {
+        const settings = { ...this.settings, ...settingOverride };
+        if (settings.remoteType == REMOTE_MINIO) {
+            return Promise.resolve(new LiveSyncJournalReplicator(this.core));
+        }
+        return undefined!;
+    }
+
+
+}

src/modules/core/ModuleTargetFilter.ts

@@ -0,0 +1,146 @@
+import { LRUCache } from "octagonal-wheels/memory/LRUCache";
+import { getPathFromUXFileInfo, id2path, isInternalMetadata, path2id, stripInternalMetadataPrefix, useMemo } from "../../common/utils";
+import { LOG_LEVEL_VERBOSE, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix, type ObsidianLiveSyncSettings, type UXFileInfoStub } from "../../lib/src/common/types";
+import { addPrefix, isAcceptedAll, stripAllPrefixes } from "../../lib/src/string_and_binary/path";
+import { AbstractModule } from "../AbstractModule";
+import type { ICoreModule } from "../ModuleTypes";
+import { EVENT_REQUEST_RELOAD_SETTING_TAB, EVENT_SETTING_SAVED, eventHub } from "../../common/events";
+import { isDirty } from "../../lib/src/common/utils";
+export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
+
+    reloadIgnoreFiles() {
+        this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
+
+    }
+    $everyOnload(): Promise<boolean> {
+        eventHub.onEvent(EVENT_SETTING_SAVED, (evt: CustomEvent<ObsidianLiveSyncSettings>) => {
+            this.reloadIgnoreFiles();
+        });
+        eventHub.onEvent(EVENT_REQUEST_RELOAD_SETTING_TAB, (evt: CustomEvent<ObsidianLiveSyncSettings>) => {
+            this.reloadIgnoreFiles();
+        });
+        return Promise.resolve(true);
+    }
+
+    $$id2path(id: DocumentID, entry?: EntryHasPath, stripPrefix?: boolean): FilePathWithPrefix {
+        const tempId = id2path(id, entry);
+        if (stripPrefix && isInternalMetadata(tempId)) {
+            const out = stripInternalMetadataPrefix(tempId);
+            return out;
+        }
+        return tempId;
+    }
+    async $$path2id(filename: FilePathWithPrefix | FilePath, prefix?: string): Promise<DocumentID> {
+        const destPath = addPrefix(filename, prefix ?? "");
+        return await path2id(destPath, this.settings.usePathObfuscation ? this.settings.passphrase : "", !this.settings.handleFilenameCaseSensitive);
+    }
+
+
+    $$isFileSizeExceeded(size: number) {
+        if (this.settings.syncMaxSizeInMB > 0 && size > 0) {
+            if (this.settings.syncMaxSizeInMB * 1024 * 1024 < size) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    get fileListPossiblyChanged() {
+        if (isDirty("totalFileEventCount", this.core.totalFileEventCount)) {
+            return true;
+        }
+        return false;
+    }
+
+    async $$isTargetFile(file: string | UXFileInfoStub, keepFileCheckList = false) {
+
+        const fileCount = useMemo<Record<string, number>>({
+            key: "fileCount", // forceUpdate: !keepFileCheckList,
+        }, (ctx, prev) => {
+            if (keepFileCheckList && prev) return prev;
+            if (!keepFileCheckList && prev && !this.fileListPossiblyChanged) {
+                return prev;
+            }
+            const fileList = (ctx.get("fileList") ?? []) as FilePathWithPrefix[];
+            // const fileNameList = (ctx.get("fileNameList") ?? []) as FilePath[];
+            // const fileNames = 
+            const vaultFiles = this.core.storageAccess.getFileNames().sort();
+            if (prev && vaultFiles.length == fileList.length) {
+                const fl3 = new Set([...fileList, ...vaultFiles]);
+                if (fileList.length == fl3.size && vaultFiles.length == fl3.size) {
+                    return prev;
+                }
+            }
+            ctx.set("fileList", vaultFiles);
+
+
+            const fileCount: Record<string, number> = {};
+            for (const file of vaultFiles) {
+                const lc = file.toLowerCase();
+                if (!fileCount[lc]) {
+                    fileCount[lc] = 1;
+                } else {
+                    fileCount[lc]++;
+                }
+            }
+            return fileCount;
+        })
+
+        const filepath = getPathFromUXFileInfo(file);
+        const lc = filepath.toLowerCase();
+        if (this.core.shouldCheckCaseInsensitive) {
+            if (lc in fileCount && fileCount[lc] > 1) {
+                return false;
+            }
+        }
+        const fileNameLC = getPathFromUXFileInfo(file).split("/").pop()?.toLowerCase();
+        if (this.settings.useIgnoreFiles) {
+            if (this.ignoreFiles.some(e => e.toLowerCase() == fileNameLC)) {
+                // We must reload ignore files due to the its change.
+                await this.readIgnoreFile(filepath);
+            }
+            if (await this.core.$$isIgnoredByIgnoreFiles(file)) {
+                return false;
+            }
+        }
+        if (!this.localDatabase?.isTargetFile(filepath)) return false;
+        return true
+    }
+
+    ignoreFileCache = new LRUCache<string, string[] | false>(300, 250000, true);
+    ignoreFiles = [] as string[]
+    async readIgnoreFile(path: string) {
+        try {
+            const file = await this.core.storageAccess.readFileText(path);
+            const gitignore = file.split(/\r?\n/g);
+            this.ignoreFileCache.set(path, gitignore);
+            return gitignore;
+        } catch (ex) {
+            this._log(`Failed to read ignore file ${path}`);
+            this._log(ex, LOG_LEVEL_VERBOSE);
+            this.ignoreFileCache.set(path, false);
+            return false;
+        }
+    }
+    async getIgnoreFile(path: string) {
+        if (this.ignoreFileCache.has(path)) {
+            return this.ignoreFileCache.get(path) ?? false;
+        } else {
+            return await this.readIgnoreFile(path);
+        }
+    }
+    async $$isIgnoredByIgnoreFiles(file: string | UXFileInfoStub): Promise<boolean> {
+        if (!this.settings.useIgnoreFiles) {
+            return false;
+        }
+        const filepath = getPathFromUXFileInfo(file)
+        if (this.ignoreFileCache.has(filepath)) {
+            // Renew
+            await this.readIgnoreFile(filepath);
+        }
+        if (!await isAcceptedAll(stripAllPrefixes(filepath), this.ignoreFiles, (filename) => this.getIgnoreFile(filename))) {
+            return true;
+        }
+        return false;
+    }
+}