bump

- Now remote database cleaning-up will be detected automatically.
- A solution selection dialogue will be shown if synchronisation is rejected after cleaning or rebuilding the remote database.
- During fetching or rebuilding, we can configure `Hidden file synchronisation` on the spot.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.18.3",
+    "version": "0.18.6",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.18.3",
+    "version": "0.18.6",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.18.3",
+            "version": "0.18.6",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.18.3",
+    "version": "0.18.6",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/CmdHiddenFileSync.ts

@@ -613,7 +613,7 @@ export class HiddenFileSync extends LiveSyncCommands {
 
 
     showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
-        return new Promise((res) => {
+        return runWithLock("conflict:merge-data", false, () => new Promise((res) => {
             Logger("Opening data-merging dialog", LOG_LEVEL.VERBOSE);
             const docs = [docA, docB];
             const path = stripAllPrefixes(docA.path);
@@ -624,6 +624,8 @@ export class HiddenFileSync extends LiveSyncCommands {
                     let needFlush = false;
                     if (!result && !keep) {
                         Logger(`Skipped merging: ${filename}`);
+                        res(false);
+                        return;
                     }
                     //Delete old revisions
                     if (result || keep) {
@@ -665,7 +667,7 @@ export class HiddenFileSync extends LiveSyncCommands {
                 }
             });
             modal.open();
-        });
+        }));
     }
 
     async scanInternalFiles(): Promise<InternalFileInfo[]> {

src/CmdSetupLiveSync.ts

@@ -5,6 +5,8 @@ import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { askSelectString, askYesNo, askString } from "./utils";
 import { decrypt, encrypt } from "./lib/src/e2ee_v2";
 import { LiveSyncCommands } from "./LiveSyncCommands";
+import { delay } from "./lib/src/utils";
+import { confirmWithMessage } from "./dialogs";
 
 export class SetupLiveSync extends LiveSyncCommands {
     onunload() { }
@@ -97,7 +99,8 @@ export class SetupLiveSync extends LiveSyncCommands {
                     const setupAsNew = "Set it up as secondary or subsequent device";
                     const setupAgain = "Reconfigure and reconstitute the data";
                     const setupManually = "Leave everything to me";
-
+                    newSettingW.syncInternalFiles = false;
+                    newSettingW.usePluginSync = false;
                     const setupType = await askSelectString(this.app, "How would you like to set it up?", [setupAsNew, setupAgain, setupJustImport, setupManually]);
                     if (setupType == setupJustImport) {
                         this.plugin.settings = newSettingW;
@@ -106,11 +109,7 @@ export class SetupLiveSync extends LiveSyncCommands {
                     } else if (setupType == setupAsNew) {
                         this.plugin.settings = newSettingW;
                         this.plugin.usedPassphrase = "";
-                        await this.plugin.saveSettings();
-                        await this.plugin.resetLocalDatabase();
-                        await this.plugin.localDatabase.initializeDatabase();
-                        await this.plugin.markRemoteResolved();
-                        await this.plugin.replicate(true);
+                        await this.fetchLocal();
                     } else if (setupType == setupAgain) {
                         const confirm = "I know this operation will rebuild all my databases with files on this device, and files that are on the remote database and I didn't synchronize to any other devices will be lost and want to proceed indeed.";
                         if (await askSelectString(this.app, "Do you really want to do this?", ["Cancel", confirm]) != confirm) {
@@ -118,15 +117,7 @@ export class SetupLiveSync extends LiveSyncCommands {
                         }
                         this.plugin.settings = newSettingW;
                         this.plugin.usedPassphrase = "";
-                        await this.plugin.saveSettings();
-                        await this.plugin.resetLocalDatabase();
-                        await this.plugin.localDatabase.initializeDatabase();
-                        await this.plugin.initializeDatabase(true);
-                        await this.plugin.tryResetRemoteDatabase();
-                        await this.plugin.markRemoteLocked();
-                        await this.plugin.markRemoteResolved();
-                        await this.plugin.replicate(true);
-
+                        await this.rebuildEverything();
                     } else if (setupType == setupManually) {
                         const keepLocalDB = await askYesNo(this.app, "Keep local DB?");
                         const keepRemoteDB = await askYesNo(this.app, "Keep remote DB?");
@@ -134,6 +125,8 @@ export class SetupLiveSync extends LiveSyncCommands {
                             // nothing to do. so peaceful.
                             this.plugin.settings = newSettingW;
                             this.plugin.usedPassphrase = "";
+                            this.suspendAllSync();
+                            this.suspendExtraSync();
                             await this.plugin.saveSettings();
                             const replicate = await askYesNo(this.app, "Unlock and replicate?");
                             if (replicate == "yes") {
@@ -189,4 +182,112 @@ export class SetupLiveSync extends LiveSyncCommands {
             Logger("Couldn't parse or decrypt configuration uri.", LOG_LEVEL.NOTICE);
         }
     }
+
+    suspendExtraSync() {
+        Logger("Hidden files and plugin synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL.NOTICE)
+        this.plugin.settings.syncInternalFiles = false;
+        this.plugin.settings.usePluginSync = false;
+        this.plugin.settings.autoSweepPlugins = false;
+    }
+    async askHiddenFileConfiguration(opt: { enableFetch?: boolean, enableOverwrite?: boolean }) {
+        this.plugin.addOnSetup.suspendExtraSync();
+        const message = `Would you like to enable \`Hidden File Synchronization\`?
+${opt.enableFetch ? " - Fetch: Use files stored from other devices. \n" : ""}${opt.enableOverwrite ? "- Overwrite: Use files from this device. \n" : ""}- Keep it disabled: Do not use hidden file synchronization.
+
+Of course, we are able to disable this feature.`
+        const CHOICE_FETCH = "Fetch";
+        const CHOICE_OVERWRITE = "Overwrite";
+        const CHOICE_DISMISS = "keep it disabled";
+        const choices = [];
+        if (opt?.enableFetch) {
+            choices.push(CHOICE_FETCH);
+        }
+        if (opt?.enableOverwrite) {
+            choices.push(CHOICE_OVERWRITE);
+        }
+        choices.push(CHOICE_DISMISS);
+
+        const ret = await confirmWithMessage(this.plugin, "Hidden file sync", message, choices, CHOICE_DISMISS, 40);
+        if (ret == CHOICE_FETCH) {
+            await this.configureHiddenFileSync("FETCH");
+        } else if (ret == CHOICE_OVERWRITE) {
+            await this.configureHiddenFileSync("OVERWRITE");
+        } else if (ret == CHOICE_DISMISS) {
+            await this.configureHiddenFileSync("DISABLE");
+        }
+    }
+    async configureHiddenFileSync(mode: "FETCH" | "OVERWRITE" | "MERGE" | "DISABLE") {
+        this.plugin.addOnSetup.suspendExtraSync();
+        if (mode == "DISABLE") {
+            this.plugin.settings.syncInternalFiles = false;
+            await this.plugin.saveSettings();
+            return;
+        }
+        Logger("Gathering files for enabling Hidden File Sync", LOG_LEVEL.NOTICE);
+        if (mode == "FETCH") {
+            await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pullForce", true);
+        } else if (mode == "OVERWRITE") {
+            await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pushForce", true);
+        } else if (mode == "MERGE") {
+            await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("safe", true);
+        }
+        this.plugin.settings.syncInternalFiles = true;
+        await this.plugin.saveSettings();
+        Logger(`Done! Restarting the app is strongly recommended!`, LOG_LEVEL.NOTICE);
+
+    }
+
+    suspendAllSync() {
+        this.plugin.settings.liveSync = false;
+        this.plugin.settings.periodicReplication = false;
+        this.plugin.settings.syncOnSave = false;
+        this.plugin.settings.syncOnStart = false;
+        this.plugin.settings.syncOnFileOpen = false;
+        this.plugin.settings.syncAfterMerge = false;
+        //this.suspendExtraSync();
+    }
+    async fetchLocal() {
+        this.suspendExtraSync();
+        await this.plugin.realizeSettingSyncMode();
+        await this.plugin.resetLocalDatabase();
+        await delay(1000);
+        await this.plugin.markRemoteResolved();
+        await this.plugin.openDatabase();
+        this.plugin.isReady = true;
+        await delay(500);
+        await this.plugin.replicateAllFromServer(true);
+        await delay(1000);
+        await this.plugin.replicateAllFromServer(true);
+        await this.askHiddenFileConfiguration({ enableFetch: true });
+    }
+    async rebuildRemote() {
+        this.suspendExtraSync();
+        await this.plugin.realizeSettingSyncMode();
+        await this.plugin.markRemoteLocked();
+        await this.plugin.tryResetRemoteDatabase();
+        await this.plugin.markRemoteLocked();
+        await delay(500);
+        await this.askHiddenFileConfiguration({ enableOverwrite: true });
+        await delay(1000);
+        await this.plugin.replicateAllToServer(true);
+        await delay(1000);
+        await this.plugin.replicateAllToServer(true);
+    }
+    async rebuildEverything() {
+        this.suspendExtraSync();
+        await this.plugin.realizeSettingSyncMode();
+        await this.plugin.resetLocalDatabase();
+        await delay(1000);
+        await this.plugin.initializeDatabase(true);
+        await this.plugin.markRemoteLocked();
+        await this.plugin.tryResetRemoteDatabase();
+        await this.plugin.markRemoteLocked();
+        await delay(500);
+        await this.askHiddenFileConfiguration({ enableOverwrite: true });
+        await delay(1000);
+        await this.plugin.replicateAllToServer(true);
+        await delay(1000);
+        await this.plugin.replicateAllToServer(true);
+
+    }
 }

src/ObsidianLiveSyncSettingTab.ts

@@ -1,4 +1,4 @@
-import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps";
+import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps";
 import { DEFAULT_SETTINGS, LOG_LEVEL, ObsidianLiveSyncSettings, ConfigPassphraseStore, RemoteDBSettings } from "./lib/src/types";
 import { delay } from "./lib/src/utils";
 import { Semaphore } from "./lib/src/semaphore";
@@ -7,24 +7,9 @@ import { Logger } from "./lib/src/logger";
 import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb.js";
 import { testCrypt } from "./lib/src/e2ee_v2";
 import ObsidianLiveSyncPlugin from "./main";
+import { balanceChunks, localDatabaseCleanUp, performRebuildDB, remoteDatabaseCleanup, requestToCouchDB } from "./utils";
 
-const requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, key?: string, body?: string) => {
-    const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${username}:${password}`));
-    const encoded = window.btoa(utf8str);
-    const authHeader = "Basic " + encoded;
-    // const origin = "capacitor://localhost";
-    const transformedHeaders: Record<string, string> = { authorization: authHeader, origin: origin };
-    const uri = `${baseUri}/_node/_local/_config${key ? "/" + key : ""}`;
 
-    const requestParam: RequestUrlParam = {
-        url: uri,
-        method: body ? "PUT" : "GET",
-        headers: transformedHeaders,
-        contentType: "application/json",
-        body: body ? JSON.stringify(body) : undefined,
-    };
-    return await requestUrl(requestParam);
-};
 export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
     plugin: ObsidianLiveSyncPlugin;
     selectedScreen = "";
@@ -74,7 +59,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
 <label class='sls-setting-label c-40'><input type='radio' name='disp' value='40' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🔧</div></label>
 <label class='sls-setting-label c-50 wizardHidden'><input type='radio' name='disp' value='50' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🧰</div></label>
 <label class='sls-setting-label c-60 wizardHidden'><input type='radio' name='disp' value='60' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🔌</div></label>
-<!-- <label class='sls-setting-label c-70 wizardHidden'><input type='radio' name='disp' value='70' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🚑</div></label>-->
+<label class='sls-setting-label c-70 wizardHidden'><input type='radio' name='disp' value='70' class='sls-setting-tab' ><div class='sls-setting-menu-btn'>🎛️</div></label>
         `;
         const menuTabs = w.querySelectorAll(".sls-setting-label");
         const changeDisplay = (screen: string) => {
@@ -420,25 +405,16 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             if (!encrypt) {
                 passphrase = "";
             }
-            this.plugin.settings.liveSync = false;
-            this.plugin.settings.periodicReplication = false;
-            this.plugin.settings.syncOnSave = false;
-            this.plugin.settings.syncOnStart = false;
-            this.plugin.settings.syncOnFileOpen = false;
-            this.plugin.settings.syncAfterMerge = false;
+            this.plugin.addOnSetup.suspendAllSync();
+            this.plugin.addOnSetup.suspendExtraSync();
             this.plugin.settings.encrypt = encrypt;
             this.plugin.settings.passphrase = passphrase;
             this.plugin.settings.useDynamicIterationCount = useDynamicIterationCount;
             this.plugin.settings.usePathObfuscation = usePathObfuscation;
-
             await this.plugin.saveSettings();
             markDirtyControl();
             if (sendToServer) {
-                await this.plugin.initializeDatabase(true);
-                await this.plugin.markRemoteLocked();
-                await this.plugin.tryResetRemoteDatabase();
-                await this.plugin.markRemoteLocked();
-                await this.plugin.replicateAllToServer(true);
+                await this.plugin.addOnSetup.rebuildRemote()
             } else {
                 await this.plugin.markRemoteResolved();
                 await this.plugin.replicate(true);
@@ -489,78 +465,22 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             if (!encrypt) {
                 passphrase = "";
             }
-            this.plugin.settings.liveSync = false;
-            this.plugin.settings.periodicReplication = false;
-            this.plugin.settings.syncOnSave = false;
-            this.plugin.settings.syncOnStart = false;
-            this.plugin.settings.syncOnFileOpen = false;
-            this.plugin.settings.syncAfterMerge = false;
-            this.plugin.settings.syncInternalFiles = false;
-            this.plugin.settings.usePluginSync = false;
+            this.plugin.addOnSetup.suspendAllSync();
+            this.plugin.addOnSetup.suspendExtraSync();
             this.plugin.settings.encrypt = encrypt;
             this.plugin.settings.passphrase = passphrase;
             this.plugin.settings.useDynamicIterationCount = useDynamicIterationCount;
             this.plugin.settings.usePathObfuscation = usePathObfuscation;
-            Logger("Hidden files and plugin synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL.NOTICE)
+            Logger("All synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL.NOTICE)
             await this.plugin.saveSettings();
             markDirtyControl();
             applyDisplayEnabled();
             // @ts-ignore
-            this.plugin.app.setting.close()
+            this.plugin.app.setting.close();
             await delay(2000);
-            if (method == "localOnly") {
-                await this.plugin.resetLocalDatabase();
-                await delay(1000);
-                await this.plugin.markRemoteResolved();
-                await this.plugin.openDatabase();
-                this.plugin.isReady = true;
-                await this.plugin.replicateAllFromServer(true);
-            }
-            if (method == "remoteOnly") {
-                await this.plugin.markRemoteLocked();
-                await this.plugin.tryResetRemoteDatabase();
-                await this.plugin.markRemoteLocked();
-                await this.plugin.replicateAllToServer(true);
-            }
-            if (method == "rebuildBothByThisDevice") {
-                await this.plugin.resetLocalDatabase();
-                await delay(1000);
-                await this.plugin.initializeDatabase(true);
-                await this.plugin.markRemoteLocked();
-                await this.plugin.tryResetRemoteDatabase();
-                await this.plugin.markRemoteLocked();
-                await this.plugin.replicateAllToServer(true);
-            }
+            await performRebuildDB(this.plugin, method);
         }
 
-        new Setting(containerRemoteDatabaseEl)
-            .setName("Overwrite remote database")
-            .setDesc("Overwrite remote database with local DB and passphrase.")
-            .setClass("wizardHidden")
-            .addButton((button) =>
-                button
-                    .setButtonText("Send")
-                    .setWarning()
-                    .setDisabled(false)
-                    .onClick(async () => {
-                        await rebuildDB("remoteOnly");
-                    })
-            )
-
-        new Setting(containerRemoteDatabaseEl)
-            .setName("Rebuild everything")
-            .setDesc("Rebuild local and remote database with local files.")
-            .setClass("wizardHidden")
-            .addButton((button) =>
-                button
-                    .setButtonText("Rebuild")
-                    .setWarning()
-                    .setDisabled(false)
-                    .onClick(async () => {
-                        await rebuildDB("rebuildBothByThisDevice");
-                    })
-            )
-
         new Setting(containerRemoteDatabaseEl)
             .setName("Test Database Connection")
             .setDesc("Open database connection. If the remote database is not found and you have the privilege to create a database, the database will be created.")
@@ -744,19 +664,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             text: "",
         });
 
-        new Setting(containerRemoteDatabaseEl)
-            .setName("Lock remote database")
-            .setDesc("Lock remote database to prevent synchronization with other devices.")
-            .setClass("wizardHidden")
-            .addButton((button) =>
-                button
-                    .setButtonText("Lock")
-                    .setDisabled(false)
-                    .setWarning()
-                    .onClick(async () => {
-                        await this.plugin.markRemoteLocked();
-                    })
-            );
         let rebuildRemote = false;
 
         new Setting(containerRemoteDatabaseEl)
@@ -821,21 +728,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 })
             );
 
-        new Setting(containerLocalDatabaseEl)
-            .setName("Fetch rebuilt DB")
-            .setDesc("Restore or reconstruct local database from remote database.")
-            .setClass("wizardHidden")
-            .addButton((button) =>
-                button
-                    .setButtonText("Fetch")
-                    .setWarning()
-                    .setDisabled(false)
-                    .onClick(async () => {
-                        await rebuildDB("localOnly");
-                    })
-            )
-
-
         let newDatabaseName = this.plugin.settings.additionalSuffixOfDatabaseName + "";
         new Setting(containerLocalDatabaseEl)
             .setName("Database suffix")
@@ -1149,32 +1041,25 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 .addButton((button) => {
                     button.setButtonText("Merge")
                         .onClick(async () => {
-                            this.plugin.settings.syncInternalFiles = true;
-                            this.display();
-                            await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("safe", true);
-                            await this.plugin.saveSettings();
-                            this.display();
+                            // @ts-ignore
+                            this.plugin.app.setting.close()
+                            await this.plugin.addOnSetup.configureHiddenFileSync("MERGE");
                         })
                 })
                 .addButton((button) => {
                     button.setButtonText("Fetch")
                         .onClick(async () => {
-                            this.plugin.settings.syncInternalFiles = true;
-                            this.display();
-                            await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pullForce", true);
-                            await this.plugin.saveSettings();
-                            Logger(`Restarting the app is strongly recommended!`, LOG_LEVEL.NOTICE);
-                            this.display();
+                            // @ts-ignore
+                            this.plugin.app.setting.close()
+                            await this.plugin.addOnSetup.configureHiddenFileSync("FETCH");
                         })
                 })
                 .addButton((button) => {
                     button.setButtonText("Overwrite")
                         .onClick(async () => {
-                            this.plugin.settings.syncInternalFiles = true;
-                            this.display();
-                            await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pushForce", true);
-                            await this.plugin.saveSettings();
-                            this.display();
+                            // @ts-ignore
+                            this.plugin.app.setting.close()
+                            await this.plugin.addOnSetup.configureHiddenFileSync("OVERWRITE");
                         })
                 });
         }
@@ -1714,18 +1599,8 @@ ${stringifyYaml(pluginConfig)}`;
                 })
             );
 
-        new Setting(containerHatchEl)
-            .setName("Discard local database to reset or uninstall Self-hosted LiveSync")
-            .addButton((button) =>
-                button
-                    .setButtonText("Discard")
-                    .setWarning()
-                    .setDisabled(false)
-                    .onClick(async () => {
-                        await this.plugin.resetLocalDatabase();
-                        await this.plugin.initializeDatabase();
-                    })
-            );
+
+
 
         addScreenElement("50", containerHatchEl);
         // With great respect, thank you TfTHacker!
@@ -1803,73 +1678,136 @@ ${stringifyYaml(pluginConfig)}`;
 
         addScreenElement("60", containerPluginSettings);
 
-        // const containerCorruptedDataEl = containerEl.createDiv();
+        const containerMaintenanceEl = containerEl.createDiv();
 
-        // containerCorruptedDataEl.createEl("h3", { text: "Corrupted or missing data" });
-        // containerCorruptedDataEl.createEl("h4", { text: "Corrupted" });
-        // if (Object.keys(this.plugin.localDatabase.corruptedEntries).length > 0) {
-        //     const cx = containerCorruptedDataEl.createEl("div", { text: "If you have a copy of these files on any device, simply edit them once and sync. If not, there's nothing we can do except deleting them. sorry.." });
-        //     for (const k in this.plugin.localDatabase.corruptedEntries) {
-        //         const xx = cx.createEl("div", { text: `${k}` });
+        containerMaintenanceEl.createEl("h3", { text: "Maintain databases" });
 
-        //         const ba = xx.createEl("button", { text: `Delete this` }, (e) => {
-        //             e.addEventListener("click", async () => {
-        //                 await this.plugin.localDatabase.deleteDBEntry(k as string as FilePathWithPrefix /* should be explained */);
-        //                 xx.remove();
-        //             });
-        //         });
-        //         ba.addClass("mod-warning");
-        //         //TODO: FIX LATER
-        //         // xx.createEl("button", { text: `Restore from file` }, (e) => {
-        //         //     e.addEventListener("click", async () => {
-        //         //         const f = await this.app.vault.getFiles().filter((e) => this.plugin.path2id(e.path) == k);
-        //         //         if (f.length == 0) {
-        //         //             Logger("Not found in vault", LOG_LEVEL.NOTICE);
-        //         //             return;
-        //         //         }
-        //         //         await this.plugin.updateIntoDB(f[0]);
-        //         //         xx.remove();
-        //         //     });
-        //         // });
-        //         // xx.addClass("mod-warning");
-        //     }
-        // } else {
-        //     containerCorruptedDataEl.createEl("div", { text: "There is no corrupted data." });
-        // }
-        // containerCorruptedDataEl.createEl("h4", { text: "Missing or waiting" });
-        // if (Object.keys(this.plugin.queuedFiles).length > 0) {
-        //     const cx = containerCorruptedDataEl.createEl("div", {
-        //         text: "These files have missing or waiting chunks. Perhaps these chunks will arrive in a while after replication. But if they don't, you have to restore it's database entry from a existing local file by hitting the button below.",
-        //     });
-        //     const files = [...new Set([...this.plugin.queuedFiles.map((e) => e.entry._id)])];
-        //     for (const k of files) {
-        //         const xx = cx.createEl("div", { text: `${this.plugin.id2path(k)}` });
+        containerMaintenanceEl.createEl("h4", { text: "The remote database" });
 
-        //         const ba = xx.createEl("button", { text: `Delete this` }, (e) => {
-        //             e.addEventListener("click", async () => {
-        //                 await this.plugin.localDatabase.deleteDBEntry(k);
-        //                 xx.remove();
-        //             });
-        //         });
-        //         ba.addClass("mod-warning");
-        //         xx.createEl("button", { text: `Restore from file` }, (e) => {
-        //             e.addEventListener("click", async () => {
-        //                 const f = await this.app.vault.getFiles().filter((e) => this.plugin.path2id(e.path) == k);
-        //                 if (f.length == 0) {
-        //                     Logger("Not found in vault", LOG_LEVEL.NOTICE);
-        //                     return;
-        //                 }
-        //                 await this.plugin.updateIntoDB(f[0]);
-        //                 xx.remove();
-        //             });
-        //         });
-        //         xx.addClass("mod-warning");
-        //     }
-        // } else {
-        //     containerCorruptedDataEl.createEl("div", { text: "There is no missing or waiting chunk." });
-        // }
-        // applyDisplayEnabled();
-        // addScreenElement("70", containerCorruptedDataEl);
+        new Setting(containerMaintenanceEl)
+            .setName("Lock remote database")
+            .setDesc("Lock remote database to prevent synchronization with other devices.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Lock")
+                    .setDisabled(false)
+                    .setWarning()
+                    .onClick(async () => {
+                        await this.plugin.markRemoteLocked();
+                    })
+            );
+
+        new Setting(containerMaintenanceEl)
+            .setName("Overwrite remote database")
+            .setDesc("Overwrite remote database with local DB and passphrase.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Send")
+                    .setWarning()
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await rebuildDB("remoteOnly");
+                    })
+            )
+
+        new Setting(containerMaintenanceEl)
+            .setName("(Beta) Clean the remote database")
+            .setDesc("")
+            .addButton((button) =>
+                button.setButtonText("Count")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await remoteDatabaseCleanup(this.plugin, true);
+                    })
+            ).addButton((button) =>
+                button.setButtonText("Perform cleaning")
+                    .setDisabled(false)
+                    .setWarning()
+                    .onClick(async () => {
+                        // @ts-ignore
+                        this.plugin.app.setting.close()
+                        await remoteDatabaseCleanup(this.plugin, false);
+                        await balanceChunks(this.plugin, false);
+                    })
+            );
+
+        containerMaintenanceEl.createEl("h4", { text: "The local database" });
+
+        new Setting(containerMaintenanceEl)
+            .setName("Fetch rebuilt DB")
+            .setDesc("Restore or reconstruct local database from remote database.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Fetch")
+                    .setWarning()
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await rebuildDB("localOnly");
+                    })
+            )
+
+        new Setting(containerMaintenanceEl)
+            .setName("(Beta) Clean the local database")
+            .setDesc("This feature requires enabling 'Use new Adapter'")
+            .addButton((button) =>
+                button.setButtonText("Count")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await localDatabaseCleanUp(this.plugin, false, true);
+                    })
+            ).addButton((button) =>
+                button.setButtonText("Perform cleaning")
+                    .setDisabled(false)
+                    .setWarning()
+                    .onClick(async () => {
+                        // @ts-ignore
+                        this.plugin.app.setting.close()
+                        await localDatabaseCleanUp(this.plugin, false, false);
+                    })
+            );
+
+        new Setting(containerMaintenanceEl)
+            .setName("Discard local database to reset or uninstall Self-hosted LiveSync")
+            .addButton((button) =>
+                button
+                    .setButtonText("Discard")
+                    .setWarning()
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.resetLocalDatabase();
+                        await this.plugin.initializeDatabase();
+                    })
+            );
+
+        containerMaintenanceEl.createEl("h4", { text: "Both databases" });
+
+        new Setting(containerMaintenanceEl)
+            .setName("Rebuild everything")
+            .setDesc("Rebuild local and remote database with local files.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Rebuild")
+                    .setWarning()
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await rebuildDB("rebuildBothByThisDevice");
+                    })
+            )
+
+        new Setting(containerMaintenanceEl)
+            .setName("(Beta) Complement each other with possible missing chunks.")
+            .setDesc("")
+            .addButton((button) =>
+                button
+                    .setButtonText("Balance")
+                    .setWarning()
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await balanceChunks(this.plugin, false);
+                    })
+            )
+        applyDisplayEnabled();
+        addScreenElement("70", containerMaintenanceEl);
 
         applyDisplayEnabled();
         if (this.selectedScreen == "") {

src/StorageEventManager.ts

@@ -57,8 +57,8 @@ export class StorageEventManagerObsidian extends StorageEventManager {
     watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) {
         if (file instanceof TFile) {
             this.appendWatchEvent([
-                { type: "CREATE", file },
                 { type: "DELETE", file: { path: oldFile as FilePath, mtime: file.stat.mtime, ctime: file.stat.ctime, size: file.stat.size, deleted: true } },
+                { type: "CREATE", file },
             ], ctx);
         }
     }

src/dialogs.ts

@@ -1,4 +1,5 @@
-import { App, FuzzySuggestModal, Modal, Setting } from "./deps";
+import { ButtonComponent } from "obsidian";
+import { App, FuzzySuggestModal, MarkdownRenderer, Modal, Plugin, Setting } from "./deps";
 import ObsidianLiveSyncPlugin from "./main";
 
 //@ts-ignore
@@ -123,4 +124,103 @@ export class PopoverSelectString extends FuzzySuggestModal<string> {
             }
         }, 100);
     }
-}
+}
+
+export class MessageBox extends Modal {
+
+    plugin: Plugin;
+    title: string;
+    contentMd: string;
+    buttons: string[];
+    result: string;
+    isManuallyClosed = false;
+    defaultAction: string | undefined;
+    timeout: number | undefined;
+    timer: ReturnType<typeof setInterval> = undefined;
+    defaultButtonComponent: ButtonComponent | undefined;
+
+    onSubmit: (result: string | boolean) => void;
+
+    constructor(plugin: Plugin, title: string, contentMd: string, buttons: string[], defaultAction: (typeof buttons)[number], timeout: number, onSubmit: (result: (typeof buttons)[number] | false) => void) {
+        super(plugin.app);
+        this.plugin = plugin;
+        this.title = title;
+        this.contentMd = contentMd;
+        this.buttons = buttons;
+        this.onSubmit = onSubmit;
+        this.defaultAction = defaultAction;
+        this.timeout = timeout;
+        if (this.timeout) {
+            this.timer = setInterval(() => {
+                this.timeout--;
+                if (this.timeout < 0) {
+                    if (this.timer) {
+                        clearInterval(this.timer);
+                        this.timer = undefined;
+                    }
+                    this.result = defaultAction;
+                    this.isManuallyClosed = true;
+                    this.close();
+                } else {
+                    this.defaultButtonComponent.setButtonText(`( ${this.timeout} ) ${defaultAction}`);
+                }
+            }, 1000);
+        }
+    }
+
+    onOpen() {
+        const { contentEl } = this;
+        contentEl.addEventListener("click", () => {
+            if (this.timer) {
+                clearInterval(this.timer);
+                this.timer = undefined;
+            }
+        })
+        contentEl.createEl("h1", { text: this.title });
+        const div = contentEl.createDiv();
+        MarkdownRenderer.renderMarkdown(this.contentMd, div, "/", null);
+        const buttonSetting = new Setting(contentEl);
+        for (const button of this.buttons) {
+            buttonSetting.addButton((btn) => {
+                btn
+                    .setButtonText(button)
+                    .onClick(() => {
+                        this.isManuallyClosed = true;
+                        this.result = button;
+                        if (this.timer) {
+                            clearInterval(this.timer);
+                            this.timer = undefined;
+                        }
+                        this.close();
+                    })
+                if (button == this.defaultAction) {
+                    this.defaultButtonComponent = btn;
+                }
+                return btn;
+            }
+            )
+        }
+    }
+
+    onClose() {
+        const { contentEl } = this;
+        contentEl.empty();
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+        if (this.isManuallyClosed) {
+            this.onSubmit(this.result);
+        } else {
+            this.onSubmit(false);
+        }
+    }
+}
+
+
+export function confirmWithMessage(plugin: Plugin, title: string, contentMd: string, buttons: string[], defaultAction?: (typeof buttons)[number], timeout?: number): Promise<(typeof buttons)[number] | false> {
+    return new Promise((res) => {
+        const dialog = new MessageBox(plugin, title, contentMd, buttons, defaultAction, timeout, (result) => res(result));
+        dialog.open();
+    });
+};

src/main.ts

@@ -4,14 +4,14 @@ import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "di
 import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, RequestUrlParam, RequestUrlResponse, requestUrl } from "./deps";
 import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, ConfigPassphraseStore, CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, DatabaseConnectingStatus, EntryHasPath, DocumentID, FilePathWithPrefix, FilePath, AnyEntry } from "./lib/src/types";
 import { InternalFileInfo, queueItem, CacheData, FileEventItem, FileWatchEventQueueMax } from "./types";
-import { delay, getDocData, isDocContentSame } from "./lib/src/utils";
+import { getDocData, isDocContentSame } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { LogDisplayModal } from "./LogDisplayModal";
 import { ConflictResolveModal } from "./ConflictResolveModal";
 import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
 import { DocumentHistoryModal } from "./DocumentHistoryModal";
-import { applyPatch, cancelAllPeriodicTask, cancelAllTasks, cancelTask, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, flattenObject, path2id, scheduleTask, tryParseJSON, createFile, modifyFile, isValidPath, getAbstractFileByPath, touch, recentlyTouched, isIdOfInternalMetadata, isPluginMetadata, stripInternalMetadataPrefix, isChunk, askSelectString, askYesNo, askString, PeriodicProcessor, clearTouched, getPath, getPathWithoutPrefix, getPathFromTFile } from "./utils";
+import { applyPatch, cancelAllPeriodicTask, cancelAllTasks, cancelTask, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, flattenObject, path2id, scheduleTask, tryParseJSON, createFile, modifyFile, isValidPath, getAbstractFileByPath, touch, recentlyTouched, isIdOfInternalMetadata, isPluginMetadata, stripInternalMetadataPrefix, isChunk, askSelectString, askYesNo, askString, PeriodicProcessor, clearTouched, getPath, getPathWithoutPrefix, getPathFromTFile, localDatabaseCleanUp, balanceChunks, performRebuildDB } from "./utils";
 import { encrypt, tryDecrypt } from "./lib/src/e2ee_v2";
 import { enableEncryption, isCloudantURI, isErrorOfMissingDoc, isValidRemoteCouchDBURI } from "./lib/src/utils_couchdb";
 import { getGlobalStore, ObservableStore, observeStores } from "./lib/src/store";
@@ -29,6 +29,7 @@ import { LiveSyncCommands } from "./LiveSyncCommands";
 import { PluginAndTheirSettings } from "./CmdPluginAndTheirSettings";
 import { HiddenFileSync } from "./CmdHiddenFileSync";
 import { SetupLiveSync } from "./CmdSetupLiveSync";
+import { confirmWithMessage } from "./dialogs";
 
 setNoticeClass(Notice);
 
@@ -387,25 +388,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         try {
             if (this.isRedFlagRaised() || this.isRedFlag2Raised() || this.isRedFlag3Raised()) {
                 this.settings.batchSave = false;
-                this.settings.liveSync = false;
-                this.settings.periodicReplication = false;
-                this.settings.syncOnSave = false;
-                this.settings.syncOnStart = false;
-                this.settings.syncOnFileOpen = false;
-                this.settings.syncAfterMerge = false;
-                this.settings.autoSweepPlugins = false;
-                this.settings.usePluginSync = false;
+                this.addOnSetup.suspendAllSync();
+                this.addOnSetup.suspendExtraSync();
                 this.settings.suspendFileWatching = true;
-                this.settings.syncInternalFiles = false;
                 await this.saveSettings();
                 if (this.isRedFlag2Raised()) {
                     Logger(`${FLAGMD_REDFLAG2} has been detected! Self-hosted LiveSync suspends all sync and rebuild everything.`, LOG_LEVEL.NOTICE);
-                    await this.resetLocalDatabase();
-                    await this.initializeDatabase(true);
-                    await this.markRemoteLocked();
-                    await this.tryResetRemoteDatabase();
-                    await this.markRemoteLocked();
-                    await this.replicateAllToServer(true);
+                    await this.addOnSetup.rebuildEverything();
                     await this.deleteRedFlag2();
                     if (await askYesNo(this.app, "Do you want to disable Suspend file watching and restart obsidian now?") == "yes") {
                         this.settings.suspendFileWatching = false;
@@ -415,12 +404,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                     }
                 } else if (this.isRedFlag3Raised()) {
                     Logger(`${FLAGMD_REDFLAG3} has been detected! Self-hosted LiveSync will discard the local database and fetch everything from the remote once again.`, LOG_LEVEL.NOTICE);
-                    await this.resetLocalDatabase();
-                    await delay(1000);
-                    await this.markRemoteResolved();
-                    await this.openDatabase();
-                    this.isReady = true;
-                    await this.replicateAllFromServer(true);
+                    await this.addOnSetup.fetchLocal();
                     await this.deleteRedFlag3();
                     if (await askYesNo(this.app, "Do you want to disable Suspend file watching and restart obsidian now?") == "yes") {
                         this.settings.suspendFileWatching = false;
@@ -912,8 +896,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                 const file = queue.args.file;
                 const key = `file-last-proc-${queue.type}-${file.path}`;
                 const last = Number(await this.kvDB.get(key) || 0);
+                let mtime = file.mtime;
                 if (queue.type == "DELETE") {
                     await this.deleteFromDBbyPath(file.path);
+                    mtime = file.mtime - 1;
+                    const keyD1 = `file-last-proc-CREATE-${file.path}`;
+                    const keyD2 = `file-last-proc-CHANGED-${file.path}`;
+                    await this.kvDB.set(keyD1, mtime);
+                    await this.kvDB.set(keyD2, mtime);
                 } else if (queue.type == "INTERNAL") {
                     await this.addOnHiddenFileSync.watchVaultRawEventsAsync(file.path);
                 } else {
@@ -930,6 +920,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin
 
                     const cache = queue.args.cache;
                     if (queue.type == "CREATE" || queue.type == "CHANGED") {
+                        const keyD1 = `file-last-proc-DELETED-${file.path}`;
+                        await this.kvDB.set(keyD1, mtime);
                         if (!await this.updateIntoDB(targetFile, false, cache)) {
                             Logger(`DB -> STORAGE: failed, cancel the relative operations: ${targetFile.path}`, LOG_LEVEL.INFO);
                             // cancel running queues and remove one of atomic operation
@@ -942,7 +934,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                         await this.watchVaultRenameAsync(targetFile, queue.args.oldPath);
                     }
                 }
-                await this.kvDB.set(key, file.mtime);
+                await this.kvDB.set(key, mtime);
             } while (this.vaultManager.getQueueLength() > 0);
             return true;
         })
@@ -1114,13 +1106,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin
             // This occurs not only when files are deleted, but also when conflicts are resolved.
             // We have to check no other revisions are left.
             const lastDocs = await this.localDatabase.getDBEntry(path);
+            if (path != file.path) {
+                Logger(`delete skipped: ${file.path} :Not exactly matched`, LOG_LEVEL.VERBOSE);
+            }
             if (lastDocs === false) {
                 await this.deleteVaultItem(file);
             } else {
                 // it perhaps delete some revisions.
                 // may be we have to reload this
                 await this.pullFile(path, null, true);
-                Logger(`delete skipped:${lastDocs._id}`, LOG_LEVEL.VERBOSE);
+                Logger(`delete skipped:${file.path}`, LOG_LEVEL.VERBOSE);
             }
             return;
         }
@@ -1381,8 +1376,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin
 
     //---> Sync
     async parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): Promise<void> {
+        const docsSorted = docs.sort((a, b) => b.mtime - a.mtime);
         L1:
-        for (const change of docs) {
+        for (const change of docsSorted) {
             for (const proc of this.addOns) {
                 if (await proc.parseReplicationResultItem(change)) {
                     continue L1;
@@ -1547,7 +1543,43 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         await this.applyBatchChange();
         await Promise.all(this.addOns.map(e => e.beforeReplicate(showMessage)));
         await this.loadQueuedFiles();
-        return await this.replicator.openReplication(this.settings, false, showMessage);
+        const ret = await this.replicator.openReplication(this.settings, false, showMessage);
+        if (!ret) {
+            if (this.replicator.remoteLockedAndDeviceNotAccepted) {
+                if (this.replicator.remoteCleaned) {
+                    const message = `
+The remote database has been cleaned up.
+To synchronize, this device must also be cleaned up or fetch everything again once.
+Fetching may takes some time. Cleaning up is not stable yet but fast.
+`
+                    const CHOICE_CLEANUP = "Clean up";
+                    const CHOICE_FETCH = "Fetch again";
+                    const CHOICE_DISMISS = "Dismiss";
+                    const ret = await confirmWithMessage(this, "Locked", message, [CHOICE_CLEANUP, CHOICE_FETCH, CHOICE_DISMISS], CHOICE_DISMISS, 10);
+                    if (ret == CHOICE_CLEANUP) {
+                        await localDatabaseCleanUp(this, true, false);
+                        await balanceChunks(this, false);
+                    }
+                    if (ret == CHOICE_FETCH) {
+                        await performRebuildDB(this, "localOnly");
+                    }
+                } else {
+                    const message = `
+The remote database has been rebuilt.
+To synchronize, this device must fetch everything again once.
+Or if you are sure know what had been happened, we can unlock the database from the setting dialog.
+                    `
+                    const CHOICE_FETCH = "Fetch again";
+                    const CHOICE_DISMISS = "Dismiss";
+                    const ret = await confirmWithMessage(this, "Locked", message, [CHOICE_FETCH, CHOICE_DISMISS], CHOICE_DISMISS, 10);
+                    if (ret == CHOICE_FETCH) {
+                        await performRebuildDB(this, "localOnly");
+                    }
+                }
+            }
+        }
+
+        return ret;
     }
 
     async initializeDatabase(showingNotice?: boolean, reopenDatabase = true) {
@@ -1578,12 +1610,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         return await this.replicator.replicateAllFromServer(this.settings, showingNotice);
     }
 
-    async markRemoteLocked() {
-        return await this.replicator.markRemoteLocked(this.settings, true);
+    async markRemoteLocked(lockByClean?: boolean) {
+        return await this.replicator.markRemoteLocked(this.settings, true, lockByClean);
     }
 
     async markRemoteUnlocked() {
-        return await this.replicator.markRemoteLocked(this.settings, false);
+        return await this.replicator.markRemoteLocked(this.settings, false, false);
     }
 
     async markRemoteResolved() {
@@ -2043,60 +2075,62 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     }
 
     showMergeDialog(filename: FilePathWithPrefix, conflictCheckResult: diff_result): Promise<boolean> {
-        return new Promise((res, rej) => {
-            Logger("open conflict dialog", LOG_LEVEL.VERBOSE);
-            new ConflictResolveModal(this.app, filename, conflictCheckResult, async (selected) => {
-                const testDoc = await this.localDatabase.getDBEntry(filename, { conflicts: true }, false, false, true);
-                if (testDoc === false) {
-                    Logger("Missing file..", LOG_LEVEL.VERBOSE);
-                    return res(true);
-                }
-                if (!testDoc._conflicts) {
-                    Logger("Nothing have to do with this conflict", LOG_LEVEL.VERBOSE);
-                    return res(true);
-                }
-                const toDelete = selected;
-                const toKeep = conflictCheckResult.left.rev != toDelete ? conflictCheckResult.left.rev : conflictCheckResult.right.rev;
-                if (toDelete == "") {
-                    // concat both,
-                    // delete conflicted revision and write a new file, store it again.
-                    const p = conflictCheckResult.diff.map((e) => e[1]).join("");
-                    await this.localDatabase.deleteDBEntry(filename, { rev: testDoc._conflicts[0] });
-                    const file = getAbstractFileByPath(stripAllPrefixes(filename)) as TFile;
-                    if (file) {
-                        await this.app.vault.modify(file, p);
-                        await this.updateIntoDB(file);
+        return runWithLock("resolve-conflict:" + filename, false, () =>
+            new Promise((res, rej) => {
+                Logger("open conflict dialog", LOG_LEVEL.VERBOSE);
+                new ConflictResolveModal(this.app, filename, conflictCheckResult, async (selected) => {
+                    const testDoc = await this.localDatabase.getDBEntry(filename, { conflicts: true }, false, false, true);
+                    if (testDoc === false) {
+                        Logger("Missing file..", LOG_LEVEL.VERBOSE);
+                        return res(true);
+                    }
+                    if (!testDoc._conflicts) {
+                        Logger("Nothing have to do with this conflict", LOG_LEVEL.VERBOSE);
+                        return res(true);
+                    }
+                    const toDelete = selected;
+                    const toKeep = conflictCheckResult.left.rev != toDelete ? conflictCheckResult.left.rev : conflictCheckResult.right.rev;
+                    if (toDelete == "") {
+                        // concat both,
+                        // delete conflicted revision and write a new file, store it again.
+                        const p = conflictCheckResult.diff.map((e) => e[1]).join("");
+                        await this.localDatabase.deleteDBEntry(filename, { rev: testDoc._conflicts[0] });
+                        const file = getAbstractFileByPath(stripAllPrefixes(filename)) as TFile;
+                        if (file) {
+                            await this.app.vault.modify(file, p);
+                            await this.updateIntoDB(file);
+                        } else {
+                            const newFile = await this.app.vault.create(filename, p);
+                            await this.updateIntoDB(newFile);
+                        }
+                        await this.pullFile(filename);
+                        Logger("concat both file");
+                        if (this.settings.syncAfterMerge && !this.suspended) {
+                            await this.replicate();
+                        }
+                        setTimeout(() => {
+                            //resolved, check again.
+                            this.showIfConflicted(filename);
+                        }, 500);
+                    } else if (toDelete == null) {
+                        Logger("Leave it still conflicted");
                     } else {
-                        const newFile = await this.app.vault.create(filename, p);
-                        await this.updateIntoDB(newFile);
+                        await this.localDatabase.deleteDBEntry(filename, { rev: toDelete });
+                        await this.pullFile(filename, null, true, toKeep);
+                        Logger(`Conflict resolved:${filename}`);
+                        if (this.settings.syncAfterMerge && !this.suspended) {
+                            await this.replicate();
+                        }
+                        setTimeout(() => {
+                            //resolved, check again.
+                            this.showIfConflicted(filename);
+                        }, 500);
                     }
-                    await this.pullFile(filename);
-                    Logger("concat both file");
-                    if (this.settings.syncAfterMerge && !this.suspended) {
-                        await this.replicate();
-                    }
-                    setTimeout(() => {
-                        //resolved, check again.
-                        this.showIfConflicted(filename);
-                    }, 500);
-                } else if (toDelete == null) {
-                    Logger("Leave it still conflicted");
-                } else {
-                    await this.localDatabase.deleteDBEntry(filename, { rev: toDelete });
-                    await this.pullFile(filename, null, true, toKeep);
-                    Logger(`Conflict resolved:${filename}`);
-                    if (this.settings.syncAfterMerge && !this.suspended) {
-                        await this.replicate();
-                    }
-                    setTimeout(() => {
-                        //resolved, check again.
-                        this.showIfConflicted(filename);
-                    }, 500);
-                }
 
-                return res(true);
-            }).open();
-        });
+                    return res(true);
+                }).open();
+            })
+        );
     }
     conflictedCheckFiles: FilePath[] = [];
 

src/utils.ts

@@ -1,10 +1,12 @@
-import { DataWriteOptions, normalizePath, TFile, Platform, TAbstractFile, App, Plugin_2 } from "./deps";
+import { DataWriteOptions, normalizePath, TFile, Platform, TAbstractFile, App, Plugin_2, RequestUrlParam, requestUrl } from "./deps";
 import { path2id_base, id2path_base, isValidFilenameInLinux, isValidFilenameInDarwin, isValidFilenameInWidows, isValidFilenameInAndroid, stripAllPrefixes } from "./lib/src/path";
 
 import { Logger } from "./lib/src/logger";
-import { AnyEntry, DocumentID, EntryHasPath, FilePath, FilePathWithPrefix, LOG_LEVEL } from "./lib/src/types";
+import { AnyEntry, DocumentID, EntryDoc, EntryHasPath, FilePath, FilePathWithPrefix, LOG_LEVEL, NewEntry } from "./lib/src/types";
 import { CHeader, ICHeader, ICHeaderLength, PSCHeader } from "./types";
 import { InputStringDialog, PopoverSelectString } from "./dialogs";
+import ObsidianLiveSyncPlugin from "./main";
+import { runWithLock } from "./lib/src/lock";
 
 // For backward compatibility, using the path for determining id.
 // Only CouchDB unacceptable ID (that starts with an underscore) has been prefixed with "/".
@@ -325,14 +327,16 @@ export function isValidPath(filename: string) {
 let touchedFiles: string[] = [];
 
 export function getAbstractFileByPath(path: FilePath): TAbstractFile | null {
-    // Hidden API but so useful.
-    // @ts-ignore
-    if ("getAbstractFileByPathInsensitive" in app.vault && (app.vault.adapter?.insensitive ?? false)) {
-        // @ts-ignore
-        return app.vault.getAbstractFileByPathInsensitive(path);
-    } else {
-        return app.vault.getAbstractFileByPath(path);
-    }
+    // Disabled temporary.
+    return app.vault.getAbstractFileByPath(path);
+    // // Hidden API but so useful.
+    // // @ts-ignore
+    // if ("getAbstractFileByPathInsensitive" in app.vault && (app.vault.adapter?.insensitive ?? false)) {
+    //     // @ts-ignore
+    //     return app.vault.getAbstractFileByPathInsensitive(path);
+    // } else {
+    //    return app.vault.getAbstractFileByPath(path);
+    // }
 }
 export function trimPrefix(target: string, prefix: string) {
     return target.startsWith(prefix) ? target.substring(prefix.length) : target;
@@ -426,3 +430,292 @@ export class PeriodicProcessor {
         if (this._timer) clearInterval(this._timer);
     }
 }
+
+function sizeToHumanReadable(size: number | undefined) {
+    if (!size) return "-";
+    const i = Math.floor(Math.log(size) / Math.log(1024));
+    return Number.parseInt((size / Math.pow(1024, i)).toFixed(2)) + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
+}
+
+export const _requestToCouchDBFetch = async (baseUri: string, username: string, password: string, path?: string, body?: string | any, method?: string) => {
+    const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${username}:${password}`));
+    const encoded = window.btoa(utf8str);
+    const authHeader = "Basic " + encoded;
+    const transformedHeaders: Record<string, string> = { authorization: authHeader, "content-type": "application/json" };
+    const uri = `${baseUri}/${path}`;
+    const requestParam = {
+        url: uri,
+        method: method || (body ? "PUT" : "GET"),
+        headers: new Headers(transformedHeaders),
+        contentType: "application/json",
+        body: JSON.stringify(body),
+    };
+    return await fetch(uri, requestParam);
+}
+
+export const _requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, path?: string, body?: any, method?: string) => {
+    const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${username}:${password}`));
+    const encoded = window.btoa(utf8str);
+    const authHeader = "Basic " + encoded;
+    const transformedHeaders: Record<string, string> = { authorization: authHeader, origin: origin };
+    const uri = `${baseUri}/${path}`;
+    const requestParam: RequestUrlParam = {
+        url: uri,
+        method: method || (body ? "PUT" : "GET"),
+        headers: transformedHeaders,
+        contentType: "application/json",
+        body: body ? JSON.stringify(body) : undefined,
+    };
+    return await requestUrl(requestParam);
+}
+export const requestToCouchDB = async (baseUri: string, username: string, password: string, origin: string, key?: string, body?: string, method?: string) => {
+    const uri = `_node/_local/_config${key ? "/" + key : ""}`;
+    return await _requestToCouchDB(baseUri, username, password, origin, uri, body, method);
+};
+
+export async function performRebuildDB(plugin: ObsidianLiveSyncPlugin, method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice") {
+    if (method == "localOnly") {
+        await plugin.addOnSetup.fetchLocal();
+    }
+    if (method == "remoteOnly") {
+        await plugin.addOnSetup.rebuildRemote();
+    }
+    if (method == "rebuildBothByThisDevice") {
+        await plugin.addOnSetup.rebuildEverything();
+    }
+}
+
+export const gatherChunkUsage = async (db: PouchDB.Database<EntryDoc>) => {
+    const used = new Map();
+    const unreferenced = new Map();
+    const removed = new Map();
+    const missing = new Map();
+    const xx = await db.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
+    for (const xxd of xx.rows) {
+        const chunk = xxd.id
+        unreferenced.set(chunk, xxd.value.rev);
+    }
+
+    const x = await db.find({ limit: 999999999, selector: { children: { $exists: true, $type: "array" } }, fields: ["_id", "path", "mtime", "children"] });
+    for (const temp of x.docs) {
+        for (const chunk of (temp as NewEntry).children) {
+            used.set(chunk, (used.has(chunk) ? used.get(chunk) : 0) + 1);
+            if (unreferenced.has(chunk)) {
+                removed.set(chunk, unreferenced.get(chunk));
+                unreferenced.delete(chunk);
+            } else {
+                if (!removed.has(chunk)) {
+                    if (!missing.has(temp._id)) {
+                        missing.set(temp._id, []);
+                    }
+                    missing.get(temp._id).push(chunk);
+                }
+            }
+        }
+    }
+
+    return { used, unreferenced, missing };
+}
+
+export const localDatabaseCleanUp = async (plugin: ObsidianLiveSyncPlugin, force: boolean, dryRun: boolean) => {
+
+    await runWithLock("clean-up:local", true, async () => {
+        const db = plugin.localDatabase.localDatabase;
+        if ((db as any)?.adapter != "indexeddb") {
+            if (force && !dryRun) {
+                Logger("Fetch from the remote database", LOG_LEVEL.NOTICE, "clean-up-db");
+                await performRebuildDB(plugin, "localOnly");
+                return;
+            } else {
+                Logger("This feature requires enabling `Use new adapter`. Please enable it", LOG_LEVEL.NOTICE, "clean-up-db");
+                return;
+            }
+        }
+        Logger(`The remote database has been locked for garbage collection`, LOG_LEVEL.NOTICE, "clean-up-db");
+        Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
+
+        const { unreferenced, missing } = await gatherChunkUsage(db);
+        if (missing.size != 0) {
+            Logger(`Some chunks are not found! We have to rescue`, LOG_LEVEL.NOTICE);
+            Logger(missing, LOG_LEVEL.VERBOSE);
+        } else {
+            Logger(`All chunks are OK`, LOG_LEVEL.NOTICE);
+        }
+        const payload = {} as Record<string, string[]>;
+        for (const [id, rev] of unreferenced) {
+            payload[id] = [rev];
+        }
+        const removeItems = Object.keys(payload).length;
+        if (removeItems == 0) {
+            Logger(`No unreferenced chunks found (Local)`, LOG_LEVEL.NOTICE);
+            await plugin.markRemoteResolved();
+        }
+        if (dryRun) {
+            Logger(`There are ${removeItems} unreferenced chunks (Local)`, LOG_LEVEL.NOTICE);
+            return;
+        }
+        Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
+        for (const [id, rev] of unreferenced) {
+            //@ts-ignore
+            const ret = await db.purge(id, rev);
+            Logger(ret, LOG_LEVEL.VERBOSE);
+        }
+        plugin.localDatabase.refreshSettings();
+        Logger(`Compacting local database...`, LOG_LEVEL.NOTICE, "clean-up-db");
+        await db.compact();
+        await plugin.markRemoteResolved();
+        Logger("Done!", LOG_LEVEL.NOTICE, "clean-up-db");
+    })
+}
+
+
+export const balanceChunks = async (plugin: ObsidianLiveSyncPlugin, dryRun: boolean) => {
+
+    await runWithLock("clean-up:balance", true, async () => {
+        const localDB = plugin.localDatabase.localDatabase;
+        Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
+
+        const ret = await plugin.replicator.connectRemoteCouchDBWithSetting(plugin.settings, plugin.isMobile);
+        if (typeof ret === "string") {
+            Logger(`Connect error: ${ret}`, LOG_LEVEL.NOTICE, "clean-up-db");
+            return;
+        }
+        const localChunks = new Map<string, string>();
+        const xx = await localDB.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
+        for (const xxd of xx.rows) {
+            const chunk = xxd.id
+            localChunks.set(chunk, xxd.value.rev);
+        }
+        // const info = ret.info;
+        const remoteDB = ret.db;
+        const remoteChunks = new Map<string, string>();
+        const xxr = await remoteDB.allDocs({ startkey: "h:", endkey: `h:\u{10ffff}` });
+        for (const xxd of xxr.rows) {
+            const chunk = xxd.id
+            remoteChunks.set(chunk, xxd.value.rev);
+        }
+        const localToRemote = new Map<string, string>([...localChunks]);
+        const remoteToLocal = new Map<string, string>([...remoteChunks]);
+        for (const id of new Set([...localChunks.keys(), ...remoteChunks.keys()])) {
+            if (remoteChunks.has(id)) {
+                localToRemote.delete(id);
+            }
+            if (localChunks.has(id)) {
+                remoteToLocal.delete(id);
+            }
+        }
+
+        function arrayToChunkedArray<T>(src: T[], size = 25) {
+            const ret = [] as T[][];
+            let i = 0;
+            while (i < src.length) {
+                ret.push(src.slice(i, i += size));
+            }
+            return ret;
+        }
+
+        if (localToRemote.size == 0) {
+            Logger(`No chunks need to be sent`, LOG_LEVEL.NOTICE);
+        } else {
+            Logger(`${localToRemote.size} chunks need to be sent`, LOG_LEVEL.NOTICE);
+            if (!dryRun) {
+                const w = arrayToChunkedArray([...localToRemote]);
+                for (const chunk of w) {
+                    for (const [id,] of chunk) {
+                        const queryRet = await localDB.allDocs({ keys: [id], include_docs: true });
+                        const docs = queryRet.rows.filter(e => !("error" in e)).map(x => x.doc);
+
+                        const ret = await remoteDB.bulkDocs(docs, { new_edits: false });
+                        Logger(ret, LOG_LEVEL.VERBOSE);
+                    }
+                }
+                Logger(`Done! ${remoteToLocal.size} chunks have been sent`, LOG_LEVEL.NOTICE);
+            }
+        }
+        if (remoteToLocal.size == 0) {
+            Logger(`No chunks need to be retrieved`, LOG_LEVEL.NOTICE);
+        } else {
+            Logger(`${remoteToLocal.size} chunks need to be retrieved`, LOG_LEVEL.NOTICE);
+            if (!dryRun) {
+                const w = arrayToChunkedArray([...remoteToLocal]);
+                for (const chunk of w) {
+                    for (const [id,] of chunk) {
+                        const queryRet = await remoteDB.allDocs({ keys: [id], include_docs: true });
+                        const docs = queryRet.rows.filter(e => !("error" in e)).map(x => x.doc);
+
+                        const ret = await localDB.bulkDocs(docs, { new_edits: false });
+                        Logger(ret, LOG_LEVEL.VERBOSE);
+                    }
+                }
+                Logger(`Done! ${remoteToLocal.size} chunks have been retrieved`, LOG_LEVEL.NOTICE);
+            }
+        }
+    })
+}
+
+export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryRun: boolean) => {
+    const getSize = function (info: PouchDB.Core.DatabaseInfo, key: "active" | "external" | "file") {
+        return Number.parseInt((info as any)?.sizes?.[key] ?? 0);
+    }
+    await runWithLock("clean-up:remote", true, async () => {
+        try {
+            const ret = await plugin.replicator.connectRemoteCouchDBWithSetting(plugin.settings, plugin.isMobile);
+            if (typeof ret === "string") {
+                Logger(`Connect error: ${ret}`, LOG_LEVEL.NOTICE, "clean-up-db");
+                return;
+            }
+            const info = ret.info;
+            Logger(JSON.stringify(info), LOG_LEVEL.VERBOSE, "clean-up-db");
+            Logger(`Database active-size: ${sizeToHumanReadable(getSize(info, "active"))}, external-size:${sizeToHumanReadable(getSize(info, "external"))}, file-size: ${sizeToHumanReadable(getSize(info, "file"))}`, LOG_LEVEL.NOTICE);
+
+            if (!dryRun) {
+                Logger(`The remote database has been locked for garbage collection`, LOG_LEVEL.NOTICE, "clean-up-db");
+                await plugin.markRemoteLocked(true);
+            }
+            Logger(`Gathering chunk usage information`, LOG_LEVEL.NOTICE, "clean-up-db");
+            const db = ret.db;
+
+            const { unreferenced, missing } = await gatherChunkUsage(db);
+            if (missing.size != 0) {
+                Logger(`Some chunks are not found! We have to rescue`, LOG_LEVEL.NOTICE);
+                Logger(missing, LOG_LEVEL.VERBOSE);
+            } else {
+                Logger(`All chunks are OK`, LOG_LEVEL.NOTICE);
+            }
+            const payload = {} as Record<string, string[]>;
+            for (const [id, rev] of unreferenced) {
+                payload[id] = [rev];
+            }
+            const removeItems = Object.keys(payload).length;
+            if (removeItems == 0) {
+                Logger(`No unreferenced chunk found (Remote)`, LOG_LEVEL.NOTICE);
+                return;
+            }
+            if (dryRun) {
+                Logger(`There are ${removeItems} unreferenced chunks (Remote)`, LOG_LEVEL.NOTICE);
+                return;
+            }
+            Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
+            const rets = await _requestToCouchDBFetch(
+                `${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
+                plugin.settings.couchDB_USER,
+                plugin.settings.couchDB_PASSWORD,
+                "_purge",
+                payload, "POST");
+            // const result = await rets();
+            Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
+            Logger(`Compacting database...`, LOG_LEVEL.NOTICE, "clean-up-db");
+            await db.compact();
+            const endInfo = await db.info();
+
+            Logger(`Processed database active-size: ${sizeToHumanReadable(getSize(endInfo, "active"))}, external-size:${sizeToHumanReadable(getSize(endInfo, "external"))}, file-size: ${sizeToHumanReadable(getSize(endInfo, "file"))}`, LOG_LEVEL.NOTICE);
+            Logger(`Reduced sizes: active-size: ${sizeToHumanReadable(getSize(info, "active") - getSize(endInfo, "active"))}, external-size:${sizeToHumanReadable(getSize(info, "external") - getSize(endInfo, "external"))}, file-size: ${sizeToHumanReadable(getSize(info, "file") - getSize(endInfo, "file"))}`, LOG_LEVEL.NOTICE);
+            Logger(JSON.stringify(endInfo), LOG_LEVEL.VERBOSE, "clean-up-db");
+            Logger(`Local database cleaning up...`);
+            await localDatabaseCleanUp(plugin, true, false);
+        } catch (ex) {
+            Logger("Failed to clean up db.")
+            Logger(ex, LOG_LEVEL.VERBOSE);
+        }
+    });
+}

updates.md

@@ -20,37 +20,24 @@ Note: **When changing this configuration, we need to rebuild both of the local a
     - No longer `Touch hidden files`.
 - 0.18.3
   - Fixed Pop-up is now correctly shown after hidden file synchronisation.
-
-### 0.17.0
-- 0.17.0 has no surfaced changes but the design of saving chunks has been changed. They have compatibility but changing files after upgrading makes different chunks than before 0.16.x.
-  Please rebuild databases once if you have been worried about storage usage.
-
+- 0.18.4
+  - Fixed:
+      - `Fetch` and `Rebuild database` will work more safely.
+      - Case-sensitive renaming now works fine.
+        Revoked the logic which was made at #130, however, looks fine now.
+- 0.18.5
   - Improved:
-    - Splitting markdown
-    - Saving chunks
+    - Actions for maintaining databases moved to the `🎛️Maintain databases`.
+    - Clean-up of unreferenced chunks has been implemented on an **experimental**.
+      - This feature requires enabling `Use new adapter`.
+      - Be sure to fully all devices synchronised before perform it.
+      - After cleaning up the remote, all devices will be locked out. If we are sure had it be synchronised, we can perform only cleaning-up locally. If not, we have to perform `Fetch`.
 
-  - Changed:
-    - Chunk ID numbering rules
+- 0.18.6
+  - New features:
+    - Now remote database cleaning-up will be detected automatically.
+    - A solution selection dialogue will be shown if synchronisation is rejected after cleaning or rebuilding the remote database.
+    - During fetching or rebuilding, we can configure `Hidden file synchronisation` on the spot.
+      - It let us free from conflict resolution on initial synchronising.
 
-#### Minors
-- __0.17.1 to 0.17.30 has been moved into `update_old.md`__
-- 0.17.31
-  - Fixed:
-    - Now `redflag3` can be run surely.
-    - Synchronisation can now be aborted.
-  - Note: The synchronisation flow has been rewritten drastically. Please do not haste to inform me if you have noticed anything.
-- 0.17.32
-  - Fixed:
-    - Now periodic internal file scanning works well.
-    - The handler of Window-visibility-changed has been fixed.
-    - And minor fixes possibly included.
-  - Refactored:
-    - Unused logic has been removed.
-    - Some utility functions have been moved into suitable files.
-    - Function names have been renamed.
-- 0.17.33
-  - Maintenance update: Refactored; the responsibilities that `LocalDatabase` had were shared. (Hoping) No changes in behaviour.
-- 0.17.34
-  - Fixed: The `Fetch` that was broken at 0.17.33 has been fixed.
-  - Refactored again: Internal file sync, plug-in sync and Set up URI have been moved into each file.
-... To continue on to `updates_old.md`.
+... To continue on to `updates_old.md`.

updates_old.md

@@ -155,6 +155,26 @@
     - Fixed a problem about reading chunks online when a file has more chunks than the concurrency limit.
   - Rollbacked:
     - Logs are kept only for 100 lines, again.
+- 0.17.31
+  - Fixed:
+    - Now `redflag3` can be run surely.
+    - Synchronisation can now be aborted.
+  - Note: The synchronisation flow has been rewritten drastically. Please do not haste to inform me if you have noticed anything.
+- 0.17.32
+  - Fixed:
+    - Now periodic internal file scanning works well.
+    - The handler of Window-visibility-changed has been fixed.
+    - And minor fixes possibly included.
+  - Refactored:
+    - Unused logic has been removed.
+    - Some utility functions have been moved into suitable files.
+    - Function names have been renamed.
+- 0.17.33
+  - Maintenance update: Refactored; the responsibilities that `LocalDatabase` had were shared. (Hoping) No changes in behaviour.
+- 0.17.34
+  - Fixed: The `Fetch` that was broken at 0.17.33 has been fixed.
+  - Refactored again: Internal file sync, plug-in sync and Set up URI have been moved into each file.
+
 ### 0.16.0
 - Now hidden files need not be scanned. Changes will be detected automatically.
   - If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`.