Fixed boot issue and Improved deletion

- fixed bootup delay when vault contains many files.
- status bar improvement.
- add new feature, using trash when file has been delete in remote.

README.md

@@ -7,7 +7,7 @@ Runs in Mac, Android, Windows, and iOS.
 
 ![obsidian_live_sync_demo](https://user-images.githubusercontent.com/45774780/137355323-f57a8b09-abf2-4501-836c-8cb7d2ff24a3.gif)
 
-**It's beta. Please make sure to back your vault up!**
+**It's getting almost stable now, But Please make sure to back your vault up!**
 
 Limitations: Folder deletion handling is not completed.
 
@@ -16,6 +16,7 @@ Limitations: Folder deletion handling is not completed.
 -   Live Sync
 -   Self-Hosted data synchronization with conflict detection and resolving in Obsidian.
 -   Off-line sync is also available.
+-   Receive WebClip from [obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf)
 
 ## IMPORTANT NOTICE
 
@@ -32,20 +33,48 @@ If you want to synchronize to both backend, sync one by one, please.
    Setup details are in Couldant Setup Section.
 5. Setup LiveSync or SyncOnSave or SyncOnStart as you like.
 
-## When your database looks corrupted
+## Test Server
+
+Setting up an instance of Cloudant or local CouchDB is a little complicated, so I made the [Tasting server of obsidian-livesync](https://olstaste.vrtmrz.net/) up. Try free!  
+Note: Please read "Limitations" carefully. Do not send your private vault.
+
+## WebClipper is also available.
+
+Available from on Chrome Web Store:[obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf)  
+Repo is here: [obsidian-livesync-webclip](https://github.com/vrtmrz/obsidian-livesync-webclip). (Docs are work in progress.)
+
+## When your database looks corrupted or too heavy to replicate to a new device.
 
 obsidian-livesync changes data treatment of markdown files since 0.1.0  
 When you are troubled with synchronization, **Please reset local and remote databases**.  
 _Note: Without synchronization, your files won't be deleted._
 
+1.  Update plugin on all devices.
 1.  Disable any synchronizations on all devices.
-2.  From the most reliable device<sup>(_The device_)</sup>, back your vault up.
-3.  Click "Reset local database" on all devices.
-4.  From _The device_ click "Reset remote database".
-5.  From _The device_ click "Init Database again".
-6.  Enable any sync or Hit the Replication button.
+1.  From the most reliable device<sup>(_The device_)</sup>, back your vault up.
+1.  Press "Drop History"-> "Execute" button from _The device_.
+1.  Wait for a while, so obsidian-livesync will say "completed."
+1.  In other devices, replication will be canceled automatically. Click "Reset local database" and click "I'm ready, mark this device 'resolved'" on all devices.  
+    If it doesn't be shown. replicate once.
+1.  It's all done. But if you are sure to resolve all devices and the warning is noisy, click "I'm ready, unlock the database". it unlocks the database completely.
+
+# Designed architecture
+
+## How does this plugin synchronize.
+
+![Synchronization](images/1.png)
+
+1. When notes are created or modified, Obsidian raises some events. obsidian-live-sync catch these events and reflect changes into Local PouchDB.
+2. PouchDB automatically or manually replicates changes to remote CouchDB.
+3. Another device is watching remote CouchDB's changes, so retrieve new changes.
+4. obsidian-live-sync reflects replicated changeset into Obsidian's vault.
+
+Note: The figure is drawn as single-directional, between two devices. But everything occurs bi-directionally between many devices at once in real.
+
+## Techniques to keep bandwidth low.
+
+![dedupe](images/2.png)
 
-And wait for a minute. your data will be uploaded and synchronized with all devices again.
 
 ## Cloudant Setup
 

main.ts

@@ -1,4 +1,4 @@
-import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder } from "obsidian";
+import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder, normalizePath } from "obsidian";
 import { PouchDB } from "./pouchdb-browser-webpack/dist/pouchdb-browser";
 import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
 import xxhash from "xxhash-wasm";
@@ -21,6 +21,8 @@ const LOG_LEVEL = {
 type LOG_LEVEL = typeof LOG_LEVEL[keyof typeof LOG_LEVEL];
 
 const VERSIONINFO_DOCID = "obsydian_livesync_version";
+const MILSTONE_DOCID = "_local/obsydian_livesync_milestone";
+const NODEINFO_DOCID = "_local/obsydian_livesync_nodeinfo";
 
 interface ObsidianLiveSyncSettings {
     couchDB_URI: string;
@@ -37,6 +39,7 @@ interface ObsidianLiveSyncSettings {
     longLineThreshold: number;
     showVerboseLog: boolean;
     suspendFileWatching: boolean;
+    trashInsteadDelete: boolean;
 }
 
 const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
@@ -54,6 +57,7 @@ const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
     longLineThreshold: 250,
     showVerboseLog: false,
     suspendFileWatching: false,
+    trashInsteadDelete: false,
 };
 interface Entry {
     _id: string;
@@ -110,8 +114,27 @@ interface EntryVersionInfo {
     version: number;
     _deleted?: boolean;
 }
+
+interface EntryMilestoneInfo {
+    _id: typeof MILSTONE_DOCID;
+    _rev?: string;
+    type: "milestoneinfo";
+    _deleted?: boolean;
+    created: number;
+    accepted_nodes: string[];
+    locked: boolean;
+}
+
+interface EntryNodeInfo {
+    _id: typeof NODEINFO_DOCID;
+    _rev?: string;
+    _deleted?: boolean;
+    type: "nodeinfo";
+    nodeid: string;
+}
+
 type EntryBody = Entry | NewEntry | PlainEntry;
-type EntryDoc = EntryBody | LoadedEntry | EntryLeaf | EntryVersionInfo;
+type EntryDoc = EntryBody | LoadedEntry | EntryLeaf | EntryVersionInfo | EntryMilestoneInfo | EntryNodeInfo;
 
 type diff_result_leaf = {
     rev: string;
@@ -195,6 +218,13 @@ const escapeStringToHTML = (str: string) => {
         return escape[match];
     });
 };
+
+function resolveWithIgnoreKnownError<T>(p: Promise<T>, def: T): Promise<T> {
+    return new Promise((res, rej) => {
+        p.then(res).catch((ex) => (ex.status && ex.status == 404 ? res(def) : rej(ex)));
+    });
+}
+
 const isValidRemoteCouchDBURI = (uri: string): boolean => {
     if (uri.startsWith("https://")) return true;
     if (uri.startsWith("http://")) return true;
@@ -251,22 +281,21 @@ const bumpRemoteVersion = async (db: PouchDB.Database, barrier: number = VER): P
         version: barrier,
         type: "versioninfo",
     };
-    try {
-        let versionInfo = (await db.get(VERSIONINFO_DOCID)) as EntryVersionInfo;
-        if (versionInfo.type != "versioninfo") {
-            return false;
-        }
-        vi._rev = versionInfo._rev;
-    } catch (ex) {
-        if (ex.status && ex.status == 404) {
-            // no op.
-        } else {
-            throw ex;
-        }
+    let versionInfo = (await resolveWithIgnoreKnownError(db.get(VERSIONINFO_DOCID), vi)) as EntryVersionInfo;
+    if (versionInfo.type != "versioninfo") {
+        return false;
     }
+    vi._rev = versionInfo._rev;
     await db.put(vi);
     return true;
 };
+function isValidPath(filename: string): boolean {
+    let regex = /[\u0000-\u001f]|[\\"':?<>|*$]/g;
+    let x = filename.replace(regex, "_");
+    let win = /(\\|\/)(COM\d|LPT\d|CON|PRN|AUX|NUL|CLOCK$)($|\.)/gi;
+    let sx = (x = x.replace(win, "/_"));
+    return sx == filename;
+}
 
 // Default Logger.
 let Logger: (message: any, levlel?: LOG_LEVEL) => Promise<void> = async (message, _) => {
@@ -276,12 +305,15 @@ let Logger: (message: any, levlel?: LOG_LEVEL) => Promise<void> = async (message
     console.log(newmessage);
 };
 
+type DatabaseConnectingStatus = "NOT_CONNECTED" | "PAUSED" | "CONNECTED" | "COMPLETED" | "CLOSED" | "ERRORED";
+
 //<--Functions
 class LocalPouchDB {
     auth: Credential;
     dbname: string;
     settings: ObsidianLiveSyncSettings;
     localDatabase: PouchDB.Database<EntryDoc>;
+    nodeid: string = "";
 
     recentModifiedDocs: string[] = [];
     h32: (input: string, seed?: number) => string;
@@ -294,6 +326,8 @@ class LocalPouchDB {
     } = {};
 
     corruptedEntries: { [key: string]: EntryDoc } = {};
+    remoteLocked = false;
+    remoteLockedAndDeviceNotAccepted = false;
 
     constructor(settings: ObsidianLiveSyncSettings, dbname: string) {
         this.auth = {
@@ -345,6 +379,17 @@ class LocalPouchDB {
             revs_limit: 100,
             deterministic_revs: true,
         });
+        // initialize local node information.
+        let nodeinfo: EntryNodeInfo = await resolveWithIgnoreKnownError<EntryNodeInfo>(this.localDatabase.get(NODEINFO_DOCID), {
+            _id: NODEINFO_DOCID,
+            type: "nodeinfo",
+            nodeid: "",
+        });
+        if (nodeinfo.nodeid == "") {
+            nodeinfo.nodeid = Math.random().toString(36).slice(-10);
+            await this.localDatabase.put(nodeinfo);
+        }
+        this.nodeid = nodeinfo.nodeid;
 
         // Traceing the leaf id
         let changes = this.localDatabase
@@ -356,6 +401,7 @@ class LocalPouchDB {
             .on("change", (e) => {
                 if (e.deleted) return;
                 this.leafArrived(e.id);
+                this.docSeq = `${e.seq}`;
             });
         this.changeHandler = changes;
         await this.prepareHashFunctions();
@@ -438,6 +484,44 @@ class LocalPouchDB {
         }
     }
 
+    async getDBEntryMeta(id: string, opt?: PouchDB.Core.GetOptions): Promise<false | LoadedEntry> {
+        try {
+            let obj: EntryDocResponse = null;
+            if (opt) {
+                obj = await this.localDatabase.get(id, opt);
+            } else {
+                obj = await this.localDatabase.get(id);
+            }
+
+            if (obj.type && obj.type == "leaf") {
+                //do nothing for leaf;
+                return false;
+            }
+
+            // retrieve metadata only
+            if (!obj.type || (obj.type && obj.type == "notes") || obj.type == "newnote" || obj.type == "plain") {
+                let note = obj as Entry;
+                let doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = {
+                    data: "",
+                    _id: note._id,
+                    ctime: note.ctime,
+                    mtime: note.mtime,
+                    size: note.size,
+                    _deleted: obj._deleted,
+                    _rev: obj._rev,
+                    _conflicts: obj._conflicts,
+                    children: [],
+                    datatype: "newnote",
+                };
+            }
+        } catch (ex) {
+            if (ex.status && ex.status == 404) {
+                return false;
+            }
+            throw ex;
+        }
+        return false;
+    }
     async getDBEntry(id: string, opt?: PouchDB.Core.GetOptions, retryCount = 5): Promise<false | LoadedEntry> {
         try {
             let obj: EntryDocResponse = null;
@@ -560,7 +644,54 @@ class LocalPouchDB {
             throw ex;
         }
     }
-
+    async deleteDBEntryPrefix(prefix: string): Promise<boolean> {
+        // delete database entries by prefix.
+        // it called from folder deletion.
+        let c = 0;
+        let readCount = 0;
+        let delDocs: string[] = [];
+        do {
+            let result = await this.localDatabase.allDocs({ include_docs: false, skip: c, limit: 100, conflicts: true });
+            readCount = result.rows.length;
+            if (readCount > 0) {
+                //there are some result
+                for (let v of result.rows) {
+                    // let doc = v.doc;
+                    if (v.id.startsWith(prefix) || v.id.startsWith("/" + prefix)) {
+                        delDocs.push(v.id);
+                        console.log("!" + v.id);
+                    } else {
+                        if (!v.id.startsWith("h:")) {
+                            console.log("?" + v.id);
+                        }
+                    }
+                }
+            }
+            c += readCount;
+        } while (readCount != 0);
+        // items collected.
+        //bulk docs to delete?
+        let deleteCount = 0;
+        let notfound = 0;
+        for (let v of delDocs) {
+            try {
+                let item = await this.localDatabase.get(v);
+                item._deleted = true;
+                await this.localDatabase.put(item);
+                this.updateRecentModifiedDocs(item._id, item._rev, true);
+                deleteCount++;
+            } catch (ex) {
+                if (ex.status && ex.status == 404) {
+                    notfound++;
+                    // NO OP. It should be timing problem.
+                } else {
+                    throw ex;
+                }
+            }
+        }
+        Logger(`deleteDBEntryPrefix:deleted ${deleteCount} items, skipped ${notfound}`);
+        return true;
+    }
     async putDBEntry(note: LoadedEntry) {
         let leftData = note.data;
         let savenNotes = [];
@@ -721,12 +852,72 @@ class LocalPouchDB {
     }
 
     syncHandler: PouchDB.Replication.Sync<{}> = null;
-
+    syncStatus: DatabaseConnectingStatus = "NOT_CONNECTED";
+    docArrived: number = 0;
+    docSent: number = 0;
+    docSeq: string = "";
+    updateInfo: () => void = () => {
+        console.log("default updinfo");
+    };
     async migrate(from: number, to: number): Promise<boolean> {
         Logger(`Database updated from ${from} to ${to}`, LOG_LEVEL.NOTICE);
         // no op now,
         return true;
     }
+    replicateAllToServer(setting: ObsidianLiveSyncSettings) {
+        return new Promise(async (res, rej) => {
+            this.closeReplication();
+            Logger("send all data to server", LOG_LEVEL.NOTICE);
+            this.syncStatus = "CLOSED";
+            this.updateInfo();
+            let uri = setting.couchDB_URI;
+            let auth: Credential = {
+                username: setting.couchDB_USER,
+                password: setting.couchDB_PASSWORD,
+            };
+            let dbret = await connectRemoteCouchDB(uri, auth);
+            if (dbret === false) {
+                Logger(`could not connect to ${uri}`, LOG_LEVEL.NOTICE);
+                return rej(`could not connect to ${uri}`);
+            }
+
+            let syncOptionBase: PouchDB.Replication.SyncOptions = {
+                batch_size: 250,
+                batches_limit: 40,
+            };
+
+            let db = dbret.db;
+            //replicate once
+            let replicate = this.localDatabase.replicate.to(db, syncOptionBase);
+            replicate
+                .on("active", () => {
+                    this.syncStatus = "CONNECTED";
+                    this.updateInfo();
+                })
+                .on("change", async (e) => {
+                    // no op.
+                    this.docSent += e.docs_written;
+                    this.docArrived += e.docs_read;
+                    this.updateInfo();
+                    Logger(`sending..:${e.docs.length}`);
+                })
+                .on("complete", async (info) => {
+                    this.syncStatus = "COMPLETED";
+                    this.updateInfo();
+                    Logger("Completed", LOG_LEVEL.NOTICE);
+                    replicate.cancel();
+                    replicate.removeAllListeners();
+                    res(true);
+                })
+                .on("error", (e) => {
+                    this.syncStatus = "ERRORED";
+                    this.updateInfo();
+                    Logger("Pulling Replication error", LOG_LEVEL.NOTICE);
+                    Logger(e);
+                    rej(e);
+                });
+        });
+    }
     async openReplication(setting: ObsidianLiveSyncSettings, keepAlive: boolean, showResult: boolean, callback: (e: PouchDB.Core.ExistingDocument<{}>[]) => Promise<void>) {
         if (setting.versionUpFlash != "") {
             new Notice("Open settings and check message, please.");
@@ -752,6 +943,26 @@ class LocalPouchDB {
             return;
         }
 
+        let defMilestonePoint: EntryMilestoneInfo = {
+            _id: MILSTONE_DOCID,
+            type: "milestoneinfo",
+            created: (new Date() as any) / 1,
+            locked: false,
+            accepted_nodes: [this.nodeid],
+        };
+
+        let remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defMilestonePoint);
+        this.remoteLocked = remoteMilestone.locked;
+        this.remoteLockedAndDeviceNotAccepted = remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1;
+
+        if (remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1) {
+            Logger("Remote database marked as 'Auto Sync Locked'. And this devide does not marked as resolved device. see settings dialog.", LOG_LEVEL.NOTICE);
+            return;
+        }
+        if (typeof remoteMilestone._rev == "undefined") {
+            await dbret.db.put(remoteMilestone);
+        }
+
         let syncOptionBase: PouchDB.Replication.SyncOptions = {
             batch_size: 250,
             batches_limit: 40,
@@ -760,14 +971,22 @@ class LocalPouchDB {
 
         let db = dbret.db;
         //replicate once
+        this.syncStatus = "CONNECTED";
         let replicate = this.localDatabase.replicate.from(db, syncOptionBase);
         replicate
+            .on("active", () => {
+                this.syncStatus = "CONNECTED";
+                this.updateInfo();
+            })
             .on("change", async (e) => {
                 // when in first run, replication will send us tombstone data
                 // and in normal cases, all leavs should sent before the entry that contains these item.
                 // so skip to completed all, we should treat all changes.
                 try {
                     callback(e.docs);
+                    this.docArrived += e.docs_read;
+                    this.docSent += e.docs_written;
+                    this.updateInfo();
                     Logger(`pulled ${e.docs.length} doc(s)`);
                 } catch (ex) {
                     Logger("Replication callback error");
@@ -775,6 +994,8 @@ class LocalPouchDB {
                 }
             })
             .on("complete", async (info) => {
+                this.syncStatus = "COMPLETED";
+                this.updateInfo();
                 replicate.cancel();
                 replicate.removeAllListeners();
                 this.syncHandler = null;
@@ -785,10 +1006,15 @@ class LocalPouchDB {
                 this.syncHandler = this.localDatabase.sync(db, syncOption);
                 this.syncHandler
                     .on("active", () => {
+                        this.syncStatus = "CONNECTED";
+                        this.updateInfo();
                         Logger("Replication activated");
                     })
                     .on("change", async (e) => {
                         try {
+                            this.docArrived += e.change.docs_read;
+                            this.docSent += e.change.docs_written;
+                            this.updateInfo();
                             callback(e.change.docs);
                             Logger(`replicated ${e.change.docs.length} doc(s)`);
                         } catch (ex) {
@@ -797,23 +1023,33 @@ class LocalPouchDB {
                         }
                     })
                     .on("complete", (e) => {
+                        this.syncStatus = "COMPLETED";
+                        this.updateInfo();
                         Logger("Replication completed", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
                         this.syncHandler = null;
                     })
                     .on("denied", (e) => {
+                        this.syncStatus = "ERRORED";
+                        this.updateInfo();
                         Logger("Replication denied", LOG_LEVEL.NOTICE);
                         // Logger(e);
                     })
                     .on("error", (e) => {
+                        this.syncStatus = "ERRORED";
+                        this.updateInfo();
                         Logger("Replication error", LOG_LEVEL.NOTICE);
                         // Logger(e);
                     })
                     .on("paused", (e) => {
+                        this.syncStatus = "PAUSED";
+                        this.updateInfo();
                         Logger("replication paused", LOG_LEVEL.VERBOSE);
                         // Logger(e);
                     });
             })
             .on("error", (e) => {
+                this.syncStatus = "ERRORED";
+                this.updateInfo();
                 Logger("Pulling Replication error", LOG_LEVEL.NOTICE);
                 Logger(e);
             });
@@ -823,6 +1059,8 @@ class LocalPouchDB {
         if (this.syncHandler == null) {
             return;
         }
+        this.syncStatus = "CLOSED";
+        this.updateInfo();
         this.syncHandler.cancel();
         this.syncHandler.removeAllListeners();
         this.syncHandler = null;
@@ -869,6 +1107,71 @@ class LocalPouchDB {
         if (con2 === false) return;
         Logger("Remote Database Created or Connected", LOG_LEVEL.NOTICE);
     }
+    async markRemoteLocked(setting: ObsidianLiveSyncSettings, locked: boolean) {
+        let uri = setting.couchDB_URI;
+        let auth: Credential = {
+            username: setting.couchDB_USER,
+            password: setting.couchDB_PASSWORD,
+        };
+        let dbret = await connectRemoteCouchDB(uri, auth);
+        if (dbret === false) {
+            Logger(`could not connect to ${uri}`, LOG_LEVEL.NOTICE);
+            return;
+        }
+
+        if (!(await checkRemoteVersion(dbret.db, this.migrate.bind(this), VER))) {
+            Logger("Remote database is newer or corrupted, make sure to latest version of obsidian-livesync installed", LOG_LEVEL.NOTICE);
+            return;
+        }
+        let defInitPoint: EntryMilestoneInfo = {
+            _id: MILSTONE_DOCID,
+            type: "milestoneinfo",
+            created: (new Date() as any) / 1,
+            locked: locked,
+            accepted_nodes: [this.nodeid],
+        };
+
+        let remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint);
+        remoteMilestone.accepted_nodes = [this.nodeid];
+        remoteMilestone.locked = locked;
+        if (locked) {
+            Logger("Lock remote database to prevent data corruption", LOG_LEVEL.NOTICE);
+        } else {
+            Logger("Unlock remote database to prevent data corruption", LOG_LEVEL.NOTICE);
+        }
+        await dbret.db.put(remoteMilestone);
+    }
+    async markRemoteResolved(setting: ObsidianLiveSyncSettings) {
+        let uri = setting.couchDB_URI;
+        let auth: Credential = {
+            username: setting.couchDB_USER,
+            password: setting.couchDB_PASSWORD,
+        };
+        let dbret = await connectRemoteCouchDB(uri, auth);
+        if (dbret === false) {
+            Logger(`could not connect to ${uri}`, LOG_LEVEL.NOTICE);
+            return;
+        }
+
+        if (!(await checkRemoteVersion(dbret.db, this.migrate.bind(this), VER))) {
+            Logger("Remote database is newer or corrupted, make sure to latest version of obsidian-livesync installed", LOG_LEVEL.NOTICE);
+            return;
+        }
+        let defInitPoint: EntryMilestoneInfo = {
+            _id: MILSTONE_DOCID,
+            type: "milestoneinfo",
+            created: (new Date() as any) / 1,
+            locked: false,
+            accepted_nodes: [this.nodeid],
+        };
+        // check local database hash status and remote replicate hash status
+        let remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint);
+        // remoteMilestone.locked = false;
+        remoteMilestone.accepted_nodes = Array.from(new Set([...remoteMilestone.accepted_nodes, this.nodeid]));
+        // this.remoteLocked = false;
+        Logger("Mark this device as 'resolved'.", LOG_LEVEL.NOTICE);
+        await dbret.db.put(remoteMilestone);
+    }
 
     async garbageCollect() {
         // get all documents of NewEntry2
@@ -973,6 +1276,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         });
 
         this.statusBar = this.addStatusBarItem();
+        this.statusBar.addClass("syncstatusbar");
+        this.refreshStatusText = this.refreshStatusText.bind(this);
 
         this.statusBar2 = this.addStatusBarItem();
         let delay = this.settings.savingDelay;
@@ -1064,6 +1369,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
         let vaultName = this.app.vault.getName();
         this.localDatabase = new LocalPouchDB(this.settings, vaultName);
+        this.localDatabase.updateInfo = () => {
+            this.refreshStatusText();
+        };
         await this.localDatabase.initializeDatabase();
     }
     async garbageCollect() {
@@ -1152,8 +1460,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
     //--> Basic document Functions
     async addLog(message: any, level: LOG_LEVEL = LOG_LEVEL.INFO) {
-        // debugger;
-
         if (level < LOG_LEVEL.INFO && this.settings && this.settings.lessInformationInLog) {
             return;
         }
@@ -1167,9 +1473,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
         this.logMessage = [].concat(this.logMessage).concat([newmessage]).slice(-100);
         console.log(valutName + ":" + newmessage);
-        if (this.statusBar2 != null) {
-            this.statusBar2.setText(newmessage.substring(0, 60));
-        }
+        // if (this.statusBar2 != null) {
+        //     this.statusBar2.setText(newmessage.substring(0, 60));
+        // }
         if (level >= LOG_LEVEL.NOTICE) {
             new Notice(messagecontent);
         }
@@ -1202,16 +1508,32 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (doc.datatype == "newnote") {
             let bin = base64ToArrayBuffer(doc.data);
             if (bin != null) {
+                if (!isValidPath(doc._id)) {
+                    Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
+                    return;
+                }
                 await this.ensureDirectory(doc._id);
-                let newfile = await this.app.vault.createBinary(doc._id, bin, { ctime: doc.ctime, mtime: doc.mtime });
-                Logger("live : write to local (newfile:b) " + doc._id);
-                await this.app.vault.trigger("create", newfile);
+                try {
+                    let newfile = await this.app.vault.createBinary(normalizePath(doc._id), bin, { ctime: doc.ctime, mtime: doc.mtime });
+                    Logger("live : write to local (newfile:b) " + doc._id);
+                    await this.app.vault.trigger("create", newfile);
+                } catch (ex) {
+                    Logger("could not write to local (newfile:bin) " + doc._id, LOG_LEVEL.NOTICE);
+                }
             }
         } else if (doc.datatype == "plain") {
+            if (!isValidPath(doc._id)) {
+                Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
+                return;
+            }
             await this.ensureDirectory(doc._id);
-            let newfile = await this.app.vault.create(doc._id, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
-            Logger("live : write to local (newfile:p) " + doc._id);
-            await this.app.vault.trigger("create", newfile);
+            try {
+                let newfile = await this.app.vault.create(normalizePath(doc._id), doc.data, { ctime: doc.ctime, mtime: doc.mtime });
+                Logger("live : write to local (newfile:p) " + doc._id);
+                await this.app.vault.trigger("create", newfile);
+            } catch (ex) {
+                Logger("could not write to local (newfile:plain) " + doc._id, LOG_LEVEL.NOTICE);
+            }
         } else {
             Logger("live : New data imcoming, but we cound't parse that." + doc.datatype, LOG_LEVEL.NOTICE);
         }
@@ -1219,7 +1541,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
     async deleteVaultItem(file: TFile | TFolder) {
         let dir = file.parent;
-        await this.app.vault.delete(file);
+        if (this.settings.trashInsteadDelete) {
+            await this.app.vault.trash(file, false);
+        } else {
+            await this.app.vault.delete(file);
+        }
         Logger(`deleted:${file.path}`);
         Logger(`other items:${dir.children.length}`);
         if (dir.children.length == 0) {
@@ -1251,17 +1577,33 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             if (doc.datatype == "newnote") {
                 let bin = base64ToArrayBuffer(doc.data);
                 if (bin != null) {
+                    if (!isValidPath(doc._id)) {
+                        Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
+                        return;
+                    }
                     await this.ensureDirectory(doc._id);
-                    await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
-                    Logger(msg);
-                    await this.app.vault.trigger("modify", file);
+                    try {
+                        await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
+                        Logger(msg);
+                        await this.app.vault.trigger("modify", file);
+                    } catch (ex) {
+                        Logger("could not write to local (modify:bin) " + doc._id, LOG_LEVEL.NOTICE);
+                    }
                 }
             }
             if (doc.datatype == "plain") {
+                if (!isValidPath(doc._id)) {
+                    Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${doc._id}`, LOG_LEVEL.NOTICE);
+                    return;
+                }
                 await this.ensureDirectory(doc._id);
-                await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
-                Logger(msg);
-                await this.app.vault.trigger("modify", file);
+                try {
+                    await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
+                    Logger(msg);
+                    await this.app.vault.trigger("modify", file);
+                } catch (ex) {
+                    Logger("could not write to local (modify:plain) " + doc._id, LOG_LEVEL.NOTICE);
+                }
             } else {
                 Logger("live : New data imcoming, but we cound't parse that.:" + doc.datatype + "-", LOG_LEVEL.NOTICE);
             }
@@ -1293,12 +1635,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
     //---> Sync
     async parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): Promise<void> {
+        this.refreshStatusText();
         for (var change of docs) {
             if (this.localDatabase.isSelfModified(change._id, change._rev)) {
                 return;
             }
             Logger("replication change arrived", LOG_LEVEL.VERBOSE);
-            if (change.type != "leaf" && change.type != "versioninfo") {
+            if (change.type != "leaf" && change.type != "versioninfo" && change.type != "milestoneinfo" && change.type != "nodeinfo") {
                 await this.handleDBChanged(change);
             }
             if (change.type == "versioninfo") {
@@ -1310,7 +1653,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             this.gcHook();
         }
     }
-    async realizeSettingSyncMode() {
+    realizeSettingSyncMode() {
         this.localDatabase.closeReplication();
         if (this.settings.liveSync) {
             this.localDatabase.openReplication(this.settings, true, false, this.parseReplicationResult);
@@ -1318,8 +1661,29 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
     }
     refreshStatusText() {
-        let statusStr = this.localDatabase.status();
-        this.statusBar.setText("Sync:" + statusStr);
+        let sent = this.localDatabase.docSent;
+        let arrived = this.localDatabase.docArrived;
+        let w = "";
+        switch (this.localDatabase.syncStatus) {
+            case "CLOSED":
+            case "COMPLETED":
+            case "NOT_CONNECTED":
+                w = "⏹";
+                break;
+            case "PAUSED":
+                w = "💤";
+                break;
+
+            case "CONNECTED":
+                w = "⚡";
+                break;
+            case "ERRORED":
+                w = "⚠";
+                break;
+            default:
+                w = "?";
+        }
+        this.statusBar.setText(`Sync:${w} ↑${sent} ↓${arrived}`);
     }
     async replicate(showMessage?: boolean) {
         if (this.settings.versionUpFlash != "") {
@@ -1333,12 +1697,25 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         await this.openDatabase();
         await this.syncAllFiles();
     }
+    async replicateAllToServer() {
+        return await this.localDatabase.replicateAllToServer(this.settings);
+    }
+    async markRemoteLocked() {
+        return await this.localDatabase.markRemoteLocked(this.settings, true);
+    }
+    async markRemoteUnlocked() {
+        return await this.localDatabase.markRemoteLocked(this.settings, false);
+    }
+    async markRemoteResolved() {
+        return await this.localDatabase.markRemoteResolved(this.settings);
+    }
     async syncAllFiles() {
         // synchronize all files between database and storage.
+
         const filesStorage = this.app.vault.getFiles();
         const filesStorageName = filesStorage.map((e) => e.path);
         const wf = await this.localDatabase.localDatabase.allDocs();
-        const filesDatabase = wf.rows.map((e) => e.id);
+        const filesDatabase = wf.rows.filter((e) => !e.id.startsWith("h:")).map((e) => normalizePath(e.id));
 
         const onlyInStorage = filesStorage.filter((e) => filesDatabase.indexOf(e.path) == -1);
         const onlyInDatabase = filesDatabase.filter((e) => filesStorageName.indexOf(e) == -1);
@@ -1346,22 +1723,29 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         const onlyInStorageNames = onlyInStorage.map((e) => e.path);
 
         const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1);
-
+        Logger("Initialize and checking database files");
+        Logger("Updating database by new files");
+        this.statusBar.setText(`UPDATE DATABASE`);
         // just write to DB from storage.
         for (let v of onlyInStorage) {
             await this.updateIntoDB(v);
         }
         // simply realize it
+        this.statusBar.setText(`UPDATE STORAGE`);
+        Logger("Writing files that only in database");
         for (let v of onlyInDatabase) {
             await this.pullFile(v, filesStorage);
         }
         // have to sync below..
+        this.statusBar.setText(`CHECK FILE STATUS`);
         for (let v of syncFiles) {
             await this.syncFileBetweenDBandStorage(v, filesStorage);
         }
+        Logger("Initialized");
     }
     async deleteFolderOnDB(folder: TFolder) {
         Logger(`delete folder:${folder.path}`);
+        await this.localDatabase.deleteDBEntryPrefix(folder.path + "/");
         for (var v of folder.children) {
             let entry = v as TFile & TFolder;
             Logger(`->entry:${entry.path}`, LOG_LEVEL.VERBOSE);
@@ -1369,12 +1753,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 Logger(`->is dir`, LOG_LEVEL.VERBOSE);
                 await this.deleteFolderOnDB(entry);
                 try {
-                    await this.app.vault.delete(entry);
+                    if (this.settings.trashInsteadDelete) {
+                        await this.app.vault.trash(entry, false);
+                    } else {
+                        await this.app.vault.delete(entry);
+                    }
                 } catch (ex) {
                     if (ex.code && ex.code == "ENOENT") {
                         //NO OP.
                     } else {
-                        Logger(`error while delete filder:${entry.path}`, LOG_LEVEL.NOTICE);
+                        Logger(`error while delete folder:${entry.path}`, LOG_LEVEL.NOTICE);
                         Logger(ex);
                     }
                 }
@@ -1384,7 +1772,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             }
         }
         try {
-            await this.app.vault.delete(folder);
+            if (this.settings.trashInsteadDelete) {
+                await this.app.vault.trash(folder, false);
+            } else {
+                await this.app.vault.delete(folder);
+            }
         } catch (ex) {
             if (ex.code && ex.code == "ENOENT") {
                 //NO OP.
@@ -1400,7 +1792,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             let entry = v as TFile & TFolder;
             if (entry.children) {
                 await this.deleteFolderOnDB(entry);
-                await this.app.vault.delete(entry);
+                if (this.settings.trashInsteadDelete) {
+                    await this.app.vault.trash(entry, false);
+                } else {
+                    await this.app.vault.delete(entry);
+                }
             } else {
                 await this.deleteFromDB(entry);
             }
@@ -1528,7 +1924,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (!fileList) {
             fileList = this.app.vault.getFiles();
         }
-        let targetFiles = fileList.filter((e) => e.path == filename);
+        let targetFiles = fileList.filter((e) => e.path == normalizePath(filename));
         if (targetFiles.length == 0) {
             //have to create;
             let doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null);
@@ -1547,7 +1943,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         //when to opened file;
     }
     async syncFileBetweenDBandStorage(file: TFile, fileList?: TFile[]) {
-        let doc = await this.localDatabase.getDBEntry(file.path);
+        let doc = await this.localDatabase.getDBEntryMeta(file.path);
         if (doc === false) return;
         if (file.stat.mtime > doc.mtime) {
             //newer local file.
@@ -1556,7 +1952,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         } else if (file.stat.mtime < doc.mtime) {
             //newer database file.
             Logger("sync : older storage files so write from database:" + file.path);
-            await this.doc2storate_modify(doc, file);
+            let docx = await this.localDatabase.getDBEntry(file.path);
+            if (docx != false) {
+                await this.doc2storate_modify(docx, file);
+            }
         } else {
             //eq.case
         }
@@ -1788,14 +2187,17 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     });
                 text.inputEl.setAttribute("type", "password");
             });
-        new Setting(containerEl).setName("Test DB").addButton((button) =>
-            button
-                .setButtonText("Test Database Connection")
-                .setDisabled(false)
-                .onClick(async () => {
-                    await this.testConnection();
-                })
-        );
+        new Setting(containerEl)
+            .setName("Test Database Connection")
+            .setDesc("Open database connection. If the remote database is not found and you have the privilege to create a database, the database will be created.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Test")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.testConnection();
+                    })
+            );
 
         containerEl.createEl("h3", { text: "Database configuration" });
 
@@ -1897,6 +2299,16 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 })
             );
 
+        new Setting(containerEl)
+            .setName("Trash deleted files")
+            .setDesc("Do not delete files that deleted in remote, just move to trash.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.trashInsteadDelete).onChange(async (value) => {
+                    this.plugin.settings.trashInsteadDelete = value;
+                    await this.plugin.saveSettings();
+                })
+            );
+
         new Setting(containerEl)
             .setName("Minimum chunk size")
             .setDesc("(letters), minimum chunk size.")
@@ -1931,24 +2343,6 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 text.inputEl.setAttribute("type", "number");
             });
 
-        new Setting(containerEl).setName("Local Database Operations").addButton((button) =>
-            button
-                .setButtonText("Reset local database")
-                .setDisabled(false)
-                .onClick(async () => {
-                    await this.plugin.resetLocalDatabase();
-                })
-        );
-        new Setting(containerEl).setName("Re-init").addButton((button) =>
-            button
-                .setButtonText("Init Database again")
-                .setDisabled(false)
-                .onClick(async () => {
-                    await this.plugin.resetLocalDatabase();
-                    await this.plugin.initializeDatabase();
-                })
-        );
-
         new Setting(containerEl).setName("Garbage Collect").addButton((button) =>
             button
                 .setButtonText("Garbage Collection")
@@ -1960,6 +2354,60 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
 
         containerEl.createEl("h3", { text: "Hatch" });
 
+        if (this.plugin.localDatabase.remoteLockedAndDeviceNotAccepted) {
+            let c = containerEl.createEl("div", {
+                text: "To prevent unwanted vault corruption, the remote database has been locked for synchronization, and this device was not marked as 'resolved'. it caused by some operations like this. re-initialized. Local database initialization should be required. please back your vault up, reset local database, and press 'Mark this device as resolved'. ",
+            });
+            c.createEl("button", { text: "I'm ready, mark this device 'resolved'" }, (e) => {
+                e.addEventListener("click", async () => {
+                    await this.plugin.markRemoteResolved();
+                    c.remove();
+                });
+            });
+            c.addClass("op-warn");
+        } else {
+            if (this.plugin.localDatabase.remoteLocked) {
+                let c = containerEl.createEl("div", {
+                    text: "To prevent unwanted vault corruption, the remote database has been locked for synchronization. (This device is marked 'resolved') When all your devices are marked 'resolved', unlock the database.",
+                });
+                c.createEl("button", { text: "I'm ready, unlock the database" }, (e) => {
+                    e.addEventListener("click", async () => {
+                        await this.plugin.markRemoteUnlocked();
+                        c.remove();
+                    });
+                });
+                c.addClass("op-warn");
+            }
+        }
+
+        new Setting(containerEl)
+            .setName("Drop History")
+            .setDesc("Initialize local and remote database, and create local database from storage and put all into server. And also, lock the database to prevent data corruption.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Execute")
+                    .setWarning()
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.resetLocalDatabase();
+                        await this.plugin.initializeDatabase();
+                        await this.plugin.tryResetRemoteDatabase();
+                        await this.plugin.markRemoteLocked();
+                        await this.plugin.replicateAllToServer();
+                    })
+            );
+        new Setting(containerEl)
+            .setName("Lock remote database")
+            .setDesc("Lock remote database for synchronize")
+            .addButton((button) =>
+                button
+                    .setButtonText("Lock")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.markRemoteLocked();
+                    })
+            );
+
         new Setting(containerEl)
             .setName("Suspend file watching")
             .setDesc("if enables it, all file operations are ignored.")
@@ -1970,23 +2418,41 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 })
             );
 
-        new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
-            button
-                .setButtonText("Reset remote database")
-                .setDisabled(false)
-                .onClick(async () => {
-                    await this.plugin.tryResetRemoteDatabase();
-                })
-        );
-        new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
-            button
-                .setButtonText("Create remote database")
-                .setDisabled(false)
-                .onClick(async () => {
-                    await this.plugin.tryResetRemoteDatabase();
-                })
-        );
-
+        new Setting(containerEl)
+            .setName("Reset remote database")
+            .setDesc("Reset remote database, this affects only database. If you replicate again, remote database will restored by local database.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Reset")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.tryResetRemoteDatabase();
+                    })
+            );
+        new Setting(containerEl)
+            .setName("Reset local database")
+            .setDesc("Reset local database, this affects only database. If you replicate again, local database will restored by remote database.")
+            .addButton((button) =>
+                button
+                    .setButtonText("Reset")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.resetLocalDatabase();
+                    })
+            );
+        new Setting(containerEl)
+            .setName("Initialize local database again")
+            .setDesc("WARNING: Reset local database and reconstruct by storage data. It affects local database, but if you replicate remote as is, remote data will be merged or corrupted.")
+            .addButton((button) =>
+                button
+                    .setButtonText("INITIALIZE")
+                    .setWarning()
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.resetLocalDatabase();
+                        await this.plugin.initializeDatabase();
+                    })
+            );
         containerEl.createEl("h3", { text: "Corrupted data" });
 
         if (Object.keys(this.plugin.localDatabase.corruptedEntries).length > 0) {

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Obsidian Live sync",
-    "version": "0.1.6",
+    "version": "0.1.9",
     "minAppVersion": "0.9.12",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.1.6",
+    "version": "0.1.9",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.1.6",
+            "version": "0.1.9",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.1.6",
+    "version": "0.1.9",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "scripts": {

styles.css

@@ -19,7 +19,11 @@
     white-space: pre-wrap;
 }
 .op-warn {
-    border:1px solid salmon;
-    padding:2px;
+    border: 1px solid salmon;
+    padding: 2px;
     border-radius: 4px;
-}
+}
+.syncstatusbar {
+    -webkit-filter: grayscale(100%);
+    filter: grayscale(100%);
+}