Fixed parameter mistake.

Getting ready for webclip

README.md

@@ -1,21 +1,26 @@
 # obsidian-livesync
 
-This is the obsidian plugin that enables livesync between multi terminals.  
+This is the obsidian plugin that enables livesync between multi-devices.  
 Runs in Mac, Android, Windows, and iOS.
 
 <!-- <div><video controls src="https://user-images.githubusercontent.com/45774780/137352386-a274736d-a38b-4069-ac41-759c73e36a23.mp4"  muted="false"></video></div> -->
 
 ![obsidian_live_sync_demo](https://user-images.githubusercontent.com/45774780/137355323-f57a8b09-abf2-4501-836c-8cb7d2ff24a3.gif)
 
-**It's beta. Please make sure back your vault up!**
+**It's beta. Please make sure to back your vault up!**
 
-Limitations: File deletion handling is not completed.
+Limitations: Folder deletion handling is not completed.
 
 ## This plugin enables..
 
--   Live sync
+-   Live Sync
 -   Self-Hosted data synchronization with conflict detection and resolving in Obsidian.
--   Off line sync is also available.
+-   Off-line sync is also available.
+
+## IMPORTANT NOTICE
+
+**Please make sure to disable other synchronize solutions to avoid content corruption or duplication.**
+If you want to synchronize to both backend, sync one by one, please.
 
 ## How to use
 
@@ -31,14 +36,14 @@ Limitations: File deletion handling is not completed.
 
 obsidian-livesync changes data treatment of markdown files since 0.1.0  
 When you are troubled with synchronization, **Please reset local and remote databases**.  
-*Note: Without synchronization, your files won't be deleted.*
+_Note: Without synchronization, your files won't be deleted._
 
 1.  Disable any synchronizations on all devices.
 2.  From the most reliable device<sup>(_The device_)</sup>, back your vault up.
 3.  Click "Reset local database" on all devices.
 4.  From _The device_ click "Reset remote database".
 5.  From _The device_ click "Init Database again".
-6.  Enable any sync or Hit Replication button.
+6.  Enable any sync or Hit the Replication button.
 
 And wait for a minute. your data will be uploaded and synchronized with all devices again.
 
@@ -52,7 +57,7 @@ And wait for a minute. your data will be uploaded and synchronized with all devi
 1. In IBM Cloud Catalog, search "Cloudant".
    ![step 2](instruction_images/cloudant_2.png)
 
-1. You can choise "Lite plan" in free.
+1. You can choose "Lite plan" for free.
    ![step 3](instruction_images/cloudant_3.png)
 
 Select Multitenant(it's the default) and the region as you like.

main.ts

@@ -11,7 +11,7 @@ const MAX_DOC_SIZE_BIN = 102400; // 100kb
 const VER = 10;
 
 const RECENT_MOFIDIED_DOCS_QTY = 30;
-
+const LEAF_WAIT_TIMEOUT = 30000; // in synchronization, waiting missing leaf time out.
 const LOG_LEVEL = {
     VERBOSE: 1,
     INFO: 10,
@@ -34,6 +34,7 @@ interface ObsidianLiveSyncSettings {
     minimumChunkSize: number;
     longLineThreshold: number;
     showVerboseLog: boolean;
+    suspendFileWatching: boolean;
 }
 
 const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
@@ -45,11 +46,12 @@ const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
     syncOnStart: false,
     savingDelay: 200,
     lessInformationInLog: false,
-    gcDelay: 30,
+    gcDelay: 300,
     versionUpFlash: "",
     minimumChunkSize: 20,
     longLineThreshold: 250,
     showVerboseLog: false,
+    suspendFileWatching: false,
 };
 interface Entry {
     _id: string;
@@ -99,7 +101,9 @@ interface EntryLeaf {
     _rev?: string;
 }
 
-type EntryDoc = Entry | NewEntry | PlainEntry | LoadedEntry | EntryLeaf;
+type EntryBody = Entry | NewEntry | PlainEntry;
+type EntryDoc = EntryBody | LoadedEntry | EntryLeaf;
+
 type diff_result_leaf = {
     rev: string;
     data: string;
@@ -143,7 +147,11 @@ function base64ToArrayBuffer(base64: string): ArrayBuffer {
         }
         return bytes.buffer;
     } catch (ex) {
-        return null;
+        return new Uint16Array(
+            [].map.call(base64, function (c: string) {
+                return c.charCodeAt(0);
+            })
+        ).buffer;
     }
 }
 function base64ToString(base64: string): string {
@@ -156,7 +164,7 @@ function base64ToString(base64: string): string {
         }
         return new TextDecoder().decode(bytes);
     } catch (ex) {
-        return null;
+        return base64;
     }
 }
 
@@ -188,7 +196,7 @@ const connectRemoteCouchDB = async (uri: string, auth: { username: string; passw
         let info = await db.info();
         return { db: db, info: info };
     } catch (ex) {
-        return;
+        return false;
     }
 };
 
@@ -212,6 +220,8 @@ class LocalPouchDB {
         [key: string]: string;
     } = {};
 
+    corruptedEntries: { [key: string]: EntryDoc } = {};
+
     constructor(app: App, plugin: ObsidianLiveSyncPlugin, dbname: string) {
         this.plugin = plugin;
         this.app = app;
@@ -251,23 +261,112 @@ class LocalPouchDB {
         let idrev = id + rev;
         return this.recentModifiedDocs.indexOf(idrev) !== -1;
     }
+    changeHandler: PouchDB.Core.Changes<{}> = null;
     async initializeDatabase() {
         if (this.localDatabase != null) this.localDatabase.close();
+        if (this.changeHandler != null) {
+            this.changeHandler.cancel();
+        }
         this.localDatabase = null;
         this.localDatabase = new PouchDB<EntryDoc>(this.dbname + "-livesync", {
             auto_compaction: true,
             revs_limit: 100,
             deterministic_revs: true,
         });
+
+        // Traceing the leaf id
+        let changes = this.localDatabase
+            .changes({
+                since: "now",
+                live: true,
+                filter: (doc) => doc.type == "leaf",
+            })
+            .on("change", (e) => {
+                if (e.deleted) return;
+                this.leafArrived(e.id);
+            });
+        this.changeHandler = changes;
         await this.prepareHashFunctions();
     }
+
     async prepareHashFunctions() {
         if (this.h32 != null) return;
         const { h32, h64 } = await xxhash();
         this.h32 = h32;
         this.h64 = h64;
     }
-    async getDBEntry(id: string, opt?: PouchDB.Core.GetOptions): Promise<false | LoadedEntry> {
+
+    // leaf waiting
+    leafArrivedCallbacks: { [key: string]: (() => void)[] } = {};
+
+    leafArrived(id: string) {
+        if (typeof this.leafArrivedCallbacks[id] !== "undefined") {
+            for (let func of this.leafArrivedCallbacks[id]) {
+                func();
+            }
+            delete this.leafArrivedCallbacks[id];
+        }
+    }
+    // wait
+    waitForLeafReady(id: string): Promise<boolean> {
+        return new Promise((res) => {
+            // Set timeout.
+            let timer = setTimeout(() => res(false), LEAF_WAIT_TIMEOUT);
+            if (typeof this.leafArrivedCallbacks[id] == "undefined") {
+                this.leafArrivedCallbacks[id] = [];
+            }
+            this.leafArrivedCallbacks[id].push(() => {
+                clearTimeout(timer);
+                res(true);
+            });
+        });
+    }
+
+    async getDBLeaf(id: string): Promise<string> {
+        // when in cache, use that.
+        if (this.hashCacheRev[id]) {
+            return this.hashCacheRev[id];
+        }
+        try {
+            let w = await this.localDatabase.get(id);
+            if (w.type == "leaf") {
+                this.hashCache[w.data] = id;
+                this.hashCacheRev[id] = w.data;
+                return w.data;
+            }
+            throw new Error(`retrive leaf, but it was not leaf.`);
+        } catch (ex) {
+            if (ex.status && ex.status == 404) {
+                // just leaf is not ready.
+                // wait for on
+                if ((await this.waitForLeafReady(id)) === false) {
+                    throw new Error(`time out (waiting leaf)`);
+                }
+                try {
+                    // retrive again.
+                    let w = await this.localDatabase.get(id);
+
+                    if (w.type == "leaf") {
+                        this.hashCache[w.data] = id;
+                        this.hashCacheRev[id] = w.data;
+                        return w.data;
+                    }
+                    throw new Error(`retrive leaf, but it was not leaf.`);
+                } catch (ex) {
+                    if (ex.status && ex.status == 404) {
+                        throw new Error("leaf is not found");
+                    }
+                    this.addLog(`Something went wrong on retriving leaf`);
+                    throw ex;
+                }
+            } else {
+                this.addLog(`Something went wrong on retriving leaf`);
+                throw ex;
+            }
+        }
+    }
+
+    async getDBEntry(id: string, opt?: PouchDB.Core.GetOptions, retryCount = 5): Promise<false | LoadedEntry> {
         try {
             let obj: EntryDocResponse = null;
             if (opt) {
@@ -296,33 +395,24 @@ class LocalPouchDB {
                     children: [],
                     datatype: "newnote",
                 };
+                if (typeof this.corruptedEntries[doc._id] != "undefined") {
+                    delete this.corruptedEntries[doc._id];
+                }
                 return doc;
                 // simple note
             }
             if (obj.type == "newnote" || obj.type == "plain") {
                 // search childrens
                 try {
-                    let childrens = [];
+                    let childrens;
-                    for (var v of obj.children) {
+                    try {
-                        if (typeof this.hashCacheRev[v] !== "undefined") {
+                        childrens = await Promise.all(obj.children.map((e) => this.getDBLeaf(e)));
-                            childrens.push(this.hashCacheRev[v]);
+                    } catch (ex) {
-                        } else {
+                        this.addLog(`Something went wrong on reading elements of ${obj._id} from database.`, LOG_LEVEL.NOTICE);
-                            try {
+                        this.corruptedEntries[obj._id] = obj;
-                                let elem = await this.localDatabase.get(v);
+                        return false;
-                                if (elem.type && elem.type == "leaf") {
-                                    childrens.push(elem.data);
-                                } else {
-                                    throw new Error("linked document is not leaf");
-                                }
-                            } catch (ex) {
-                                if (ex.status && ex.status == 404) {
-                                    this.addLog(`Missing document content!, could not read ${v} of ${obj._id}(${obj._rev}) from database.`, LOG_LEVEL.NOTICE);
-                                    return false;
-                                }
-                                throw ex;
-                            }
-                        }
                     }
+
                     let data = childrens.join("");
                     let doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = {
                         data: data,
@@ -336,7 +426,9 @@ class LocalPouchDB {
                         datatype: obj.type,
                         _conflicts: obj._conflicts,
                     };
-
+                    if (typeof this.corruptedEntries[doc._id] != "undefined") {
+                        delete this.corruptedEntries[doc._id];
+                    }
                     return doc;
                 } catch (ex) {
                     if (ex.status && ex.status == 404) {
@@ -373,6 +465,9 @@ class LocalPouchDB {
                 obj._deleted = true;
                 let r = await this.localDatabase.put(obj);
                 this.updateRecentModifiedDocs(r.id, r.rev, true);
+                if (typeof this.corruptedEntries[obj._id] != "undefined") {
+                    delete this.corruptedEntries[obj._id];
+                }
                 return true;
                 // simple note
             }
@@ -381,6 +476,9 @@ class LocalPouchDB {
                 let r = await this.localDatabase.put(obj);
                 this.addLog(`entry removed:${obj._id}-${r.rev}`);
                 this.updateRecentModifiedDocs(r.id, r.rev, true);
+                if (typeof this.corruptedEntries[obj._id] != "undefined") {
+                    delete this.corruptedEntries[obj._id];
+                }
                 return true;
             }
         } catch (ex) {
@@ -528,7 +626,6 @@ class LocalPouchDB {
             size: note.size,
             type: plainSplit ? "plain" : "newnote",
         };
-
         // Here for upsert logic,
         try {
             let old = await this.localDatabase.get(newDoc._id);
@@ -545,6 +642,9 @@ class LocalPouchDB {
         }
         let r = await this.localDatabase.put(newDoc);
         this.updateRecentModifiedDocs(r.id, r.rev, newDoc._deleted);
+        if (typeof this.corruptedEntries[note._id] != "undefined") {
+            delete this.corruptedEntries[note._id];
+        }
         this.addLog(`note saven:${newDoc._id}:${r.rev}`);
     }
 
@@ -576,11 +676,13 @@ class LocalPouchDB {
         let syncOption: PouchDB.Replication.SyncOptions = keepAlive ? { live: true, retry: true, heartbeat: 30000, ...syncOptionBase } : { ...syncOptionBase };
 
         let db = dbret.db;
-
         //replicate once
         let replicate = this.localDatabase.replicate.from(db, syncOptionBase);
         replicate
             .on("change", async (e) => {
+                // when in first run, replication will send us tombstone data
+                // and in normal cases, all leavs should sent before the entry that contains these item.
+                // so skip to completed all, we should treat all changes.
                 try {
                     callback(e.docs);
                     this.addLog(`pulled ${e.docs.length} doc(s)`);
@@ -590,10 +692,11 @@ class LocalPouchDB {
                 }
             })
             .on("complete", async (info) => {
-                replicate.removeAllListeners();
                 replicate.cancel();
+                replicate.removeAllListeners();
                 this.syncHandler = null;
                 if (this.syncHandler != null) {
+                    this.syncHandler.cancel();
                     this.syncHandler.removeAllListeners();
                 }
                 this.syncHandler = this.localDatabase.sync(db, syncOption);
@@ -644,10 +747,14 @@ class LocalPouchDB {
     }
 
     async resetDatabase() {
+        if (this.changeHandler != null) {
+            this.changeHandler.cancel();
+        }
         await this.closeReplication();
         await this.localDatabase.destroy();
         this.localDatabase = null;
         await this.initializeDatabase();
+        this.disposeHashCache();
         this.addLog("Local Database Reset", LOG_LEVEL.NOTICE);
     }
     async tryResetRemoteDatabase(setting: ObsidianLiveSyncSettings) {
@@ -741,7 +848,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     //localDatabase: PouchDB.Database<EntryDoc>;
     localDatabase: LocalPouchDB;
     logMessage: string[] = [];
-    // onLogChanged: () => void;
     statusBar: HTMLElement;
     statusBar2: HTMLElement;
 
@@ -827,6 +933,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 this.replicate();
             },
         });
+        // this.addCommand({
+        //     id: "livesync-test",
+        //     name: "test reset db and replicate",
+        //     callback: async () => {
+        //         await this.resetLocalDatabase();
+        //         await this.replicate();
+        //     },
+        // });
         this.addCommand({
             id: "livesync-gc",
             name: "garbage collect now",
@@ -904,6 +1018,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     watchWindowVisiblity() {
+        if (this.settings.suspendFileWatching) return;
         let isHidden = document.hidden;
         if (isHidden) {
             this.localDatabase.closeReplication();
@@ -919,16 +1034,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     watchWorkspaceOpen(file: TFile) {
+        if (this.settings.suspendFileWatching) return;
         if (file == null) return;
         this.localDatabase.disposeHashCache();
         this.showIfConflicted(file);
         this.gcHook();
     }
     watchVaultChange(file: TFile, ...args: any[]) {
+        if (this.settings.suspendFileWatching) return;
         this.updateIntoDB(file);
         this.gcHook();
     }
     watchVaultDelete(file: TFile & TFolder) {
+        if (this.settings.suspendFileWatching) return;
         if (file.children) {
             //folder
             this.deleteFolderOnDB(file);
@@ -939,6 +1057,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         this.gcHook();
     }
     watchVaultRename(file: TFile & TFolder, oldFile: any) {
+        if (this.settings.suspendFileWatching) return;
         if (file.children) {
             // this.renameFolder(file,oldFile);
             this.addLog(`folder name changed:(this operation is not supported) ${file.path}`);
@@ -995,7 +1114,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
     }
 
-    async doc2storage_create(docEntry: Entry, force?: boolean) {
+    async doc2storage_create(docEntry: EntryBody, force?: boolean) {
         let doc = await this.localDatabase.getDBEntry(docEntry._id, { rev: docEntry._rev });
         if (doc === false) return;
         if (doc.datatype == "newnote") {
@@ -1026,7 +1145,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             await this.deleteVaultItem(dir);
         }
     }
-    async doc2storate_modify(docEntry: Entry, file: TFile, force?: boolean) {
+    async doc2storate_modify(docEntry: EntryBody, file: TFile, force?: boolean) {
         if (docEntry._deleted) {
             //basically pass.
             //but if there're no docs left, delete file.
@@ -1072,7 +1191,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             //eq.case
         }
     }
-    async handleDBChanged(change: Entry) {
+    async handleDBChanged(change: EntryBody) {
         let allfiles = this.app.vault.getFiles();
         let targetFiles = allfiles.filter((e) => e.path == change._id);
         if (targetFiles.length == 0) {
@@ -1091,13 +1210,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     //---> Sync
-    async parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<Entry>>): Promise<void> {
+    async parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): Promise<void> {
         for (var change of docs) {
             if (this.localDatabase.isSelfModified(change._id, change._rev)) {
                 return;
             }
             this.addLog("replication change arrived", LOG_LEVEL.VERBOSE);
-            await this.handleDBChanged(change);
+            if (change.type != "leaf") {
+                await this.handleDBChanged(change);
+            }
             this.gcHook();
         }
     }
@@ -1588,6 +1709,8 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 })
         );
 
+        containerEl.createEl("h3", { text: "Database configuration" });
+
         new Setting(containerEl)
             .setName("File to Database saving delay")
             .setDesc("ms, between 200 and 5000, restart required.")
@@ -1615,15 +1738,17 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                         let v = Number(value);
                         if (isNaN(v) || v > 5000) {
                             return 0;
-                            //text.inputEl.va;
                         }
                         this.plugin.settings.gcDelay = v;
                         await this.plugin.saveSettings();
                     });
                 text.inputEl.setAttribute("type", "number");
             });
+
+        containerEl.createEl("h3", { text: "Log Setting" });
+
         new Setting(containerEl)
-            .setName("Log")
+            .setName("Do not show low-priority Log")
             .setDesc("Reduce log infomations")
             .addToggle((toggle) =>
                 toggle.setValue(this.plugin.settings.lessInformationInLog).onChange(async (value) => {
@@ -1640,18 +1765,21 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     await this.plugin.saveSettings();
                 })
             );
+
+        containerEl.createEl("h3", { text: "Sync setting" });
+
         if (this.plugin.settings.versionUpFlash != "") {
             let c = containerEl.createEl("div", { text: this.plugin.settings.versionUpFlash });
             c.createEl("button", { text: "I got it and updated." }, (e) => {
                 e.addEventListener("click", async () => {
                     this.plugin.settings.versionUpFlash = "";
-                    this.plugin.saveSettings();
+                    await this.plugin.saveSettings();
                     c.remove();
                 });
             });
             c.addClass("op-warn");
         }
-        // containerEl.createDiv(this.plugin.settings.versionUpFlash);
+
         new Setting(containerEl)
             .setName("LiveSync")
             .setDesc("Sync realtime")
@@ -1664,7 +1792,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             );
         new Setting(containerEl)
             .setName("Sync on Save")
-            .setDesc("Sync on Save")
+            .setDesc("When you save file, sync automatically")
             .addToggle((toggle) =>
                 toggle.setValue(this.plugin.settings.syncOnSave).onChange(async (value) => {
                     this.plugin.settings.syncOnSave = value;
@@ -1673,13 +1801,14 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             );
         new Setting(containerEl)
             .setName("Sync on Start")
-            .setDesc("Sync on Start")
+            .setDesc("Start synchronization on Obsidian started.")
             .addToggle((toggle) =>
                 toggle.setValue(this.plugin.settings.syncOnStart).onChange(async (value) => {
                     this.plugin.settings.syncOnStart = value;
                     await this.plugin.saveSettings();
                 })
             );
+
         new Setting(containerEl)
             .setName("Minimum chunk size")
             .setDesc("(letters), minimum chunk size.")
@@ -1696,6 +1825,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     });
                 text.inputEl.setAttribute("type", "number");
             });
+
         new Setting(containerEl)
             .setName("LongLine Threshold")
             .setDesc("(letters), If the line is longer than this, make the line to chunk")
@@ -1712,6 +1842,7 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     });
                 text.inputEl.setAttribute("type", "number");
             });
+
         new Setting(containerEl).setName("Local Database Operations").addButton((button) =>
             button
                 .setButtonText("Reset local database")
@@ -1738,6 +1869,19 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     await this.plugin.garbageCollect();
                 })
         );
+
+        containerEl.createEl("h3", { text: "Hatch" });
+
+        new Setting(containerEl)
+            .setName("Suspend file watching")
+            .setDesc("if enables it, all file operations are ignored.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.suspendFileWatching).onChange(async (value) => {
+                    this.plugin.settings.suspendFileWatching = value;
+                    await this.plugin.saveSettings();
+                })
+            );
+
         new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
             button
                 .setButtonText("Reset remote database")
@@ -1754,5 +1898,21 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     await this.plugin.tryResetRemoteDatabase();
                 })
         );
+
+        containerEl.createEl("h3", { text: "Corrupted data" });
+
+        if (Object.keys(this.plugin.localDatabase.corruptedEntries).length > 0) {
+            let cx = containerEl.createEl("div", { text: "If you have copy of these items on any device, simply edit once or twice. Or not, delete this. sorry.." });
+            for (let k in this.plugin.localDatabase.corruptedEntries) {
+                let xx = cx.createEl("div", { text: `${k}` });
+
+                let ba = xx.createEl("button", { text: `Delete this` }, (e) => {
+                    e.addEventListener("click", async () => {
+                        await this.plugin.localDatabase.deleteDBEntry(k);
+                        xx.remove();
+                    });
+                });
+            }
+        }
     }
 }

manifest.json

@@ -1,9 +1,9 @@
 {
     "id": "obsidian-livesync",
     "name": "Obsidian Live sync",
-    "version": "0.1.1",
+    "version": "0.1.4",
     "minAppVersion": "0.9.12",
-    "description": "obsidian Live synchronization plugin.",
+    "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",
     "authorUrl": "https://github.com/vrtmrz",
     "isDesktopOnly": false

package.json

@@ -1,7 +1,7 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.1.1",
+    "version": "0.1.4",
-    "description": "obsidian Live synchronization plugin.",
+    "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "scripts": {
         "dev": "rollup --config rollup.config.js -w",