chunk splitting method changed

less change sensitivity

.gitignore

@@ -7,8 +7,8 @@ node_modules
 package-lock.json
 
 # build
-# main.js
+main.js
 *.js.map
 
 # obsidian
-data.json
+data.json

README.md

@@ -16,6 +16,7 @@ Limitations: File deletion handling is not completed.
 
 -   Live sync
 -   Self-Hosted data synchronization with conflict detection and resolving in Obsidian.
+-   Off line sync is also available.
 
 ## How to use the beta build
 
@@ -106,4 +107,4 @@ example values.
 | CouchDB Password    | (\*4)       | c2c11651d75497fa3d3c486e4c8bdf27                                            |
 
 # License
-The source code is licensed MIT.
+The source code is licensed MIT.

main.ts

@@ -1,11 +1,14 @@
-import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder } from "obsidian";
+import { App, debounce, Modal, Notice, Plugin, PluginSettingTab, Setting, TFile, addIcon, TFolder, ItemView } from "obsidian";
 import { PouchDB } from "./pouchdb-browser-webpack/dist/pouchdb-browser";
 import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
+import xxhash from "xxhash-wasm";
 
 // docs should be encoded as base64, so 1 char -> 1 bytes
 // and cloudant limitation is 1MB , we use 900kb;
 // const MAX_DOC_SIZE = 921600;
-const MAX_DOC_SIZE = 921600;
+const MAX_DOC_SIZE = 1000; // for .md file, but if delimiters exists. use that before.
+const MAX_DOC_SIZE_BIN = 102400; // 100kb
+const VER = 10
 
 interface ObsidianLiveSyncSettings {
     couchDB_URI: string;
@@ -16,6 +19,10 @@ interface ObsidianLiveSyncSettings {
     syncOnStart: boolean;
     savingDelay: number;
     lessInformationInLog: boolean;
+    gcDelay: number;
+    versionUpFlash: string;
+    minimumChunkSize: number;
+    longLineThreshold: number
 }
 
 const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
@@ -27,6 +34,10 @@ const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
     syncOnStart: false,
     savingDelay: 200,
     lessInformationInLog: false,
+    gcDelay: 30,
+    versionUpFlash: "",
+    minimumChunkSize: 20,
+    longLineThreshold: 250,
 };
 
 interface Entry {
@@ -50,20 +61,31 @@ interface NewEntry {
     NewNote: true;
     type: "newnote";
 }
+interface PlainEntry {
+    _id: string;
+    children: string[];
+    _rev?: string;
+    ctime: number;
+    mtime: number;
+    size: number;
+    _deleted?: boolean;
+    NewNote: true;
+    type: "plain";
+}
 type LoadedEntry = Entry & {
     children: string[];
+    datatype: "plain" | "newnote"
 };
 
 interface EntryLeaf {
     _id: string;
-    parent: string;
-    seq: number;
     data: string;
-    _rev?: string;
     _deleted?: boolean;
     type: "leaf";
+    _rev?: string;
 }
-type EntryDoc = Entry | NewEntry | LoadedEntry | EntryLeaf;
+
+type EntryDoc = Entry | NewEntry | PlainEntry | LoadedEntry | EntryLeaf;
 type diff_result_leaf = {
     rev: string;
     data: string;
@@ -164,6 +186,9 @@ class LocalPouchDB {
     addLog: (message: any, isNotify?: boolean) => Promise<void>;
     localDatabase: PouchDB.Database<EntryDoc>;
 
+    recentModifiedDocs: string[] = [];
+    h32: (input: string, seed?: number) => string;
+    h64: (input: string, seedHigh?: number, seedLow?: number) => string;
     constructor(app: App, plugin: ObsidianLiveSyncPlugin, dbname: string) {
         this.plugin = plugin;
         this.app = app;
@@ -185,7 +210,18 @@ class LocalPouchDB {
         }
         return "disabled";
     }
-    initializeDatabase() {
+    updateRecentModifiedDocs(id: string, rev: string) {
+        let idrev = id + rev;
+        this.recentModifiedDocs.push(idrev);
+        if (this.recentModifiedDocs.length > 10) {
+            this.recentModifiedDocs = this.recentModifiedDocs.slice(-30);
+        }
+    }
+    isSelfModified(id: string, rev: string): boolean {
+        let idrev = id + rev;
+        return this.recentModifiedDocs.indexOf(idrev) !== -1;
+    }
+    async initializeDatabase() {
         if (this.localDatabase != null) this.localDatabase.close();
         this.localDatabase = null;
         this.localDatabase = new PouchDB<EntryDoc>(this.dbname + "-livesync", {
@@ -193,8 +229,14 @@ class LocalPouchDB {
             revs_limit: 100,
             deterministic_revs: true,
         });
+        await this.prepareHashArg();
+    }
+    async prepareHashArg() {
+        if (this.h32 != null) return;
+        const { h32, h64 } = await xxhash();
+        this.h32 = h32;
+        this.h64 = h64;
     }
-
     async getDatabaseDoc(id: string, opt?: any): Promise<false | LoadedEntry> {
         try {
             let obj: EntryDoc & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = null;
@@ -221,31 +263,32 @@ class LocalPouchDB {
                     _deleted: obj._deleted,
                     _rev: obj._rev,
                     children: [],
+                    datatype: "newnote",
                 };
                 return doc;
                 // simple note
             }
-            if (obj.type == "newnote") {
+            if (obj.type == "newnote" || obj.type == "plain") {
                 // search childrens
                 try {
                     let childrens = [];
-                    // let childPromise = [];
                     for (var v of obj.children) {
-                        // childPromise.push(this.localDatabase.get(v));
-                        let elem = await this.localDatabase.get(v);
-                        if (elem.type && elem.type == "leaf") {
-                            childrens.push(elem);
-                        } else {
-                            throw new Error("linked document is not leaf");
+                        try {
+                            let elem = await this.localDatabase.get(v);
+                            if (elem.type && elem.type == "leaf") {
+                                childrens.push(elem.data);
+                            } else {
+                                throw new Error("linked document is not leaf");
+                            }
+                        } catch (ex) {
+                            if (ex.status && ex.status == 404) {
+                                this.addLog(`Missing document content!, could not read ${v} of ${obj._id} from database.`, true);
+                                return false;
+                            }
+                            throw ex;
                         }
                     }
-                    // let childrens = await Promise.all(childPromise);
-                    let data = childrens
-                        // .filter((e) => e.type == "leaf")
-                        // .map((e) => e as NoteLeaf)
-                        .sort((e) => e.seq)
-                        .map((e) => e.data)
-                        .join("");
+                    let data = childrens.join("");
                     let doc: LoadedEntry = {
                         data: data,
                         _id: obj._id,
@@ -255,12 +298,14 @@ class LocalPouchDB {
                         _deleted: obj._deleted,
                         _rev: obj._rev,
                         children: obj.children,
+                        datatype: obj.type
                     };
+
                     return doc;
                 } catch (ex) {
                     if (ex.status && ex.status == 404) {
                         this.addLog(`Missing document content!, could not read ${obj._id} from database.`, true);
-                        // this.addLog(ex);
+                        return false;
                     }
                     this.addLog(`Something went wrong on reading ${obj._id} from database.`, true);
                     this.addLog(ex);
@@ -289,27 +334,17 @@ class LocalPouchDB {
             }
             //Check it out and fix docs to regular case
             if (!obj.type || (obj.type && obj.type == "notes")) {
-                // let note = obj as Notes;
-                // note._deleted=true;
                 obj._deleted = true;
                 let r = await this.localDatabase.put(obj);
+                this.updateRecentModifiedDocs(r.id, r.rev);
                 return true;
                 // simple note
             }
-            if (obj.type == "newnote") {
-                // search childrens
-                for (var v of obj.children) {
-                    let d = await this.localDatabase.get(v);
-                    if (d.type != "leaf") {
-                        this.addLog(`structure went wrong:${id}-${v}`);
-                    }
-                    d._deleted = true;
-                    await this.localDatabase.put(d);
-                    this.addLog(`content removed:${(d as EntryLeaf).seq}`);
-                }
+            if (obj.type == "newnote" || obj.type == "plain") {
                 obj._deleted = true;
-                await this.localDatabase.put(obj);
+                let r = await this.localDatabase.put(obj);
                 this.addLog(`entry removed:${obj._id}`);
+                this.updateRecentModifiedDocs(r.id, r.rev);
                 return true;
             }
         } catch (ex) {
@@ -322,48 +357,144 @@ class LocalPouchDB {
 
     async putDBEntry(note: LoadedEntry) {
         let leftData = note.data;
-        let savenNotes = []; // something occured, kill this .
-        let seq = 0;
-        let now = Date.now();
+        let savenNotes = [];
+        let processed = 0;
+        let made = 0;
+        let skiped = 0;
+        let pieceSize = MAX_DOC_SIZE_BIN;
+        let plainSplit = false;
+        if (note._id.endsWith(".md")) {
+            pieceSize = MAX_DOC_SIZE;
+            plainSplit = true;
+        }
         do {
-            let piece = leftData.substring(0, MAX_DOC_SIZE);
-            leftData = leftData.substring(MAX_DOC_SIZE);
-            seq++;
-            let leafid = note._id + "-" + now + "-" + seq;
-            let d: EntryLeaf = {
-                _id: leafid,
-                parent: note._id,
-                data: piece,
-                seq: seq,
-                type: "leaf",
-            };
-            let result = await this.localDatabase.put(d);
+            // To keep low bandwith and database size,
+            // Dedup pieces on database.
+            // from 0.1.10, for best performance. we use markdown delimiters 
+            // 1. \n[^\n]{longLineThreshold}[^\n]*\n -> long sentence shuld break.
+            // 2. \n\n shold break
+            // 3. \r\n\r\n should break
+            // 4. \n# should break.
+            let cPieceSize = pieceSize
+            let minimumChunkSize = this.plugin.settings.minimumChunkSize;
+            if (minimumChunkSize < 10) minimumChunkSize = 10;
+            let longLineThreshold = this.plugin.settings.longLineThreshold;
+            if (longLineThreshold < 100) longLineThreshold = 100;
+            if (plainSplit) {
+                cPieceSize = 0;
+                // lookup for next splittion .
+                // we're standing on "\n"
+                // debugger
+                do {
+                    let n1 = leftData.indexOf("\n", cPieceSize + 1);
+                    let n2 = leftData.indexOf("\n\n", cPieceSize + 1);
+                    let n3 = leftData.indexOf("\r\n\r\n", cPieceSize + 1);
+                    let n4 = leftData.indexOf("\n#", cPieceSize + 1);
+                    if (n1 == -1 && n2 == -1 && n3 == -1 && n4 == -1) {
+                        cPieceSize = MAX_DOC_SIZE;
+                        break;
+                    }
+
+                    if (n1 > longLineThreshold) {
+                        // long sentence is an established piece
+                        cPieceSize = n1 + 1;
+                    } else {
+                        // cPieceSize = Math.min.apply([n2, n3, n4].filter((e) => e > 1));
+                        // ^ heavy.
+                        if (n2 > 0 && cPieceSize < n2) cPieceSize = n2 + 1;
+                        if (n3 > 0 && cPieceSize < n3) cPieceSize = n3 + 3;
+                        if (n4 > 0 && cPieceSize < n4) cPieceSize = n4 + 0;
+                        cPieceSize++;
+                    }
+                } while (cPieceSize < minimumChunkSize)
+                // console.log("and we use:" + cPieceSize)
+            }
+
+            let piece = leftData.substring(0, cPieceSize);
+            // if (plainSplit) {
+            //     this.addLog(`piece_len:${cPieceSize}`);
+            //     this.addLog("piece:" + piece);
+            // }
+            leftData = leftData.substring(cPieceSize);
+            processed++;
+            // Get has of piece.
+            let hashedPiece = this.h32(piece);
+            let leafid = "h:" + hashedPiece;
+            let hashQ: number = 0; // if hash collided, **IF**, count it up.
+            let tryNextHash = false;
+            let needMake = true;
+
+            do {
+                let nleafid = leafid;
+                try {
+                    nleafid = `${leafid}${hashQ}`;
+                    // console.log(nleafid);
+                    let pieceData = await this.localDatabase.get<EntryLeaf>(nleafid);
+                    if (pieceData.type == "leaf" && pieceData.data == piece) {
+                        // this.addLog("hash:data exists.");
+                        leafid = nleafid;
+                        needMake = false;
+                        tryNextHash = false;
+                    } else if (pieceData.type == "leaf") {
+                        this.addLog("hash:collision!!");
+                        hashQ++;
+                        tryNextHash = true;
+                    } else {
+                        // this.addLog("hash:no collision, it's not leaf. what's going on..");
+                        leafid = nleafid;
+                        tryNextHash = false;
+                    }
+                } catch (ex) {
+                    if (ex.status && ex.status == 404) {
+                        //not found, we can use it.
+                        // this.addLog(`hash:not found.`);
+                        leafid = nleafid;
+                        needMake = true;
+                    } else {
+                        needMake = false;
+                        throw ex;
+                    }
+                }
+            } while (tryNextHash);
+            if (needMake) {
+                //have to make
+                let d: EntryLeaf = {
+                    _id: leafid,
+                    data: piece,
+                    type: "leaf",
+                };
+                let result = await this.localDatabase.put(d);
+                this.updateRecentModifiedDocs(result.id, result.rev);
+                if (result.ok) {
+                    this.addLog(`ok:saven`);
+                    made++;
+                } else {
+                    this.addLog("save faild");
+                }
+            } else {
+                skiped++;
+            }
             savenNotes.push(leafid);
         } while (leftData != "");
-        this.addLog(`note content saven, pieces:${seq}`);
-        let newDoc: NewEntry = {
+        this.addLog(`note content saven, pieces:${processed} new:${made}, skip:${skiped}`);
+        let newDoc: PlainEntry | NewEntry = {
             NewNote: true,
             children: savenNotes,
             _id: note._id,
             ctime: note.ctime,
             mtime: note.mtime,
             size: note.size,
-            type: "newnote",
+            type: plainSplit ? "plain" : "newnote",
         };
 
         let deldocs: string[] = [];
         // Here for upsert logic,
         try {
             let old = await this.localDatabase.get(newDoc._id);
-            if (!old.type || old.type == "notes") {
+            if (!old.type || old.type == "notes" || old.type == "newnote" || old.type == "plain") {
                 // simple use rev for new doc
                 newDoc._rev = old._rev;
             }
-            if (old.type == "newnote") {
-                //when save finished, we have to garbage collect.
-                deldocs = old.children;
-                newDoc._rev = old._rev;
-            }
         } catch (ex) {
             if (ex.status && ex.status == 404) {
                 // NO OP/
@@ -371,29 +502,25 @@ class LocalPouchDB {
                 throw ex;
             }
         }
-        await this.localDatabase.put(newDoc);
+        let r = await this.localDatabase.put(newDoc);
+        this.updateRecentModifiedDocs(r.id, r.rev);
         this.addLog(`note saven:${newDoc._id}`);
-        let items = 0;
-        for (var v of deldocs) {
-            items++;
-            //TODO: Check for missing link
-            let d = await this.localDatabase.get(v);
-            d._deleted = true;
-            await this.localDatabase.put(d);
-        }
-        this.addLog(`old content deleted, pieces:${items}`);
     }
 
     syncHandler: PouchDB.Replication.Sync<{}> = null;
 
     async openReplication(setting: ObsidianLiveSyncSettings, keepAlive: boolean, showResult: boolean, callback: (e: PouchDB.Core.ExistingDocument<{}>[]) => Promise<void>) {
+        if (setting.versionUpFlash != "") {
+            new Notice("Open settings and check message, please.");
+            return;
+        }
         let uri = setting.couchDB_URI;
         let auth: Credential = {
             username: setting.couchDB_USER,
             password: setting.couchDB_PASSWORD,
         };
         if (this.syncHandler != null) {
-            this.addLog("Another replication running.", true);
+            this.addLog("Another replication running.");
             return false;
         }
         let dbret = await connectRemoteCouchDB(uri, auth);
@@ -406,6 +533,7 @@ class LocalPouchDB {
 
         //replicate once
         let replicate = this.localDatabase.replicate.from(db);
+        // console.log("replication start.")
         replicate
             .on("change", async (e) => {
                 try {
@@ -420,8 +548,14 @@ class LocalPouchDB {
                 replicate.removeAllListeners();
                 replicate.cancel();
                 // this.syncHandler = null;
+                if (this.syncHandler != null) {
+                    this.syncHandler.removeAllListeners();
+                }
                 this.syncHandler = this.localDatabase.sync(db, syncOption);
                 this.syncHandler
+                    .on("active", () => {
+                        this.addLog("Replication activated");
+                    })
                     .on("change", async (e) => {
                         try {
                             callback(e.change.docs);
@@ -431,13 +565,8 @@ class LocalPouchDB {
                             this.addLog(ex);
                         }
                     })
-                    .on("active", () => {
-                        this.addLog("Replication activated");
-                    })
                     .on("complete", (e) => {
                         this.addLog("Replication completed", showResult);
-                        // this.addLog(e);
-                        console.dir(this.syncHandler);
                         this.syncHandler = null;
                     })
                     .on("denied", (e) => {
@@ -506,6 +635,53 @@ class LocalPouchDB {
         if (con2 === false) return;
         this.addLog("Remote Database Created or Connected", true);
     }
+
+    async garbageCollect() {
+        // get all documents of NewEntry2
+        // we don't use queries , just use allDocs();
+        let c = 0;
+        let readCount = 0;
+        let hashPieces: string[] = [];
+        let usedPieces: string[] = [];
+        do {
+            let result = await this.localDatabase.allDocs({ include_docs: true, skip: c, limit: 100 });
+            readCount = result.rows.length;
+            if (readCount > 0) {
+                //there are some result
+                for (let v of result.rows) {
+                    let doc = v.doc;
+                    if (doc.type == "newnote" || doc.type == "plain") {
+                        // used pieces memo.
+                        usedPieces = Array.from(new Set([...usedPieces, ...doc.children]));
+                    }
+                    if (doc.type == "leaf") {
+                        // all pieces.
+                        hashPieces = Array.from(new Set([...hashPieces, doc._id]));
+                    }
+                    // this.addLog(`GC:processed:${v.doc._id}`);
+                }
+            }
+            c += readCount;
+        } while (readCount != 0);
+        // items collected.
+        const garbages = hashPieces.filter((e) => usedPieces.indexOf(e) == -1);
+        let deleteCount = 0;
+        for (let v of garbages) {
+            try {
+                let item = await this.localDatabase.get(v);
+                item._deleted = true;
+                await this.localDatabase.put(item);
+                deleteCount++;
+            } catch (ex) {
+                if (ex.status && ex.status == 404) {
+                    // NO OP. It should be timing problem.
+                } else {
+                    throw ex;
+                }
+            }
+        }
+        this.addLog(`GC:deleted ${deleteCount} items.`);
+    }
 }
 
 export default class ObsidianLiveSyncPlugin extends Plugin {
@@ -520,8 +696,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     async onload() {
         this.addLog = this.addLog.bind(this);
         this.addLog("loading plugin");
-        await this.openDatabase();
+        const lsname = "obsidian-live-sync-ver" + this.app.vault.getName();
+        const last_version = localStorage.getItem(lsname);
         await this.loadSettings();
+        if (!last_version || Number(last_version) < VER) {
+            this.settings.liveSync = false;
+            this.settings.syncOnSave = false;
+            this.settings.syncOnStart = false;
+            this.settings.versionUpFlash = "I changed specifications incompatiblly, so when you enable sync again, be sure to made version up all nother devides.";
+            this.saveSettings();
+        }
+        localStorage.setItem(lsname, `${VER}`);
+        await this.openDatabase();
+
         addIcon(
             "replicate",
             `<g transform="matrix(1.15 0 0 1.15 -8.31 -9.52)" fill="currentColor" fill-rule="evenodd">
@@ -561,13 +748,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
         this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this));
 
-        setTimeout(async () => {
+        this.app.workspace.onLayoutReady(async () => {
             await this.initializeDatabase();
             this.realizeSettingSyncMode();
             if (this.settings.syncOnStart) {
                 await this.replicate(false);
             }
-        }, 100);
+        });
 
         // when in mobile, too long suspended , connection won't back if setting retry:true
         this.registerInterval(
@@ -583,12 +770,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         this.watchWindowVisiblity = this.watchWindowVisiblity.bind(this);
         window.addEventListener("visibilitychange", this.watchWindowVisiblity);
     }
-
     onunload() {
+        if (this.gcTimerHandler != null) {
+            clearTimeout(this.gcTimerHandler);
+            this.gcTimerHandler = null;
+        }
         this.localDatabase.closeReplication();
         this.localDatabase.close();
-        this.addLog("unloading plugin");
         window.removeEventListener("visibilitychange", this.watchWindowVisiblity);
+        this.addLog("unloading plugin");
     }
 
     async openDatabase() {
@@ -597,7 +787,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
         let vaultName = this.app.vault.getName();
         this.localDatabase = new LocalPouchDB(this.app, this, vaultName);
-        this.localDatabase.initializeDatabase();
+        await this.localDatabase.initializeDatabase();
+    }
+    async garbageCollect() {
+        await this.localDatabase.garbageCollect();
     }
 
     async loadSettings() {
@@ -607,7 +800,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     async saveSettings() {
         await this.saveData(this.settings);
     }
-
+    gcTimerHandler: any = null;
+    gcHook() {
+        if (this.settings.gcDelay == 0) return;
+        const GC_DELAY = this.settings.gcDelay * 1000; // if leaving opening window, try GC,
+        if (this.gcTimerHandler != null) {
+            clearTimeout(this.gcTimerHandler);
+            this.gcTimerHandler = null;
+        }
+        this.gcTimerHandler = setTimeout(() => {
+            this.gcTimerHandler = null;
+            this.garbageCollect();
+        }, GC_DELAY);
+    }
     registerWatchEvents() {
         this.registerEvent(this.app.vault.on("modify", this.watchVaultChange));
         this.registerEvent(this.app.vault.on("delete", this.watchVaultDelete));
@@ -630,14 +835,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 this.localDatabase.openReplication(this.settings, false, false, this.parseReplicationResult);
             }
         }
+        this.gcHook();
     }
 
     watchWorkspaceOpen(file: TFile) {
         if (file == null) return;
         this.showIfConflicted(file);
+        this.gcHook();
     }
     watchVaultChange(file: TFile, ...args: any[]) {
         this.updateIntoDB(file);
+        this.gcHook();
     }
     watchVaultDelete(file: TFile & TFolder) {
         if (file.children) {
@@ -647,6 +855,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         } else {
             this.deleteFromDB(file);
         }
+        this.gcHook();
     }
     watchVaultRename(file: TFile & TFolder, oldFile: any) {
         if (file.children) {
@@ -656,6 +865,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             this.updateIntoDB(file);
             this.deleteFromDBbyPath(oldFile);
         }
+        this.gcHook();
     }
 
     //--> Basic document Functions
@@ -708,12 +918,21 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     async doc2storage_create(docEntry: Entry, force?: boolean) {
         let doc = await this.localDatabase.getDatabaseDoc(docEntry._id, { _rev: docEntry._rev });
         if (doc === false) return;
-        let bin = base64ToArrayBuffer(doc.data);
-        if (bin != null) {
+        if (doc.datatype == "newnote") {
+            let bin = base64ToArrayBuffer(doc.data);
+            if (bin != null) {
+                await this.ensureDirectory(doc._id);
+                let newfile = await this.app.vault.createBinary(doc._id, bin, { ctime: doc.ctime, mtime: doc.mtime });
+                this.addLog("live : write to local (newfile:b) " + doc._id);
+                await this.app.vault.trigger("create", newfile);
+            }
+        } else if (doc.datatype == "plain") {
             await this.ensureDirectory(doc._id);
-            let newfile = await this.app.vault.createBinary(doc._id, bin, { ctime: doc.ctime, mtime: doc.mtime });
-            this.addLog("live : write to local (newfile) " + doc._id);
+            let newfile = await this.app.vault.create(doc._id, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
+            this.addLog("live : write to local (newfile:p) " + doc._id);
             await this.app.vault.trigger("create", newfile);
+        } else {
+            this.addLog("live : New data imcoming, but we cound't parse that.1" + doc.datatype, true);
         }
     }
 
@@ -743,11 +962,22 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (file.stat.mtime < docEntry.mtime || force) {
             let doc = await this.localDatabase.getDatabaseDoc(docEntry._id);
             if (doc === false) return;
-            let bin = base64ToArrayBuffer(doc.data);
-            if (bin != null) {
-                await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
+            // debugger;
+            if (doc.datatype == "newnote") {
+                let bin = base64ToArrayBuffer(doc.data);
+                if (bin != null) {
+                    await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
+                    this.addLog("livesync : newer local files so write to local:" + file.path);
+                    await this.app.vault.trigger("modify", file);
+                }
+            } if (doc.datatype == "plain") {
+                await this.ensureDirectory(doc._id);
+                await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
                 this.addLog("livesync : newer local files so write to local:" + file.path);
                 await this.app.vault.trigger("modify", file);
+
+            } else {
+                this.addLog("live : New data imcoming, but we cound't parse that.2:" + doc.datatype + "-", true);
             }
         } else if (file.stat.mtime > docEntry.mtime) {
             // newer local file.
@@ -778,8 +1008,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     //---> Sync
     async parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<Entry>>): Promise<void> {
         for (var change of docs) {
+            if (this.localDatabase.isSelfModified(change._id, change._rev)) {
+                return;
+            }
             this.addLog("replication change arrived");
             await this.pouchdbChanged(change);
+            this.gcHook();
         }
     }
     async realizeSettingSyncMode() {
@@ -794,6 +1028,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         this.statusBar.setText("Sync:" + statusStr);
     }
     async replicate(showMessage?: boolean) {
+        if (this.settings.versionUpFlash != "") {
+            new Notice("Open settings and check message, please.");
+            return;
+        }
         this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
     }
     //<-- Sync
@@ -868,7 +1106,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             let entry = v as TFile & TFolder;
             if (entry.children) {
                 await this.deleteFolderOnDB(entry);
-                this.app.vault.delete(entry);
+                await this.app.vault.delete(entry);
             } else {
                 await this.deleteFromDB(entry);
             }
@@ -880,11 +1118,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         try {
             let doc = await this.localDatabase.getDatabaseDoc(path, { rev: rev });
             if (doc === false) return false;
+            let data = doc.data;
+            if (doc.datatype == "newnote") {
+                data = base64ToString(doc.data);
+            } else if (doc.datatype == "plain") {
+                data = doc.data;
+            }
             return {
                 ctime: doc.ctime,
                 mtime: doc.mtime,
                 rev: rev,
-                data: base64ToString(doc.data),
+                data: data,
             };
         } catch (ex) {
             if (ex.status && ex.status == 404) {
@@ -1006,8 +1250,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async updateIntoDB(file: TFile) {
-        let contentBin = await this.app.vault.readBinary(file);
-        let content = arrayBufferToBase64(contentBin);
+        let content = "";
+        let datatype: "plain" | "newnote" = "newnote";
+        if (file.extension != "md") {
+            let contentBin = await this.app.vault.readBinary(file);
+            content = arrayBufferToBase64(contentBin);
+            datatype = "newnote";
+        } else {
+            content = await this.app.vault.read(file);
+            datatype = "plain";
+        }
         let fullpath = file.path;
         let d: LoadedEntry = {
             _id: fullpath,
@@ -1016,6 +1268,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             mtime: file.stat.mtime,
             size: file.stat.size,
             children: [],
+            datatype: datatype
         };
         //From here
         let old = await this.localDatabase.getDatabaseDoc(fullpath);
@@ -1030,7 +1283,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
         let ret = await this.localDatabase.putDBEntry(d);
 
-        this.addLog("put database:" + fullpath);
+        this.addLog("put database:" + fullpath + "(" + datatype + ")");
         if (this.settings.syncOnSave) {
             await this.replicate();
         }
@@ -1242,6 +1495,23 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     });
                 text.inputEl.setAttribute("type", "number");
             });
+        new Setting(containerEl)
+            .setName("Auto GC delay")
+            .setDesc("(seconds), if you set zero, you have to run manually.")
+            .addText((text) => {
+                text.setPlaceholder("")
+                    .setValue(this.plugin.settings.gcDelay + "")
+                    .onChange(async (value) => {
+                        let v = Number(value);
+                        if (isNaN(v) || v < 200 || v > 5000) {
+                            return 30;
+                            //text.inputEl.va;
+                        }
+                        this.plugin.settings.gcDelay = v;
+                        await this.plugin.saveSettings();
+                    });
+                text.inputEl.setAttribute("type", "number");
+            });
         new Setting(containerEl)
             .setName("Log")
             .setDesc("Reduce log infomations")
@@ -1251,6 +1521,18 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     await this.plugin.saveSettings();
                 })
             );
+        if (this.plugin.settings.versionUpFlash != "") {
+            let c = containerEl.createEl("div", { text: this.plugin.settings.versionUpFlash });
+            c.createEl("button", { text: "I got it and updated." }, (e) => {
+                e.addEventListener("click", async () => {
+                    this.plugin.settings.versionUpFlash = "";
+                    this.plugin.saveSettings();
+                    c.remove();
+                });
+            });
+            c.addClass("op-warn")
+        }
+        // containerEl.createDiv(this.plugin.settings.versionUpFlash);
         new Setting(containerEl)
             .setName("LiveSync")
             .setDesc("Sync realtime")
@@ -1279,6 +1561,38 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     await this.plugin.saveSettings();
                 })
             );
+        new Setting(containerEl)
+            .setName("Minimum chunk size")
+            .setDesc("(letters), minimum chunk size.")
+            .addText((text) => {
+                text.setPlaceholder("")
+                    .setValue(this.plugin.settings.minimumChunkSize + "")
+                    .onChange(async (value) => {
+                        let v = Number(value);
+                        if (isNaN(v) || v < 10 || v > 1000) {
+                            return 10;
+                        }
+                        this.plugin.settings.minimumChunkSize = v;
+                        await this.plugin.saveSettings();
+                    });
+                text.inputEl.setAttribute("type", "number");
+            });
+        new Setting(containerEl)
+            .setName("LongLine Threshold")
+            .setDesc("(letters), If the line is longer than this, make the line to chunk")
+            .addText((text) => {
+                text.setPlaceholder("")
+                    .setValue(this.plugin.settings.longLineThreshold + "")
+                    .onChange(async (value) => {
+                        let v = Number(value);
+                        if (isNaN(v) || v < 10 || v > 1000) {
+                            return 10;
+                        }
+                        this.plugin.settings.longLineThreshold = v;
+                        await this.plugin.saveSettings();
+                    });
+                text.inputEl.setAttribute("type", "number");
+            });
         new Setting(containerEl)
             .setName("Local Database Operations")
             .addButton((button) =>
@@ -1298,6 +1612,29 @@ class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                         //await this.test();
                     })
             );
+        new Setting(containerEl)
+            .setName("Re-init")
+            .addButton((button) =>
+                button
+                    .setButtonText("Init Database again")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.resetLocalDatabase();
+                        await this.plugin.initializeDatabase();
+                    })
+            );
+
+        new Setting(containerEl)
+            .setName("Garbage Collect")
+            .addButton((button) =>
+                button
+                    .setButtonText("Garbage Collection")
+                    .setDisabled(false)
+                    .onClick(async () => {
+                        await this.plugin.garbageCollect();
+                        //await this.test();
+                    })
+            )
         new Setting(containerEl).setName("Remote Database Operations").addButton((button) =>
             button
                 .setButtonText("Reset remote database")

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Obsidian Live sync",
-    "version": "0.0.7",
+    "version": "0.1.0",
     "minAppVersion": "0.9.12",
     "description": "obsidian Live synchronization plugin.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,15 +1,16 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.0.6",
+    "version": "0.0.8",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.0.6",
+            "version": "0.0.8",
             "license": "MIT",
             "dependencies": {
-                "diff-match-patch": "^1.0.5"
+                "diff-match-patch": "^1.0.5",
+                "xxhash-wasm": "^0.4.2"
             },
             "devDependencies": {
                 "@rollup/plugin-commonjs": "^18.0.0",
@@ -539,6 +540,11 @@
             "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
             "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
             "dev": true
+        },
+        "node_modules/xxhash-wasm": {
+            "version": "0.4.2",
+            "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
+            "integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
         }
     },
     "dependencies": {
@@ -975,6 +981,11 @@
             "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
             "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
             "dev": true
+        },
+        "xxhash-wasm": {
+            "version": "0.4.2",
+            "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
+            "integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
         }
     }
 }

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.0.7",
+    "version": "0.10.0",
     "description": "obsidian Live synchronization plugin.",
     "main": "main.js",
     "scripts": {
@@ -22,6 +22,7 @@
         "typescript": "^4.2.4"
     },
     "dependencies": {
-        "diff-match-patch": "^1.0.5"
+        "diff-match-patch": "^1.0.5",
+        "xxhash-wasm": "^0.4.2"
     }
 }

styles.css

@@ -18,3 +18,8 @@
 .op-pre {
     white-space: pre-wrap;
 }
+.op-warn {
+    border:1px solid salmon;
+    padding:2px;
+    border-radius: 4px;
+}