bump

- JSON merging for data and canvas.

docs/setup_own_server.md

@@ -10,9 +10,11 @@ But some additional configurations are required in `local.ini` to use from Self-
 ```
 [couchdb]
 single_node=true
+max_document_size = 50000000
 
 [chttpd]
 require_valid_user = true
+max_http_request_size = 4294967296
 
 [chttpd_auth]
 require_valid_user = true
@@ -94,4 +96,4 @@ Using Caddy is a handy way to serve the server with SSL automatically.
 
 I have published [docker-compose.yml and ini files](https://github.com/vrtmrz/self-hosted-livesync-server) that launches Caddy and CouchDB at once. Please try it out.
 
-And, be sure to check the server log and be careful of malicious access.
+And, be sure to check the server log and be careful of malicious access.

docs/setup_own_server_cn.md

@@ -11,9 +11,11 @@
 ```
 [couchdb]
 single_node=true
+max_document_size = 50000000
 
 [chttpd]
 require_valid_user = true
+max_http_request_size = 4294967296
 
 [chttpd_auth]
 require_valid_user = true
@@ -92,4 +94,4 @@ Note: 不推荐将 CouchDB 挂载到根目录
 
 提供了 [docker-compose.yml 和 ini 文件](https://github.com/vrtmrz/self-hosted-livesync-server) 可以同时启动 Caddy 和 CouchDB。
 
-注意检查服务器日志，当心恶意访问。
+注意检查服务器日志，当心恶意访问。

docs/setup_own_server_ja.md

@@ -8,12 +8,14 @@ CouchDBを構築するには、[Dockerのイメージ](https://hub.docker.com/_/
 ```
 [couchdb]
 single_node=true
+max_document_size = 50000000
 
 [chttpd]
 require_valid_user = true
 
 [chttpd_auth]
 require_valid_user = true
+max_http_request_size = 4294967296
 authentication_redirect = /_utils/session.html
 
 [httpd]

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.16.7",
+    "version": "0.17.3",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.16.7",
+    "version": "0.17.3",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.16.7",
+            "version": "0.17.3",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.16.7",
+    "version": "0.17.3",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/ObsidianLiveSyncSettingTab.ts

@@ -1,7 +1,7 @@
 import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "obsidian";
 import { DEFAULT_SETTINGS, LOG_LEVEL, ObsidianLiveSyncSettings, RemoteDBSettings } from "./lib/src/types";
 import { path2id, id2path } from "./utils";
-import { delay, versionNumberString2Number } from "./lib/src/utils";
+import { delay, Semaphore, versionNumberString2Number } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb.js";
 import { testCrypt } from "./lib/src/e2ee_v2";
@@ -974,6 +974,24 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     await this.plugin.saveSettings();
                 })
             );
+        new Setting(containerSyncSettingEl)
+            .setName("Disable sensible auto merging on markdown files")
+            .setDesc("If this switch is turned on, a merge dialog will be displayed, even if the sensible-merge is possible automatically. (Turn on to previous behavior)")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.disableMarkdownAutoMerge).onChange(async (value) => {
+                    this.plugin.settings.disableMarkdownAutoMerge = value;
+                    await this.plugin.saveSettings();
+                })
+            );
+        new Setting(containerSyncSettingEl)
+            .setName("Write documents after synchronization even if they have conflict")
+            .setDesc("Turn on to previous behavior")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.writeDocumentsIfConflicted).onChange(async (value) => {
+                    this.plugin.settings.writeDocumentsIfConflicted = value;
+                    await this.plugin.saveSettings();
+                })
+            );
 
 
         new Setting(containerSyncSettingEl)
@@ -1288,7 +1306,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
 
         new Setting(containerHatchEl)
             .setName("Make report to inform the issue")
-            .setDesc("Verify and repair all files and update database without restoring")
             .addButton((button) =>
                 button
                     .setButtonText("Make report")
@@ -1368,21 +1385,28 @@ ${stringifyYaml(pluginConfig)}`;
                     .setDisabled(false)
                     .setWarning()
                     .onClick(async () => {
+                        const semaphore = Semaphore(10);
                         const files = this.app.vault.getFiles();
-                        Logger("Verify and repair all files started", LOG_LEVEL.NOTICE, "verify");
-                        // const notice = NewNotice("", 0);
                         let i = 0;
-                        for (const file of files) {
-                            i++;
-                            Logger(`Update into ${file.path}`);
-                            Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL.NOTICE, "verify");
+                        const processes = files.map(e => (async (file) => {
+                            const releaser = await semaphore.acquire(1, "verifyAndRepair");
+
                             try {
-                                await this.plugin.updateIntoDB(file);
+                                Logger(`Update into ${file.path}`);
+                                await this.plugin.updateIntoDB(file, false, null, true);
+                                i++;
+                                Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL.NOTICE, "verify");
+
                             } catch (ex) {
-                                Logger("could not update:");
+                                i++;
+                                Logger(`Error while verifyAndRepair`, LOG_LEVEL.NOTICE);
                                 Logger(ex);
+                            } finally {
+                                releaser();
                             }
                         }
+                        )(e));
+                        await Promise.all(processes);
                         Logger("done", LOG_LEVEL.NOTICE, "verify");
                     })
             );

src/main.ts

@@ -1,5 +1,5 @@
 import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, PluginManifest, App, } from "obsidian";
-import { diff_match_patch } from "diff-match-patch";
+import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
 
 import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, InternalFileEntry } from "./lib/src/types";
 import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList, InternalFileInfo } from "./types";
@@ -29,7 +29,7 @@ import { DocumentHistoryModal } from "./DocumentHistoryModal";
 
 
 
-import { clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, id2path, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger } from "./utils";
+import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
 import { decrypt, encrypt } from "./lib/src/e2ee_v2";
 
 const isDebug = false;
@@ -709,7 +709,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         clearAllPeriodic();
         clearAllTriggers();
         window.removeEventListener("visibilitychange", this.watchWindowVisibility);
-        window.removeEventListener("online", this.watchOnline)
+        window.removeEventListener("online", this.watchOnline);
         Logger("unloading plugin");
     }
 
@@ -1365,13 +1365,30 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         } else if (targetFile instanceof TFile) {
             const doc = change;
             const file = targetFile;
-            await this.doc2storage_modify(doc, file);
-            if (!this.settings.checkConflictOnlyOnOpen) {
-                this.queueConflictedCheck(file);
-            } else {
-                const af = app.workspace.getActiveFile();
-                if (af && af.path == file.path) {
+            const queueConflictCheck = () => {
+                if (!this.settings.checkConflictOnlyOnOpen) {
                     this.queueConflictedCheck(file);
+                    return true;
+                } else {
+                    const af = app.workspace.getActiveFile();
+                    if (af && af.path == file.path) {
+                        this.queueConflictedCheck(file);
+                        return true;
+                    }
+                }
+                return false;
+            }
+            if (this.settings.writeDocumentsIfConflicted) {
+                await this.doc2storage_modify(doc, file);
+                queueConflictCheck();
+            } else {
+                const d = await this.localDatabase.getDBEntryMeta(id2path(change._id), { conflicts: true })
+                if (d && !d._conflicts) {
+                    await this.doc2storage_modify(doc, file);
+                } else {
+                    if (!queueConflictCheck()) {
+                        Logger(`${id2path(change._id)} is conflicted, write to the storage has been pended.`, LOG_LEVEL.NOTICE);
+                    }
                 }
             }
         } else {
@@ -1954,6 +1971,193 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
         return false;
     }
+    //TODO: TIDY UP
+    async mergeSensibly(path: string, baseRev: string, currentRev: string, conflictedRev: string): Promise<Diff[] | false> {
+        const baseLeaf = await this.getConflictedDoc(path, baseRev);
+        const leftLeaf = await this.getConflictedDoc(path, currentRev);
+        const rightLeaf = await this.getConflictedDoc(path, conflictedRev);
+        let autoMerge = false;
+        if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
+            return false;
+        }
+        // diff between base and each revision
+        const dmp = new diff_match_patch();
+        const mapLeft = dmp.diff_linesToChars_(baseLeaf.data, leftLeaf.data);
+        const diffLeftSrc = dmp.diff_main(mapLeft.chars1, mapLeft.chars2, false);
+        dmp.diff_charsToLines_(diffLeftSrc, mapLeft.lineArray);
+        const mapRight = dmp.diff_linesToChars_(baseLeaf.data, rightLeaf.data);
+        const diffRightSrc = dmp.diff_main(mapRight.chars1, mapRight.chars2, false);
+        dmp.diff_charsToLines_(diffRightSrc, mapRight.lineArray);
+        function splitDiffPiece(src: Diff[]): Diff[] {
+            const ret = [] as Diff[];
+            do {
+                const d = src.shift();
+                const pieces = d[1].split(/([^\n]*\n)/).filter(f => f != "");
+                if (typeof (d) == "undefined") {
+                    break;
+                }
+                if (d[0] != DIFF_DELETE) {
+                    ret.push(...(pieces.map(e => [d[0], e] as Diff)));
+                }
+                if (d[0] == DIFF_DELETE) {
+                    const nd = src.shift();
+
+                    if (typeof (nd) != "undefined") {
+                        const piecesPair = nd[1].split(/([^\n]*\n)/).filter(f => f != "");
+                        if (nd[0] == DIFF_INSERT) {
+                            // it might be pair
+                            for (const pt of pieces) {
+                                ret.push([d[0], pt]);
+                                const pairP = piecesPair.shift();
+                                if (typeof (pairP) != "undefined") ret.push([DIFF_INSERT, pairP]);
+                            }
+                            ret.push(...(piecesPair.map(e => [nd[0], e] as Diff)));
+                        } else {
+                            ret.push(...(pieces.map(e => [d[0], e] as Diff)));
+                            ret.push(...(piecesPair.map(e => [nd[0], e] as Diff)));
+
+                        }
+                    } else {
+                        ret.push(...(pieces.map(e => [0, e] as Diff)));
+                    }
+                }
+            } while (src.length > 0);
+            return ret;
+        }
+
+        const diffLeft = splitDiffPiece(diffLeftSrc);
+        const diffRight = splitDiffPiece(diffRightSrc);
+
+        let rightIdx = 0;
+        let leftIdx = 0;
+        const merged = [] as Diff[];
+        autoMerge = true;
+        LOOP_MERGE:
+        do {
+            if (leftIdx >= diffLeft.length && rightIdx >= diffRight.length) {
+                break LOOP_MERGE;
+            }
+            const leftItem = diffLeft[leftIdx] ?? [0, ""];
+            const rightItem = diffRight[rightIdx] ?? [0, ""];
+            leftIdx++;
+            rightIdx++;
+            // when completely same, leave it .
+            if (leftItem[0] == DIFF_EQUAL && rightItem[0] == DIFF_EQUAL && leftItem[1] == rightItem[1]) {
+                merged.push(leftItem);
+                continue;
+            }
+            if (leftItem[0] == DIFF_DELETE && rightItem[0] == DIFF_DELETE && leftItem[1] == rightItem[1]) {
+                // when deleted evenly,
+                const nextLeftIdx = leftIdx;
+                const nextRightIdx = rightIdx;
+                const [nextLeftItem, nextRightItem] = [diffLeft[nextLeftIdx] ?? [0, ""], diffRight[nextRightIdx] ?? [0, ""]];
+                if ((nextLeftItem[0] == DIFF_INSERT && nextRightItem[0] == DIFF_INSERT) && nextLeftItem[1] != nextRightItem[1]) {
+                    //but next line looks like different
+                    autoMerge = false;
+                    break;
+                } else {
+                    merged.push(leftItem);
+                    continue;
+                }
+            }
+            // when inserted evenly
+            if (leftItem[0] == DIFF_INSERT && rightItem[0] == DIFF_INSERT) {
+                if (leftItem[1] == rightItem[1]) {
+                    merged.push(leftItem);
+                    continue;
+                } else {
+                    // sort by file date.
+                    if (leftLeaf.mtime <= rightLeaf.mtime) {
+                        merged.push(leftItem);
+                        merged.push(rightItem);
+                        continue;
+                    } else {
+                        merged.push(rightItem);
+                        merged.push(leftItem);
+                        continue;
+                    }
+                }
+
+            }
+            // when on inserting, index should be fixed again.
+            if (leftItem[0] == DIFF_INSERT) {
+                rightIdx--;
+                merged.push(leftItem);
+                continue;
+            }
+            if (rightItem[0] == DIFF_INSERT) {
+                leftIdx--;
+                merged.push(rightItem);
+                continue;
+            }
+            // except insertion, the line should not be different.
+            if (rightItem[1] != leftItem[1]) {
+                //TODO: SHOULD BE PANIC.
+                Logger(`MERGING PANIC:${leftItem[0]},${leftItem[1]} == ${rightItem[0]},${rightItem[1]}`, LOG_LEVEL.VERBOSE);
+                autoMerge = false;
+                break LOOP_MERGE;
+            }
+            if (leftItem[0] == DIFF_DELETE) {
+                if (rightItem[0] == DIFF_EQUAL) {
+                    merged.push(leftItem);
+                    continue;
+                } else {
+                    //we cannot perform auto merge.
+                    autoMerge = false;
+                    break LOOP_MERGE;
+                }
+            }
+            if (rightItem[0] == DIFF_DELETE) {
+                if (leftItem[0] == DIFF_EQUAL) {
+                    merged.push(rightItem);
+                    continue;
+                } else {
+                    //we cannot perform auto merge.
+                    autoMerge = false;
+                    break LOOP_MERGE;
+                }
+            }
+            Logger(`Weird condition:${leftItem[0]},${leftItem[1]} == ${rightItem[0]},${rightItem[1]}`, LOG_LEVEL.VERBOSE);
+            // here is the exception
+            break LOOP_MERGE;
+        } while (leftIdx < diffLeft.length || rightIdx < diffRight.length);
+        if (autoMerge) {
+            Logger(`Sensibly merge available`, LOG_LEVEL.VERBOSE);
+            return merged;
+        } else {
+            return false;
+        }
+    }
+
+    async mergeObject(path: string, baseRev: string, currentRev: string, conflictedRev: string): Promise<string | false> {
+        const baseLeaf = await this.getConflictedDoc(path, baseRev);
+        const leftLeaf = await this.getConflictedDoc(path, currentRev);
+        const rightLeaf = await this.getConflictedDoc(path, conflictedRev);
+        if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
+            return false;
+        }
+        const baseObj = { data: tryParseJSON(baseLeaf.data, {}) } as Record<string | number | symbol, any>;
+        const leftObj = { data: tryParseJSON(leftLeaf.data, {}) } as Record<string | number | symbol, any>;
+        const rightObj = { data: tryParseJSON(rightLeaf.data, {}) } as Record<string | number | symbol, any>;
+
+        const diffLeft = generatePatchObj(baseObj, leftObj);
+        const diffRight = generatePatchObj(baseObj, rightObj);
+        const patches = [
+            { mtime: leftLeaf.mtime, patch: diffLeft },
+            { mtime: rightLeaf.mtime, patch: diffRight }
+        ].sort((a, b) => a.mtime - b.mtime);
+        let newObj = { ...baseObj };
+        try {
+            for (const patch of patches) {
+                newObj = applyPatch(newObj, patch.patch);
+            }
+            return JSON.stringify(newObj.data);
+        } catch (ex) {
+            Logger("Could not merge object");
+            Logger(ex, LOG_LEVEL.VERBOSE)
+            return false;
+        }
+    }
 
     /**
      * Getting file conflicted status.
@@ -1966,9 +2170,56 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (test == null) return false;
         if (!test._conflicts) return false;
         if (test._conflicts.length == 0) return false;
+        const conflicts = test._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
+        if ((isSensibleMargeApplicable(path) || isObjectMargeApplicable(path)) && !this.settings.disableMarkdownAutoMerge) {
+            const conflictedRev = conflicts[0];
+            const conflictedRevNo = Number(conflictedRev.split("-")[0]);
+            //Search 
+            const revFrom = (await this.localDatabase.localDatabase.get(id2path(path), { revs_info: true })) as unknown as LoadedEntry & PouchDB.Core.GetMeta;
+            const commonBase = revFrom._revs_info.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first().rev ?? "";
+            let p = undefined;
+            if (commonBase) {
+                if (isSensibleMargeApplicable(path)) {
+                    const result = await this.mergeSensibly(path, commonBase, test._rev, conflictedRev);
+                    if (result) {
+                        p = result.filter(e => e[0] != DIFF_DELETE).map((e) => e[1]).join("");
+                        // can be merged.
+                        Logger(`Sensible merge:${path}`, LOG_LEVEL.INFO);
+                    } else {
+                        Logger(`Sensible merge is not applicable.`, LOG_LEVEL.VERBOSE);
+                    }
+                } else if (isObjectMargeApplicable(path)) {
+                    // can be merged.
+                    const result = await this.mergeObject(path, commonBase, test._rev, conflictedRev);
+                    if (result) {
+                        Logger(`Object merge:${path}`, LOG_LEVEL.INFO);
+                        p = result;
+                    } else {
+                        Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
+                    }
+                }
+
+                if (p != undefined) {
+                    // remove conflicted revision.
+                    await this.localDatabase.deleteDBEntry(path, { rev: conflictedRev });
+
+                    const file = getAbstractFileByPath(path) as TFile;
+                    if (file) {
+                        await this.app.vault.modify(file, p);
+                        await this.updateIntoDB(file);
+                    } else {
+                        const newFile = await this.app.vault.create(path, p);
+                        await this.updateIntoDB(newFile);
+                    }
+                    await this.pullFile(path);
+                    Logger(`Automatically merged (sensible) :${path}`, LOG_LEVEL.INFO);
+                    return true;
+                }
+            }
+        }
         // should be one or more conflicts;
         const leftLeaf = await this.getConflictedDoc(path, test._rev);
-        const rightLeaf = await this.getConflictedDoc(path, test._conflicts[0]);
+        const rightLeaf = await this.getConflictedDoc(path, conflicts[0]);
         if (leftLeaf == false) {
             // what's going on..
             Logger(`could not get current revisions:${path}`, LOG_LEVEL.NOTICE);
@@ -1976,7 +2227,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
         if (rightLeaf == false) {
             // Conflicted item could not load, delete this.
-            await this.localDatabase.deleteDBEntry(path, { rev: test._conflicts[0] });
+            await this.localDatabase.deleteDBEntry(path, { rev: conflicts[0] });
             await this.pullFile(path, null, true);
             Logger(`could not get old revisions, automatically used newer one:${path}`, LOG_LEVEL.NOTICE);
             return true;
@@ -2032,11 +2283,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 const toDelete = selected;
                 const toKeep = conflictCheckResult.left.rev != toDelete ? conflictCheckResult.left.rev : conflictCheckResult.right.rev;
                 if (toDelete == "") {
-                    //concat both,
-                    // write data,and delete both old rev.
+                    // concat both,
+                    // delete conflicted revision and write a new file, store it again.
                     const p = conflictCheckResult.diff.map((e) => e[1]).join("");
-                    await this.localDatabase.deleteDBEntry(filename, { rev: conflictCheckResult.left.rev });
-                    await this.localDatabase.deleteDBEntry(filename, { rev: conflictCheckResult.right.rev });
+                    await this.localDatabase.deleteDBEntry(filename, { rev: testDoc._conflicts[0] });
                     const file = getAbstractFileByPath(filename) as TFile;
                     if (file) {
                         await this.app.vault.modify(file, p);
@@ -2092,7 +2342,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                     Logger(ex);
                 }
             }
-        }, 1000);
+        }, 100);
     }
 
     async showIfConflicted(filename: string) {
@@ -2192,7 +2442,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
     }
 
-    async updateIntoDB(file: TFile, initialScan?: boolean, cache?: CacheData) {
+    async updateIntoDB(file: TFile, initialScan?: boolean, cache?: CacheData, force?: boolean) {
         if (!this.isTargetFile(file)) return;
         if (shouldBeIgnored(file.path)) {
             return;
@@ -2234,15 +2484,24 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             if (recentlyTouched(file)) {
                 return true;
             }
-            const old = await this.localDatabase.getDBEntry(fullPath, null, false, false);
-            if (old !== false) {
-                const oldData = { data: old.data, deleted: old._deleted || old.deleted, };
-                const newData = { data: d.data, deleted: d._deleted || d.deleted };
-                if (JSON.stringify(oldData) == JSON.stringify(newData)) {
-                    Logger(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
-                    return true;
+            try {
+                const old = await this.localDatabase.getDBEntry(fullPath, null, false, false);
+                if (old !== false) {
+                    const oldData = { data: old.data, deleted: old._deleted || old.deleted, };
+                    const newData = { data: d.data, deleted: d._deleted || d.deleted };
+                    if (JSON.stringify(oldData) == JSON.stringify(newData)) {
+                        Logger(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
+                        return true;
+                    }
+                    // d._rev = old._rev;
                 }
-                // d._rev = old._rev;
+            } catch (ex) {
+                if (force) {
+                    Logger(msg + "Error, Could not check the diff for the old one." + (force ? "force writing." : "") + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
+                } else {
+                    Logger(msg + "Error, Could not check the diff for the old one." + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
+                }
+                return !force;
             }
             return false;
         });
@@ -2702,8 +2961,33 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (!("_conflicts" in doc)) return false;
         if (doc._conflicts.length == 0) return false;
         Logger(`Hidden file conflicted:${id2filenameInternalChunk(id)}`);
+        const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
+
         const revA = doc._rev;
-        const revB = doc._conflicts[0];
+        const revB = conflicts[0];
+
+        const conflictedRev = conflicts[0];
+        const conflictedRevNo = Number(conflictedRev.split("-")[0]);
+        //Search 
+        const revFrom = (await this.localDatabase.localDatabase.get(id, { revs_info: true })) as unknown as LoadedEntry & PouchDB.Core.GetMeta;
+        const commonBase = revFrom._revs_info.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first().rev ?? "";
+        const result = await this.mergeObject(id, commonBase, doc._rev, conflictedRev);
+        if (result) {
+            Logger(`Object merge:${id}`, LOG_LEVEL.INFO);
+            const filename = id2filenameInternalChunk(id);
+            const isExists = await this.app.vault.adapter.exists(filename);
+            if (!isExists) {
+                await this.ensureDirectoryEx(filename);
+            }
+            await this.app.vault.adapter.write(filename, result);
+            const stat = await this.app.vault.adapter.stat(filename);
+            await this.storeInternalFileToDatabase({ path: filename, ...stat });
+            await this.extractInternalFileFromDatabase(filename);
+            await this.localDatabase.localDatabase.remove(id, revB);
+            return this.resolveConflictOnInternalFile(id);
+        } else {
+            Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
+        }
 
         const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
         // determine which revision should been deleted.

src/utils.ts

@@ -72,4 +72,129 @@ export function retrieveMemoObject<T>(key: string): T | false {
 }
 export function disposeMemoObject(key: string) {
     delete memos[key];
-}
+}
+
+export function isSensibleMargeApplicable(path: string) {
+    if (path.endsWith(".md")) return true;
+    return false;
+}
+export function isObjectMargeApplicable(path: string) {
+    if (path.endsWith(".canvas")) return true;
+    if (path.endsWith(".json")) return true;
+    return false;
+}
+
+export function tryParseJSON(str: string, fallbackValue?: any) {
+    try {
+        return JSON.parse(str);
+    } catch (ex) {
+        return fallbackValue;
+    }
+}
+
+const MARK_OPERATOR = `\u{0001}`;
+const MARK_DELETED = `${MARK_OPERATOR}__DELETED`;
+const MARK_ISARRAY = `${MARK_OPERATOR}__ARRAY`;
+const MARK_SWAPPED = `${MARK_OPERATOR}__SWAP`;
+
+function unorderedArrayToObject(obj: Array<any>) {
+    return obj.map(e => ({ [e.id as string]: e })).reduce((p, c) => ({ ...p, ...c }), {})
+}
+function objectToUnorderedArray(obj: object) {
+    const entries = Object.entries(obj);
+    if (entries.some(e => e[0] != e[1]?.id)) throw new Error("Item looks like not unordered array")
+    return entries.map(e => e[1]);
+}
+function generatePatchUnorderedArray(from: Array<any>, to: Array<any>) {
+    if (from.every(e => typeof (e) == "object" && ("id" in e)) && to.every(e => typeof (e) == "object" && ("id" in e))) {
+        const fObj = unorderedArrayToObject(from);
+        const tObj = unorderedArrayToObject(to);
+        const diff = generatePatchObj(fObj, tObj);
+        if (Object.keys(diff).length > 0) {
+            return { [MARK_ISARRAY]: diff };
+        } else {
+            return {};
+        }
+    }
+    return { [MARK_SWAPPED]: to };
+}
+
+export function generatePatchObj(from: Record<string | number | symbol, any>, to: Record<string | number | symbol, any>) {
+    const entries = Object.entries(from);
+    const tempMap = new Map<string | number | symbol, any>(entries);
+    const ret = {} as Record<string | number | symbol, any>;
+    const newEntries = Object.entries(to);
+    for (const [key, value] of newEntries) {
+        if (!tempMap.has(key)) {
+            //New
+            ret[key] = value;
+            tempMap.delete(key);
+        } else {
+            //Exists
+            const v = tempMap.get(key);
+            if (typeof (v) !== typeof (value) || (Array.isArray(v) !== Array.isArray(value))) {
+                //if type is not match, replace completely.
+                ret[key] = { [MARK_SWAPPED]: value };
+            } else {
+                if (typeof (v) == "object" && typeof (value) == "object" && !Array.isArray(v) && !Array.isArray(value)) {
+                    const wk = generatePatchObj(v, value);
+                    if (Object.keys(wk).length > 0) ret[key] = wk;
+                } else if (typeof (v) == "object" && typeof (value) == "object" && Array.isArray(v) && Array.isArray(value)) {
+                    const wk = generatePatchUnorderedArray(v, value);
+                    if (Object.keys(wk).length > 0) ret[key] = wk;
+                } else if (typeof (v) != "object" && typeof (value) != "object") {
+                    if (JSON.stringify(tempMap.get(key)) !== JSON.stringify(value)) {
+                        ret[key] = value;
+                    }
+                } else {
+                    if (JSON.stringify(tempMap.get(key)) !== JSON.stringify(value)) {
+                        ret[key] = { [MARK_SWAPPED]: value };
+                    }
+                }
+            }
+            tempMap.delete(key);
+        }
+    }
+    //Not used item, means deleted one
+    for (const [key,] of tempMap) {
+        ret[key] = MARK_DELETED
+    }
+    return ret;
+}
+
+
+export function applyPatch(from: Record<string | number | symbol, any>, patch: Record<string | number | symbol, any>) {
+    const ret = from;
+    const patches = Object.entries(patch);
+    for (const [key, value] of patches) {
+        if (value == MARK_DELETED) {
+            delete ret[key];
+            continue;
+        }
+        if (typeof (value) == "object") {
+            if (MARK_SWAPPED in value) {
+                ret[key] = value[MARK_SWAPPED];
+                continue;
+            }
+            if (MARK_ISARRAY in value) {
+                if (!(key in ret)) ret[key] = [];
+                if (!Array.isArray(ret[key])) {
+                    throw new Error("Patch target type is mismatched (array to something)");
+                }
+                const orgArrayObject = unorderedArrayToObject(ret[key]);
+                const appliedObject = applyPatch(orgArrayObject, value[MARK_ISARRAY]);
+                const appliedArray = objectToUnorderedArray(appliedObject);
+                ret[key] = [...appliedArray];
+            } else {
+                if (!(key in ret)) {
+                    ret[key] = value;
+                    continue;
+                }
+                ret[key] = applyPatch(ret[key], value);
+            }
+        } else {
+            ret[key] = value;
+        }
+    }
+    return ret;
+}

styles.css

@@ -106,7 +106,8 @@
 }
 
 .CodeMirror-wrap::before,
-.cm-s-obsidian>.cm-editor::before {
+.cm-s-obsidian>.cm-editor::before,
+.canvas-wrapper::before {
     content: var(--slsmessage);
     text-align: right;
     white-space: pre-wrap;
@@ -122,6 +123,10 @@
     filter: grayscale(100%);
 }
 
+.canvas-wrapper::before {
+    right: 48px;
+}
+
 .CodeMirror-wrap::before {
     right: 0px;
 }

updates.md

@@ -1,3 +1,28 @@
+### 0.17.0
+- 0.17.0 has no surfaced changes but the design of saving chunks has been changed. They have compatibility but changing files after upgrading makes different chunks than before 0.16.x.
+  Please rebuild databases once if you have been worried about storage usage.
+
+  - Improved:
+    - Splitting markdown
+    - Saving chunks
+
+  - Changed:
+    - Chunk ID numbering rules
+
+#### Minors
+- 0.17.1
+  - Fixed: Now we can verify and repair the database.
+  - Refactored inside.
+
+- 0.17.2
+  - New feature
+    - We can merge conflicted documents automatically if sensible.
+  - Fixed
+    - Writing to the storage will be pended while they have conflicts after replication.
+
+- 0.17.3
+  - Now we supported canvas! And conflicted JSON files are also synchronised with merging its content if they are obvious.
+
 ### 0.16.0
 - Now hidden files need not be scanned. Changes will be detected automatically.
   - If you want it to back to its previous behaviour, please disable `Monitor changes to internal files`.
@@ -27,28 +52,7 @@
     - Corrupted chunks will be detected automatically.
     - Now on the case-insensitive system, `aaa.md` and `AAA.md` will be treated as the same file or path at applying changesets.
 - 0.16.7 Nothing has been changed except toolsets, framework library, and as like them. Please inform me if something had been getting strange!
+- 0.16.8 Now we can synchronise without `bad_request:invalid UTF-8 JSON` even while end-to-end encryption has been disabled.
 
 Note:
 Before 0.16.5, LiveSync had some issues making chunks. In this case, synchronisation had became been always failing after a corrupted one should be made. After 0.16.6, the corrupted chunk is automatically detected. Sorry for troubling you but please do `rebuild everything` when this plug-in notified so.
-
-### 0.15.0
-- Outdated configuration items have been removed.
-- Setup wizard has been implemented!
-
-I appreciate for reviewing and giving me advice @Pouhon158!
-
-#### Minors
-- 0.15.1 Missed the stylesheet.
-- 0.15.2 The wizard has been improved and documented!
-- 0.15.3 Fixed the issue about locking/unlocking remote database while rebuilding in the wizard.
-- 0.15.4 Fixed issues about asynchronous processing (e.g., Conflict check or hidden file detection)
-- 0.15.5 Add new features for setting Self-hosted LiveSync up more easier.
-- 0.15.6 File tracking logic has been refined.
-- 0.15.7 Fixed bug about renaming file.
-- 0.15.8 Fixed bug about deleting empty directory, weird behaviour on boot-sequence on mobile devices.
-- 0.15.9 Improved chunk retrieving, now chunks are retrieved in batch on continuous requests.
-- 0.15.10 Fixed:
-  - The boot sequence has been corrected and now boots smoothly.
-  - Auto applying of batch save will be processed earlier than before.
-
-... To continue on to `updates_old.md`.