bump

- Improved
  - New configuration to avoid exceeding throttle capacity.
  - The conflicted `data.json` is no longer merged automatically.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.17.16",
+    "version": "0.17.21",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.17.16",
+    "version": "0.17.21",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.17.16",
+            "version": "0.17.21",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.17.16",
+    "version": "0.17.21",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/DocumentHistoryModal.ts

@@ -6,6 +6,7 @@ import ObsidianLiveSyncPlugin from "./main";
 import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
 import { LoadedEntry, LOG_LEVEL } from "./lib/src/types";
 import { Logger } from "./lib/src/logger";
+import { isErrorOfMissingDoc } from "./lib/src/utils_couchdb";
 import { getDocData } from "./lib/src/utils";
 
 export class DocumentHistoryModal extends Modal {
@@ -35,13 +36,13 @@ export class DocumentHistoryModal extends Modal {
         const db = this.plugin.localDatabase;
         try {
             const w = await db.localDatabase.get(path2id(this.file), { revs_info: true });
-            this.revs_info = w._revs_info.filter((e) => e.status == "available");
+            this.revs_info = w._revs_info.filter((e) => e?.status == "available");
             this.range.max = `${this.revs_info.length - 1}`;
             this.range.value = this.range.max;
             this.fileInfo.setText(`${this.file} / ${this.revs_info.length} revisions`);
             await this.loadRevs();
         } catch (ex) {
-            if (ex.status && ex.status == 404) {
+            if (isErrorOfMissingDoc(ex)) {
                 this.range.max = "0";
                 this.range.value = "";
                 this.range.disabled = true;

src/JsonResolveModal.ts

@@ -0,0 +1,54 @@
+import { App, Modal } from "obsidian";
+import { LoadedEntry } from "./lib/src/types";
+import JsonResolvePane from "./JsonResolvePane.svelte";
+
+export class JsonResolveModal extends Modal {
+    // result: Array<[number, string]>;
+    filename: string;
+    callback: (keepRev: string, mergedStr?: string) => Promise<void>;
+    docs: LoadedEntry[];
+    component: JsonResolvePane;
+
+    constructor(app: App, filename: string, docs: LoadedEntry[], callback: (keepRev: string, mergedStr?: string) => Promise<void>) {
+        super(app);
+        this.callback = callback;
+        this.filename = filename;
+        this.docs = docs;
+    }
+    async UICallback(keepRev: string, mergedStr?: string) {
+        this.close();
+        await this.callback(keepRev, mergedStr);
+        this.callback = null;
+    }
+
+    onOpen() {
+        const { contentEl } = this;
+
+        contentEl.empty();
+
+        if (this.component == null) {
+            this.component = new JsonResolvePane({
+                target: contentEl,
+                props: {
+                    docs: this.docs,
+                    callback: (keepRev, mergedStr) => this.UICallback(keepRev, mergedStr),
+                },
+            });
+        }
+        return;
+    }
+
+
+    onClose() {
+        const { contentEl } = this;
+        contentEl.empty();
+        // contentEl.empty();
+        if (this.callback != null) {
+            this.callback(null);
+        }
+        if (this.component != null) {
+            this.component.$destroy();
+            this.component = null;
+        }
+    }
+}

src/JsonResolvePane.svelte

@@ -0,0 +1,162 @@
+<script lang="ts">
+    import { Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
+    import type { LoadedEntry } from "./lib/src/types";
+    import { base64ToString } from "./lib/src/strbin";
+    import { getDocData } from "./lib/src/utils";
+    import { mergeObject } from "./utils";
+
+    export let docs: LoadedEntry[] = [];
+    export let callback: (keepRev: string, mergedStr?: string) => Promise<void> = async (_, __) => {
+        Promise.resolve();
+    };
+
+    let docA: LoadedEntry = undefined;
+    let docB: LoadedEntry = undefined;
+    let docAContent = "";
+    let docBContent = "";
+    let objA: any = {};
+    let objB: any = {};
+    let objAB: any = {};
+    let objBA: any = {};
+    let diffs: Diff[];
+    const modes = [
+        ["", "Not now"],
+        ["A", "A"],
+        ["B", "B"],
+        ["AB", "A + B"],
+        ["BA", "B + A"],
+    ] as ["" | "A" | "B" | "AB" | "BA", string][];
+    let mode: "" | "A" | "B" | "AB" | "BA" = "";
+
+    function docToString(doc: LoadedEntry) {
+        return doc.datatype == "plain" ? getDocData(doc.data) : base64ToString(doc.data);
+    }
+    function revStringToRevNumber(rev: string) {
+        return rev.split("-")[0];
+    }
+
+    function getDiff(left: string, right: string) {
+        const dmp = new diff_match_patch();
+        const mapLeft = dmp.diff_linesToChars_(left, right);
+        const diffLeftSrc = dmp.diff_main(mapLeft.chars1, mapLeft.chars2, false);
+        dmp.diff_charsToLines_(diffLeftSrc, mapLeft.lineArray);
+        return diffLeftSrc;
+    }
+    function getJsonDiff(a: object, b: object) {
+        return getDiff(JSON.stringify(a, null, 2), JSON.stringify(b, null, 2));
+    }
+    function apply() {
+        if (mode == "A") return callback(docA._rev, null);
+        if (mode == "B") return callback(docB._rev, null);
+        if (mode == "BA") return callback(null, JSON.stringify(objBA, null, 2));
+        if (mode == "AB") return callback(null, JSON.stringify(objAB, null, 2));
+        callback(null, null);
+    }
+    $: {
+        if (docs && docs.length >= 1) {
+            if (docs[0].mtime < docs[1].mtime) {
+                docA = docs[0];
+                docB = docs[1];
+            } else {
+                docA = docs[1];
+                docB = docs[0];
+            }
+            docAContent = docToString(docA);
+            docBContent = docToString(docB);
+
+            try {
+                objA = false;
+                objB = false;
+                objA = JSON.parse(docAContent);
+                objB = JSON.parse(docBContent);
+                objAB = mergeObject(objA, objB);
+                objBA = mergeObject(objB, objA);
+                if (JSON.stringify(objAB) == JSON.stringify(objBA)) {
+                    objBA = false;
+                }
+            } catch (ex) {
+                objBA = false;
+                objAB = false;
+            }
+        }
+    }
+    $: mergedObjs = {
+        "": false,
+        A: objA,
+        B: objB,
+        AB: objAB,
+        BA: objBA,
+    };
+
+    $: selectedObj = mode in mergedObjs ? mergedObjs[mode] : {};
+    $: {
+        diffs = getJsonDiff(objA, selectedObj);
+        console.dir(selectedObj);
+    }
+</script>
+
+<h1>File Conflicted</h1>
+{#if !docA || !docB}
+    <div class="message">Just for a minute, please!</div>
+    <div class="buttons">
+        <button on:click={apply}>Dismiss</button>
+    </div>
+{:else}
+    <div class="options">
+        {#each modes as m}
+            {#if m[0] == "" || mergedObjs[m[0]] != false}
+                <label class={`sls-setting-label ${m[0] == mode ? "selected" : ""}`}
+                    ><input type="radio" name="disp" bind:group={mode} value={m[0]} class="sls-setting-tab" />
+                    <div class="sls-setting-menu-btn">{m[1]}</div></label
+                >
+            {/if}
+        {/each}
+    </div>
+
+    {#if selectedObj != false}
+        <div class="op-scrollable json-source">
+            {#each diffs as diff}
+                <span class={diff[0] == DIFF_DELETE ? "deleted" : diff[0] == DIFF_INSERT ? "added" : "normal"}>{diff[1]}</span>
+            {/each}
+        </div>
+    {:else}
+        NO PREVIEW
+    {/if}
+    <div>
+        A Rev:{revStringToRevNumber(docA._rev)} ,{new Date(docA.mtime).toLocaleString()}
+        {docAContent.length} letters
+    </div>
+
+    <div>
+        B Rev:{revStringToRevNumber(docB._rev)} ,{new Date(docB.mtime).toLocaleString()}
+        {docBContent.length} letters
+    </div>
+
+    <div class="buttons">
+        <button on:click={apply}>Apply</button>
+    </div>
+{/if}
+
+<style>
+    .deleted {
+        text-decoration: line-through;
+    }
+    * {
+        box-sizing: border-box;
+    }
+
+    .scroller {
+        display: flex;
+        flex-direction: column;
+        overflow-y: scroll;
+        max-height: 60vh;
+        user-select: text;
+    }
+    .json-source {
+        white-space: pre;
+        height: auto;
+        overflow: auto;
+        min-height: var(--font-ui-medium);
+        flex-grow: 1;
+    }
+</style>

src/ObsidianLiveSyncSettingTab.ts

@@ -1279,6 +1279,20 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 text.inputEl.setAttribute("type", "number");
             });
 
+        new Setting(containerSyncSettingEl)
+            .setName("Use timeouts instead of heartbeats")
+            .setDesc("If this option is enabled, PouchDB will hold the connection open for 60 seconds, and if no change arrives in that time, close and reopen the socket, instead of holding it open indefinitely. Useful when a proxy limits request duration but can increase resource usage.")
+            .addToggle((toggle) => {
+                toggle
+                    .setValue(this.plugin.settings.useTimeouts)
+                    .onChange(async (value) => {
+                        this.plugin.settings.useTimeouts = value;
+                        await this.plugin.saveSettings();
+                    })
+                return toggle;
+            }
+        );
+
         addScreenElement("30", containerSyncSettingEl);
         const containerMiscellaneousEl = containerEl.createDiv();
         containerMiscellaneousEl.createEl("h3", { text: "Miscellaneous" });

src/main.ts

@@ -2,20 +2,20 @@ import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbst
 import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
 import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, InternalFileEntry, SALT_OF_PASSPHRASE, ConfigPassphraseStore, CouchDBConnection, FLAGMD_REDFLAG2 } from "./lib/src/types";
 import { PluginDataEntry, PERIODIC_PLUGIN_SWEEP, PluginList, DevicePluginList, InternalFileInfo, queueItem, FileInfo } from "./types";
-import { getDocData, isDocContentSame } from "./lib/src/utils";
+import { delay, getDocData, isDocContentSame } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { LocalPouchDB } from "./LocalPouchDB";
 import { LogDisplayModal } from "./LogDisplayModal";
 import { ConflictResolveModal } from "./ConflictResolveModal";
 import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
 import { DocumentHistoryModal } from "./DocumentHistoryModal";
-import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
+import { applyPatch, clearAllPeriodic, clearAllTriggers, clearTrigger, disposeMemoObject, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, memoIfNotExist, memoObject, flattenObject, path2id, retrieveMemoObject, setTrigger, tryParseJSON } from "./utils";
 import { decrypt, encrypt, tryDecrypt } from "./lib/src/e2ee_v2";
 
 const isDebug = false;
 
 import { InputStringDialog, PluginDialogModal, PopoverSelectString } from "./dialogs";
-import { isCloudantURI } from "./lib/src/utils_couchdb";
+import { isCloudantURI, isErrorOfMissingDoc } from "./lib/src/utils_couchdb";
 import { getGlobalStore, observeStores } from "./lib/src/store";
 import { lockStore, logMessageStore, logStore } from "./lib/src/stores";
 import { NewNotice, setNoticeClass, WrappedNotice } from "./lib/src/wrapper";
@@ -23,6 +23,7 @@ import { base64ToString, versionNumberString2Number, base64ToArrayBuffer, arrayB
 import { isPlainText, isValidPath, shouldBeIgnored } from "./lib/src/path";
 import { runWithLock } from "./lib/src/lock";
 import { Semaphore } from "./lib/src/semaphore";
+import { JsonResolveModal } from "./JsonResolveModal";
 
 setNoticeClass(Notice);
 
@@ -1148,9 +1149,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (!this.settings.syncInternalFiles) return;
         if (!this.settings.watchInternalFileChanges) return;
         if (!path.startsWith(this.app.vault.configDir)) return;
-        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         if (ignorePatterns.some(e => path.match(e))) return;
         this.appendWatchEvent(
             [{
@@ -1920,10 +1921,25 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 await this.syncAllFiles(showingNotice);
             }
             if (this.settings.syncInternalFiles) {
-                await this.syncInternalFilesAndDatabase("push", showingNotice);
+                try {
+                    Logger("Synchronizing hidden files...");
+                    await this.syncInternalFilesAndDatabase("push", showingNotice);
+                    Logger("Synchronizing hidden files done");
+                } catch (ex) {
+                    Logger("Synchronizing hidden files failed");
+                    Logger(ex, LOG_LEVEL.VERBOSE)
+                }
             }
             if (this.settings.usePluginSync) {
-                await this.sweepPlugin(showingNotice);
+                try {
+                    Logger("Scanning plugins...");
+                    await this.sweepPlugin(showingNotice);
+                    Logger("Scanning plugins done");
+                } catch (ex) {
+                    Logger("Scanning plugins  failed");
+                    Logger(ex, LOG_LEVEL.VERBOSE)
+                }
+
             }
             this.isReady = true;
             // run queued event once.
@@ -2132,7 +2148,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 data: data
             };
         } catch (ex) {
-            if (ex.status && ex.status == 404) {
+            if (isErrorOfMissingDoc(ex)) {
                 return false;
             }
         }
@@ -2297,24 +2313,44 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async mergeObject(path: string, baseRev: string, currentRev: string, conflictedRev: string): Promise<string | false> {
-        const baseLeaf = await this.getConflictedDoc(path, baseRev);
-        const leftLeaf = await this.getConflictedDoc(path, currentRev);
-        const rightLeaf = await this.getConflictedDoc(path, conflictedRev);
-        if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
-            return false;
-        }
-        const baseObj = { data: tryParseJSON(baseLeaf.data, {}) } as Record<string | number | symbol, any>;
-        const leftObj = { data: tryParseJSON(leftLeaf.data, {}) } as Record<string | number | symbol, any>;
-        const rightObj = { data: tryParseJSON(rightLeaf.data, {}) } as Record<string | number | symbol, any>;
-
-        const diffLeft = generatePatchObj(baseObj, leftObj);
-        const diffRight = generatePatchObj(baseObj, rightObj);
-        const patches = [
-            { mtime: leftLeaf.mtime, patch: diffLeft },
-            { mtime: rightLeaf.mtime, patch: diffRight }
-        ].sort((a, b) => a.mtime - b.mtime);
-        let newObj = { ...baseObj };
         try {
+            const baseLeaf = await this.getConflictedDoc(path, baseRev);
+            const leftLeaf = await this.getConflictedDoc(path, currentRev);
+            const rightLeaf = await this.getConflictedDoc(path, conflictedRev);
+            if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
+                return false;
+            }
+            const baseObj = { data: tryParseJSON(baseLeaf.data, {}) } as Record<string | number | symbol, any>;
+            const leftObj = { data: tryParseJSON(leftLeaf.data, {}) } as Record<string | number | symbol, any>;
+            const rightObj = { data: tryParseJSON(rightLeaf.data, {}) } as Record<string | number | symbol, any>;
+
+            const diffLeft = generatePatchObj(baseObj, leftObj);
+            const diffRight = generatePatchObj(baseObj, rightObj);
+
+            // If each value of the same key has been modified, the automatic merge should be prevented.
+            //TODO Does it have to be a configurable item?
+            const diffSetLeft = new Map(flattenObject(diffLeft));
+            const diffSetRight = new Map(flattenObject(diffRight));
+            for (const [key, value] of diffSetLeft) {
+                if (diffSetRight.has(key)) {
+                    if (diffSetRight.get(key) == value) {
+                        // No matter, if changed to the same value.
+                        diffSetRight.delete(key);
+                    }
+                }
+            }
+            for (const [key, value] of diffSetRight) {
+                if (diffSetLeft.has(key) && diffSetLeft.get(key) != value) {
+                    // Some changes are conflicted
+                    return false;
+                }
+            }
+
+            const patches = [
+                { mtime: leftLeaf.mtime, patch: diffLeft },
+                { mtime: rightLeaf.mtime, patch: diffRight }
+            ].sort((a, b) => a.mtime - b.mtime);
+            let newObj = { ...baseObj };
             for (const patch of patches) {
                 newObj = applyPatch(newObj, patch.patch);
             }
@@ -2490,7 +2526,60 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             }).open();
         });
     }
-
+    showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
+        return new Promise((res) => {
+            Logger("Opening data-merging dialog", LOG_LEVEL.VERBOSE);
+            const docs = [docA, docB];
+            const modal = new JsonResolveModal(this.app, id2path(docA._id), [docA, docB], async (keep, result) => {
+                // modal.close();
+                try {
+                    const filename = id2filenameInternalMetadata(docA._id);
+                    let needFlush = false;
+                    if (!result && !keep) {
+                        Logger(`Skipped merging: ${filename}`);
+                    }
+                    //Delete old revisions
+                    if (result || keep) {
+                        for (const doc of docs) {
+                            if (doc._rev != keep) {
+                                if (await this.localDatabase.deleteDBEntry(doc._id, { rev: doc._rev })) {
+                                    Logger(`Conflicted revision has been deleted: ${filename}`);
+                                    needFlush = true;
+                                }
+                            }
+                        }
+                    }
+                    if (!keep && result) {
+                        const isExists = await this.app.vault.adapter.exists(filename);
+                        if (!isExists) {
+                            await this.ensureDirectoryEx(filename);
+                        }
+                        await this.app.vault.adapter.write(filename, result);
+                        const stat = await this.app.vault.adapter.stat(filename);
+                        await this.storeInternalFileToDatabase({ path: filename, ...stat }, true);
+                        try {
+                            //@ts-ignore internalAPI
+                            await app.vault.adapter.reconcileInternalFile(filename);
+                        } catch (ex) {
+                            Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
+                            Logger(ex, LOG_LEVEL.VERBOSE);
+                        }
+                        Logger(`STORAGE <-- DB:${filename}: written (hidden,merged)`);
+                    }
+                    if (needFlush) {
+                        await this.extractInternalFileFromDatabase(filename, false);
+                        Logger(`STORAGE --> DB:${filename}: extracted (hidden,merged)`);
+                    }
+                    res(true);
+                } catch (ex) {
+                    Logger("Could not merge conflicted json");
+                    Logger(ex, LOG_LEVEL.VERBOSE)
+                    res(false);
+                }
+            })
+            modal.open();
+        });
+    }
     conflictedCheckFiles: string[] = [];
 
     // queueing the conflicted file check
@@ -2961,9 +3050,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async scanInternalFiles(): Promise<InternalFileInfo[]> {
-        const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         const root = this.app.vault.getRoot();
         const findRoot = root.path;
         const filenames = (await this.getFiles(findRoot, [], null, ignoreFilter)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash"));
@@ -2990,41 +3079,47 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         const content = await arrayBufferToBase64(contentBin);
         const mtime = file.mtime;
         return await runWithLock("file-" + id, false, async () => {
-            const old = await this.localDatabase.getDBEntry(id, null, false, false);
-            let saveData: LoadedEntry;
-            if (old === false) {
-                saveData = {
-                    _id: id,
-                    data: content,
-                    mtime,
-                    ctime: mtime,
-                    datatype: "newnote",
-                    size: file.size,
-                    children: [],
-                    deleted: false,
-                    type: "newnote",
+            try {
+                const old = await this.localDatabase.getDBEntry(id, null, false, false);
+                let saveData: LoadedEntry;
+                if (old === false) {
+                    saveData = {
+                        _id: id,
+                        data: content,
+                        mtime,
+                        ctime: mtime,
+                        datatype: "newnote",
+                        size: file.size,
+                        children: [],
+                        deleted: false,
+                        type: "newnote",
+                    }
+                } else {
+                    if (isDocContentSame(old.data, content) && !forceWrite) {
+                        // Logger(`internal files STORAGE --> DB:${file.path}: Not changed`);
+                        return;
+                    }
+                    saveData =
+                    {
+                        ...old,
+                        data: content,
+                        mtime,
+                        size: file.size,
+                        datatype: "newnote",
+                        children: [],
+                        deleted: false,
+                        type: "newnote",
+                    }
                 }
-            } else {
-                if (isDocContentSame(old.data, content) && !forceWrite) {
-                    // Logger(`internal files STORAGE --> DB:${file.path}: Not changed`);
-                    return;
-                }
-                saveData =
-                {
-                    ...old,
-                    data: content,
-                    mtime,
-                    size: file.size,
-                    datatype: "newnote",
-                    children: [],
-                    deleted: false,
-                    type: "newnote",
-                }
-            }
 
-            const ret = await this.localDatabase.putDBEntry(saveData, true);
-            Logger(`STORAGE --> DB:${file.path}: (hidden) Done`);
-            return ret;
+                const ret = await this.localDatabase.putDBEntry(saveData, true);
+                Logger(`STORAGE --> DB:${file.path}: (hidden) Done`);
+                return ret;
+            } catch (ex) {
+                Logger(`STORAGE --> DB:${file.path}: (hidden) Failed`);
+                Logger(ex, LOG_LEVEL.VERBOSE);
+                return false;
+            }
         });
     }
 
@@ -3032,36 +3127,41 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         const id = filename2idInternalMetadata(path2id(filename));
         const mtime = new Date().getTime();
         await runWithLock("file-" + id, false, async () => {
-            const old = await this.localDatabase.getDBEntry(id, null, false, false) as InternalFileEntry | false;
-            let saveData: InternalFileEntry;
-            if (old === false) {
-                saveData = {
-                    _id: id,
-                    mtime,
-                    ctime: mtime,
-                    size: 0,
-                    children: [],
-                    deleted: true,
-                    type: "newnote",
-                }
-            } else {
-                if (old.deleted) {
-                    Logger(`STORAGE -x> DB:${filename}: (hidden) already deleted`);
-                    return;
-                }
-                saveData =
-                {
-                    ...old,
-                    mtime,
-                    size: 0,
-                    children: [],
-                    deleted: true,
-                    type: "newnote",
+            try {
+                const old = await this.localDatabase.getDBEntry(id, null, false, false) as InternalFileEntry | false;
+                let saveData: InternalFileEntry;
+                if (old === false) {
+                    saveData = {
+                        _id: id,
+                        mtime,
+                        ctime: mtime,
+                        size: 0,
+                        children: [],
+                        deleted: true,
+                        type: "newnote",
+                    }
+                } else {
+                    if (old.deleted) {
+                        Logger(`STORAGE -x> DB:${filename}: (hidden) already deleted`);
+                        return;
+                    }
+                    saveData =
+                    {
+                        ...old,
+                        mtime,
+                        size: 0,
+                        children: [],
+                        deleted: true,
+                        type: "newnote",
+                    }
                 }
+                await this.localDatabase.localDatabase.put(saveData);
+                Logger(`STORAGE -x> DB:${filename}: (hidden) Done`);
+            } catch (ex) {
+                Logger(`STORAGE -x> DB:${filename}: (hidden) Failed`);
+                Logger(ex, LOG_LEVEL.VERBOSE);
+                return false;
             }
-            await this.localDatabase.localDatabase.put(saveData);
-            Logger(`STORAGE -x> DB:${filename}: (hidden) Done`);
-
         });
     }
     async ensureDirectoryEx(fullPath: string) {
@@ -3089,28 +3189,46 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         const id = filename2idInternalMetadata(path2id(filename));
 
         return await runWithLock("file-" + id, false, async () => {
-            const fileOnDB = await this.localDatabase.getDBEntry(id, null, false, false) as false | LoadedEntry;
-            if (fileOnDB === false) throw new Error(`File not found on database.:${id}`);
-            const deleted = "deleted" in fileOnDB ? fileOnDB.deleted : false;
-            if (deleted) {
-                if (!isExists) {
-                    Logger(`STORAGE <x- DB:${filename}: deleted (hidden) Deleted on DB, but the file is  already not found on storage.`);
-                } else {
-                    Logger(`STORAGE <x- DB:${filename}: deleted (hidden).`);
-                    await this.app.vault.adapter.remove(filename);
+            try {
+                // Check conflicted status 
+                //TODO option
+                const fileOnDB = await this.localDatabase.getDBEntry(id, { conflicts: true }, false, false) as false | LoadedEntry;
+                if (fileOnDB === false) throw new Error(`File not found on database.:${id}`);
+                // Prevent overrite for Prevent overwriting while some conflicted revision exists.
+                if (fileOnDB?._conflicts?.length) {
+                    Logger(`Hidden file ${id} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL.INFO);
+                    return;
                 }
-                return true;
-            }
-            if (!isExists) {
-                await this.ensureDirectoryEx(filename);
-                await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
-                Logger(`STORAGE <-- DB:${filename}: written (hidden,new${force ? ", force" : ""})`);
-                return true;
-            } else {
-                try {
-                    // const stat = await this.app.vault.adapter.stat(filename);
-                    // const fileMTime = ~~(stat.mtime/1000);
-                    // const docMtime = ~~(old.mtime/1000);
+                const deleted = "deleted" in fileOnDB ? fileOnDB.deleted : false;
+                if (deleted) {
+                    if (!isExists) {
+                        Logger(`STORAGE <x- DB:${filename}: deleted (hidden) Deleted on DB, but the file is  already not found on storage.`);
+                    } else {
+                        Logger(`STORAGE <x- DB:${filename}: deleted (hidden).`);
+                        await this.app.vault.adapter.remove(filename);
+                        try {
+                            //@ts-ignore internalAPI
+                            await app.vault.adapter.reconcileInternalFile(filename);
+                        } catch (ex) {
+                            Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
+                            Logger(ex, LOG_LEVEL.VERBOSE);
+                        }
+                    }
+                    return true;
+                }
+                if (!isExists) {
+                    await this.ensureDirectoryEx(filename);
+                    await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
+                    try {
+                        //@ts-ignore internalAPI
+                        await app.vault.adapter.reconcileInternalFile(filename);
+                    } catch (ex) {
+                        Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
+                        Logger(ex, LOG_LEVEL.VERBOSE);
+                    }
+                    Logger(`STORAGE <-- DB:${filename}: written (hidden,new${force ? ", force" : ""})`);
+                    return true;
+                } else {
                     const contentBin = await this.app.vault.adapter.readBinary(filename);
                     const content = await arrayBufferToBase64(contentBin);
                     if (content == fileOnDB.data && !force) {
@@ -3118,20 +3236,29 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                         return true;
                     }
                     await this.app.vault.adapter.writeBinary(filename, base64ToArrayBuffer(fileOnDB.data), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
+                    try {
+                        //@ts-ignore internalAPI
+                        await app.vault.adapter.reconcileInternalFile(filename);
+                    } catch (ex) {
+                        Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL.VERBOSE);
+                        Logger(ex, LOG_LEVEL.VERBOSE);
+                    }
                     Logger(`STORAGE <-- DB:${filename}: written (hidden, overwrite${force ? ", force" : ""})`);
                     return true;
-                } catch (ex) {
-                    Logger(ex);
-                    return false;
+
                 }
+            } catch (ex) {
+                Logger(`STORAGE <-- DB:${filename}: written (hidden, overwrite${force ? ", force" : ""}) Failed`);
+                Logger(ex, LOG_LEVEL.VERBOSE);
+                return false;
             }
         });
     }
 
     filterTargetFiles(files: InternalFileInfo[], targetFiles: string[] | false = false) {
-        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         return files.filter(file => !ignorePatterns.some(e => file.path.match(e))).filter(file => !targetFiles || (targetFiles && targetFiles.indexOf(file.path) !== -1))
     }
 
@@ -3154,63 +3281,79 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async resolveConflictOnInternalFile(id: string): Promise<boolean> {
-        // Retrieve data
-        const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true });
-        // If there is no conflict, return with false.
-        if (!("_conflicts" in doc)) return false;
-        if (doc._conflicts.length == 0) return false;
-        Logger(`Hidden file conflicted:${id2filenameInternalMetadata(id)}`);
-        const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
-        const revA = doc._rev;
-        const revB = conflicts[0];
+        try {
+            // Retrieve data
+            const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true });
+            // If there is no conflict, return with false.
+            if (!("_conflicts" in doc)) return false;
+            if (doc._conflicts.length == 0) return false;
+            Logger(`Hidden file conflicted:${id2filenameInternalMetadata(id)}`);
+            const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
+            const revA = doc._rev;
+            const revB = conflicts[0];
 
-        if (doc._id.endsWith(".json")) {
-            const conflictedRev = conflicts[0];
-            const conflictedRevNo = Number(conflictedRev.split("-")[0]);
-            //Search 
-            const revFrom = (await this.localDatabase.localDatabase.get(id, { revs_info: true })) as unknown as LoadedEntry & PouchDB.Core.GetMeta;
-            const commonBase = revFrom._revs_info.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first()?.rev ?? "";
-            const result = await this.mergeObject(id, commonBase, doc._rev, conflictedRev);
-            if (result) {
-                Logger(`Object merge:${id}`, LOG_LEVEL.INFO);
-                const filename = id2filenameInternalMetadata(id);
-                const isExists = await this.app.vault.adapter.exists(filename);
-                if (!isExists) {
-                    await this.ensureDirectoryEx(filename);
+            if (doc._id.endsWith(".json")) {
+                const conflictedRev = conflicts[0];
+                const conflictedRevNo = Number(conflictedRev.split("-")[0]);
+                //Search 
+                const revFrom = (await this.localDatabase.localDatabase.get(id, { revs_info: true })) as unknown as LoadedEntry & PouchDB.Core.GetMeta;
+                const commonBase = revFrom._revs_info.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first()?.rev ?? "";
+                const result = await this.mergeObject(id, commonBase, doc._rev, conflictedRev);
+                if (result) {
+                    Logger(`Object merge:${id}`, LOG_LEVEL.INFO);
+                    const filename = id2filenameInternalMetadata(id);
+                    const isExists = await this.app.vault.adapter.exists(filename);
+                    if (!isExists) {
+                        await this.ensureDirectoryEx(filename);
+                    }
+                    await this.app.vault.adapter.write(filename, result);
+                    const stat = await this.app.vault.adapter.stat(filename);
+                    await this.storeInternalFileToDatabase({ path: filename, ...stat });
+                    await this.extractInternalFileFromDatabase(filename);
+                    await this.localDatabase.localDatabase.remove(id, revB);
+                    return this.resolveConflictOnInternalFile(id);
+                } else {
+                    Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
                 }
-                await this.app.vault.adapter.write(filename, result);
-                const stat = await this.app.vault.adapter.stat(filename);
-                await this.storeInternalFileToDatabase({ path: filename, ...stat });
-                await this.extractInternalFileFromDatabase(filename);
-                await this.localDatabase.localDatabase.remove(id, revB);
-                return this.resolveConflictOnInternalFile(id);
-            } else {
-                Logger(`Object merge is not applicable.`, LOG_LEVEL.VERBOSE);
-            }
-        }
-        const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
-        // determine which revision should been deleted.
-        // simply check modified time
-        const mtimeA = ("mtime" in doc && doc.mtime) || 0;
-        const mtimeB = ("mtime" in revBDoc && revBDoc.mtime) || 0;
-        // Logger(`Revisions:${new Date(mtimeA).toLocaleString} and ${new Date(mtimeB).toLocaleString}`);
-        // console.log(`mtime:${mtimeA} - ${mtimeB}`);
-        const delRev = mtimeA < mtimeB ? revA : revB;
-        // delete older one.
-        await this.localDatabase.localDatabase.remove(id, delRev);
-        Logger(`Older one has been deleted:${id2filenameInternalMetadata(id)}`);
-        // check the file again 
-        return this.resolveConflictOnInternalFile(id);
 
+                const docAMerge = await this.localDatabase.getDBEntry(id, { rev: revA });
+                const docBMerge = await this.localDatabase.getDBEntry(id, { rev: revB });
+                if (docAMerge != false && docBMerge != false) {
+                    if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) {
+                        await delay(200);
+                        // Again for other conflicted revisions.
+                        return this.resolveConflictOnInternalFile(id);
+                    }
+                    return false;
+                }
+            }
+            const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
+            // determine which revision should been deleted.
+            // simply check modified time
+            const mtimeA = ("mtime" in doc && doc.mtime) || 0;
+            const mtimeB = ("mtime" in revBDoc && revBDoc.mtime) || 0;
+            // Logger(`Revisions:${new Date(mtimeA).toLocaleString} and ${new Date(mtimeB).toLocaleString}`);
+            // console.log(`mtime:${mtimeA} - ${mtimeB}`);
+            const delRev = mtimeA < mtimeB ? revA : revB;
+            // delete older one.
+            await this.localDatabase.localDatabase.remove(id, delRev);
+            Logger(`Older one has been deleted:${id2filenameInternalMetadata(id)}`);
+            // check the file again 
+            return this.resolveConflictOnInternalFile(id);
+        } catch (ex) {
+            Logger("Failed to resolve conflict (Hidden)")
+            Logger(ex, LOG_LEVEL.VERBOSE);
+            return false;
+        }
     }
     //TODO: Tidy up. Even though it is experimental feature, So dirty...
     async syncInternalFilesAndDatabase(direction: "push" | "pull" | "safe", showMessage: boolean, files: InternalFileInfo[] | false = false, targetFiles: string[] | false = false) {
         await this.resolveConflictOnInternalFiles();
         const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
         Logger("Scanning hidden files.", logLevel, "sync_internal");
-        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns.toLocaleLowerCase()
+        const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
             .replace(/\n| /g, "")
-            .split(",").filter(e => e).map(e => new RegExp(e));
+            .split(",").filter(e => e).map(e => new RegExp(e, "i"));
         if (!files) files = await this.scanInternalFiles();
         const filesOnDB = ((await this.localDatabase.localDatabase.allDocs({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
 
@@ -3249,7 +3392,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             }
         }
         const p = [] as Promise<void>[];
-        const semaphore = Semaphore(15);
+        const semaphore = Semaphore(10);
         // Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content)
         let caches: { [key: string]: { storageMtime: number; docMtime: number } } = {};
         caches = await this.localDatabase.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number } }>("diff-caches-internal") || {};

src/utils.ts

@@ -198,3 +198,65 @@ export function applyPatch(from: Record<string | number | symbol, any>, patch: R
     }
     return ret;
 }
+
+export function mergeObject(
+    objA: Record<string | number | symbol, any>,
+    objB: Record<string | number | symbol, any>
+) {
+    const newEntries = Object.entries(objB);
+    const ret: any = { ...objA };
+    if (
+        typeof objA !== typeof objB ||
+        Array.isArray(objA) !== Array.isArray(objB)
+    ) {
+        return objB;
+    }
+
+    for (const [key, v] of newEntries) {
+        if (key in ret) {
+            const value = ret[key];
+            if (
+                typeof v !== typeof value ||
+                Array.isArray(v) !== Array.isArray(value)
+            ) {
+                //if type is not match, replace completely.
+                ret[key] = v;
+            } else {
+                if (
+                    typeof v == "object" &&
+                    typeof value == "object" &&
+                    !Array.isArray(v) &&
+                    !Array.isArray(value)
+                ) {
+                    ret[key] = mergeObject(v, value);
+                } else if (
+                    typeof v == "object" &&
+                    typeof value == "object" &&
+                    Array.isArray(v) &&
+                    Array.isArray(value)
+                ) {
+                    ret[key] = [...new Set([...v, ...value])];
+                } else {
+                    ret[key] = v;
+                }
+            }
+        } else {
+            ret[key] = v;
+        }
+    }
+    return Object.entries(ret)
+        .sort()
+        .reduce((p, [key, value]) => ({ ...p, [key]: value }), {});
+}
+
+export function flattenObject(obj: Record<string | number | symbol, any>, path: string[] = []): [string, any][] {
+    if (typeof (obj) != "object") return [[path.join("."), obj]];
+    if (Array.isArray(obj)) return [[path.join("."), JSON.stringify(obj)]];
+    const e = Object.entries(obj);
+    const ret = []
+    for (const [key, value] of e) {
+        const p = flattenObject(value, [...path, key]);
+        ret.push(...p);
+    }
+    return ret;
+}

updates.md

@@ -66,6 +66,8 @@
   - Fixed:
     - Hidden files have been synchronised again.
     - Rename of files has been fixed again.
+  And, minor changes have been included.
+
 - 0.17.16:
   - Improved:
     - Plugins and their settings no longer need scanning if changes are monitored.
@@ -79,7 +81,20 @@
     - Plugins and their setting can be synchronised again.
     - Hidden files and plugins are correctly scanned while rebuilding.
     - Files with the name started `_` are also being performed conflict-checking.
-    
-  And, minor changes have been included.
+- 0.17.17
+  - Fixed: Now we can merge JSON files even if we failed to compare items like null.
+- 0.17.18
+  - Fixed: Fixed lack of error handling.
+- 0.17.19 
+  - Fixed: Error reporting has been ensured.
+- 0.17.20
+  - Improved: Changes of hidden files will be notified to Obsidian.
+- 0.17.21
+  - Fixed: Skip patterns now handle capital letters.
+  - Improved
+    - New configuration to avoid exceeding throttle capacity.
+      - We have been grateful to @karasevm!
+    - The conflicted `data.json` is no longer merged automatically.
+      - This behaviour is not configurable, unlike the `Use newer file if conflicted` of normal files.
 
 ... To continue on to `updates_old.md`.