bump

- Now, the filename of the conflicted settings will be shown on the merging dialogue
- The plugin data can be resolved when conflicted.
- The semaphore status display has been changed to count only.
- Applying to the storage will be concurrent with a few files.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.17.23",
+    "version": "0.17.27",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.17.23",
+    "version": "0.17.27",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.17.23",
+            "version": "0.17.27",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.17.23",
+    "version": "0.17.27",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/ConflictResolveModal.ts

@@ -22,7 +22,7 @@ export class ConflictResolveModal extends Modal {
         contentEl.empty();
 
         contentEl.createEl("h2", { text: "This document has conflicted changes." });
-        contentEl.createEl("span", this.filename);
+        contentEl.createEl("span", { text: this.filename });
         const div = contentEl.createDiv("");
         div.addClass("op-scrollable");
         let diff = "";

src/JsonResolvePane.svelte

@@ -3,7 +3,7 @@
     import type { LoadedEntry } from "./lib/src/types";
     import { base64ToString } from "./lib/src/strbin";
     import { getDocData } from "./lib/src/utils";
-    import { mergeObject } from "./utils";
+    import { id2path, mergeObject } from "./utils";
 
     export let docs: LoadedEntry[] = [];
     export let callback: (keepRev: string, mergedStr?: string) => Promise<void> = async (_, __) => {
@@ -93,9 +93,11 @@
         diffs = getJsonDiff(objA, selectedObj);
         console.dir(selectedObj);
     }
+    $: filename = id2path(docA?._id ?? "");
 </script>
 
-<h1>File Conflicted</h1>
+<h1>Conflicted settings</h1>
+<div><span>{filename}</span></div>
 {#if !docA || !docB}
     <div class="message">Just for a minute, please!</div>
     <div class="buttons">

src/ObsidianLiveSyncSettingTab.ts

@@ -1292,6 +1292,38 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 return toggle;
             }
             );
+        new Setting(containerSyncSettingEl)
+            .setName("A number of hashes to be cached")
+            .setDesc("")
+            .addText((text) => {
+                text.setPlaceholder("")
+                    .setValue(this.plugin.settings.hashCacheMaxCount + "")
+                    .onChange(async (value) => {
+                        let v = Number(value);
+                        if (isNaN(v) || v < 10) {
+                            v = 10;
+                        }
+                        this.plugin.settings.hashCacheMaxCount = v;
+                        await this.plugin.saveSettings();
+                    });
+                text.inputEl.setAttribute("type", "number");
+            });
+        new Setting(containerSyncSettingEl)
+            .setName("The total length of hashes to be cached")
+            .setDesc("(Mega chars)")
+            .addText((text) => {
+                text.setPlaceholder("")
+                    .setValue(this.plugin.settings.hashCacheMaxAmount + "")
+                    .onChange(async (value) => {
+                        let v = Number(value);
+                        if (isNaN(v) || v < 1) {
+                            v = 1;
+                        }
+                        this.plugin.settings.hashCacheMaxAmount = v;
+                        await this.plugin.saveSettings();
+                    });
+                text.inputEl.setAttribute("type", "number");
+            });
 
         addScreenElement("30", containerSyncSettingEl);
         const containerMiscellaneousEl = containerEl.createDiv();

src/main.ts

@@ -138,6 +138,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     deviceAndVaultName: string;
     isMobile = false;
     isReady = false;
+    packageVersion = "";
+    manifestVersion = "";
 
     watchedFileEventQueue = [] as FileEventItem[];
 
@@ -194,26 +196,10 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async fileHistory() {
-        const pageLimit = 1000;
-        let nextKey = "";
         const notes: { path: string, mtime: number }[] = [];
-        do {
-            const docs = await this.localDatabase.localDatabase.allDocs({ limit: pageLimit, startkey: nextKey, include_docs: true });
-            nextKey = "";
-            for (const row of docs.rows) {
-                const doc = row.doc;
-                nextKey = `${row.id}\u{10ffff}`;
-                if (!("type" in doc)) continue;
-                if (doc.type == "newnote" || doc.type == "plain") {
-                    notes.push({ path: id2path(doc._id), mtime: doc.mtime });
-                }
-                if (isChunk(nextKey)) {
-                    // skip the chunk zone.
-                    nextKey = CHeaderEnd;
-                }
-            }
-        } while (nextKey != "");
-
+        for await (const doc of this.localDatabase.findAllDocs()) {
+            notes.push({ path: id2path(doc._id), mtime: doc.mtime });
+        }
         notes.sort((a, b) => b.mtime - a.mtime);
         const notesList = notes.map(e => e.path);
         const target = await askSelectString(this.app, "File to view History", notesList);
@@ -222,31 +208,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
     }
     async pickFileForResolve() {
-        const pageLimit = 1000;
-        let nextKey = "";
         const notes: { path: string, mtime: number }[] = [];
-        do {
-            const docs = await this.localDatabase.localDatabase.allDocs({ limit: pageLimit, startkey: nextKey, conflicts: true, include_docs: true });
-            nextKey = "";
-            for (const row of docs.rows) {
-                const doc = row.doc;
-                nextKey = `${row.id}\u{10ffff}`;
-                if (isChunk(nextKey)) {
-                    // skip the chunk zone.
-                    nextKey = CHeaderEnd;
-                }
-                if (!("_conflicts" in doc)) continue;
-                if (isInternalMetadata(row.id)) continue;
-                // We have to check also deleted files.
-                // if (doc._deleted) continue;
-                // if ("deleted" in doc && doc.deleted) continue;
-                if (doc.type == "newnote" || doc.type == "plain") {
-                    // const docId = doc._id.startsWith("i:") ? doc._id.substring("i:".length) : doc._id;
-                    notes.push({ path: id2path(doc._id), mtime: doc.mtime });
-                }
-
-            }
-        } while (nextKey != "");
+        for await (const doc of this.localDatabase.findAllDocs({ conflicts: true })) {
+            if (!("_conflicts" in doc)) continue;
+            notes.push({ path: id2path(doc._id), mtime: doc.mtime });
+        }
         notes.sort((a, b) => b.mtime - a.mtime);
         const notesList = notes.map(e => e.path);
         if (notesList.length == 0) {
@@ -255,11 +221,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
         const target = await askSelectString(this.app, "File to view History", notesList);
         if (target) {
-            if (isInternalMetadata(target)) {
-                //NOP
-            } else {
-                await this.showIfConflicted(target);
-            }
+            await this.resolveConflicted(target);
+        }
+    }
+    async resolveConflicted(target: string) {
+        if (isInternalMetadata(target)) {
+            await this.resolveConflictOnInternalFile(target);
+        } else if (isPluginMetadata(target)) {
+            await this.resolveConflictByNewerEntry(target);
+        } else {
+            await this.showIfConflicted(target);
         }
     }
 
@@ -371,12 +342,33 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 if (this.settings.syncOnStart) {
                     this.localDatabase.openReplication(this.settings, false, false, this.parseReplicationResult);
                 }
+                this.scanStat();
             } catch (ex) {
                 Logger("Error while loading Self-hosted LiveSync", LOG_LEVEL.NOTICE);
                 Logger(ex, LOG_LEVEL.VERBOSE);
             }
     }
 
+    /**
+     * Scan status 
+     */
+    async scanStat() {
+        const notes: { path: string, mtime: number }[] = [];
+        Logger(`Additional safety scan..`, LOG_LEVEL.VERBOSE);
+        for await (const doc of this.localDatabase.findAllDocs({ conflicts: true })) {
+            if (!("_conflicts" in doc)) continue;
+            notes.push({ path: id2path(doc._id), mtime: doc.mtime });
+        }
+        if (notes.length > 0) {
+            Logger(`Some files have been left conflicted! Please resolve them by "Pick a file to resolve conflict". The list is written in the log.`, LOG_LEVEL.NOTICE);
+            for (const note of notes) {
+                Logger(`Conflicted: ${note.path}`);
+            }
+        } else {
+            Logger(`There are no conflicted files`, LOG_LEVEL.VERBOSE);
+        }
+        Logger(`Additional safety scan done`, LOG_LEVEL.VERBOSE);
+    }
     async command_copySetupURI() {
         const encryptingPassphrase = await askString(this.app, "Encrypt your settings", "The passphrase to encrypt the setup URI", "");
         if (encryptingPassphrase === false) return;
@@ -536,6 +528,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         //@ts-ignore
         const packageVersion: string = PACKAGE_VERSION || "0.0.0";
 
+        this.manifestVersion = manifestVersion;
+        this.packageVersion = packageVersion;
 
         Logger(`Self-hosted LiveSync v${manifestVersion} ${packageVersion} `);
         const lsKey = "obsidian-live-sync-ver" + this.getVaultName();
@@ -616,7 +610,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         });
 
         this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this));
-
         this.app.workspace.onLayoutReady(this.onLayoutReady.bind(this));
         this.registerObsidianProtocolHandler("setuplivesync", async (conf: any) => await this.setupWizard(conf.settings));
 
@@ -1315,7 +1308,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         const newMessage = timestamp + "->" + messageContent;
 
         console.log(vaultName + ":" + newMessage);
-        if (this.settings.writeLogToTheFile) {
+        if (this.settings?.writeLogToTheFile) {
             const time = now.toISOString().split("T")[0];
             const logDate = `${PREFIXMD_LOGFILE}${time}.md`;
             const file = this.app.vault.getAbstractFileByPath(normalizePath(logDate));
@@ -1325,7 +1318,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             this.app.vault.adapter.append(normalizePath(logDate), vaultName + ":" + newMessage + "\n");
         }
         logMessageStore.apply(e => [...e, newMessage].slice(-100));
-        this.setStatusBarText(null, messageContent.substring(0, 30));
+        this.setStatusBarText(null, messageContent);
 
         if (level >= LOG_LEVEL.NOTICE) {
             if (!key) key = messageContent;
@@ -1409,7 +1402,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             }
             return;
         }
-        const localMtime = ~~(file?.stat?.mtime || 0 / 1000);
+        const localMtime = ~~((file?.stat?.mtime || 0) / 1000);
         const docMtime = ~~(docEntry.mtime / 1000);
 
         const doc = await this.localDatabase.getDBEntry(pathSrc, { rev: docEntry._rev });
@@ -1467,33 +1460,40 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
 
     queuedEntries: EntryBody[] = [];
+    dbChangeProcRunning = false;
     handleDBChanged(change: EntryBody) {
-        // If queued same file, cancel previous one.
-        this.queuedEntries.remove(this.queuedEntries.find(e => e._id == change._id));
         // If the file is opened, we have to apply immediately
         const af = app.workspace.getActiveFile();
         if (af && af.path == id2path(change._id)) {
+            this.queuedEntries = this.queuedEntries.filter(e => e._id != change._id);
             return this.handleDBChangedAsync(change);
         }
         this.queuedEntries.push(change);
-        if (this.queuedEntries.length > 50) {
-            clearTrigger("dbchanged");
-            this.execDBchanged();
-        }
-        setTrigger("dbchanged", 500, () => this.execDBchanged());
+        this.execDBchanged();
     }
     async execDBchanged() {
-        await runWithLock("dbchanged", false, async () => {
-            const w = [...this.queuedEntries];
-            this.queuedEntries = [];
-            Logger(`Applying ${w.length} files`);
-            for (const entry of w) {
-                Logger(`Applying ${entry._id} (${entry._rev}) change...`, LOG_LEVEL.VERBOSE);
-                await this.handleDBChangedAsync(entry);
-                Logger(`Applied ${entry._id} (${entry._rev}) change...`);
-            }
+        if (this.dbChangeProcRunning) return false;
+        this.dbChangeProcRunning = true;
+        const semaphore = Semaphore(4);
+        try {
+            do {
+                const entry = this.queuedEntries.shift();
+                // If the same file is to be manipulated, leave it to the last process.
+                if (this.queuedEntries.some(e => e._id == entry._id)) continue;
+                try {
+                    const releaser = await semaphore.acquire(1);
+                    runWithLock(`dbchanged-${entry._id}`, false, async () => {
+                        Logger(`Applying ${entry._id} (${entry._rev}) change...`, LOG_LEVEL.VERBOSE);
+                        await this.handleDBChangedAsync(entry);
+                        Logger(`Applied ${entry._id} (${entry._rev}) change...`);
+                    }).finally(() => { releaser(); });
+                } catch (ex) {
+                    Logger(`Failed to apply the change of ${entry._id} (${entry._rev})`);
+                }
+            } while (this.queuedEntries.length > 0);
+        } finally {
+            this.dbChangeProcRunning = false;
         }
-        );
     }
     async handleDBChangedAsync(change: EntryBody) {
 
@@ -1853,17 +1853,23 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             const processes = e.count;
             const processesDisp = processes == 0 ? "" : ` ⏳${processes}`;
             const message = `Sync: ${w} ↑${sent}${pushLast} ↓${arrived}${pullLast}${waiting}${processesDisp}${queued}`;
-            // const locks = getLocks();
+            function getProcKind(proc: string) {
+                const p = proc.indexOf("-");
+                if (p == -1) {
+                    return proc;
+                }
+                return proc.substring(0, p);
+            }
             const pendingTask = e.pending.length
                 ? "\nPending: " +
-                Object.entries(e.pending.reduce((p, c) => ({ ...p, [c]: (p[c] ?? 0) + 1 }), {} as { [key: string]: number }))
+                Object.entries(e.pending.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
                     .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
                     .join(", ")
                 : "";
 
             const runningTask = e.running.length
                 ? "\nRunning: " +
-                Object.entries(e.running.reduce((p, c) => ({ ...p, [c]: (p[c] ?? 0) + 1 }), {} as { [key: string]: number }))
+                Object.entries(e.running.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
                     .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
                     .join(", ")
                 : "";
@@ -2724,7 +2730,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 Logger(`Reading   : ${file.path}`, LOG_LEVEL.VERBOSE);
                 const contentBin = await this.app.vault.readBinary(file);
                 Logger(`Processing: ${file.path}`, LOG_LEVEL.VERBOSE);
-                content = await arrayBufferToBase64(contentBin);
+                try {
+                    content = await arrayBufferToBase64(contentBin);
+                } catch (ex) {
+                    Logger(`The file ${file.path} could not be encoded`);
+                    Logger(ex, LOG_LEVEL.VERBOSE);
+                    return false;
+                }
                 datatype = "newnote";
             } else {
                 content = await this.app.vault.read(file);
@@ -2732,7 +2744,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             }
         } else {
             if (cache instanceof ArrayBuffer) {
-                content = await arrayBufferToBase64(cache);
+                Logger(`Processing: ${file.path}`, LOG_LEVEL.VERBOSE);
+                try {
+                    content = await arrayBufferToBase64(cache);
+                } catch (ex) {
+                    Logger(`The file ${file.path} could not be encoded`);
+                    Logger(ex, LOG_LEVEL.VERBOSE);
+                    return false;
+                }
                 datatype = "newnote"
             } else {
                 content = cache;
@@ -2752,7 +2771,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         };
         //upsert should locked
         const msg = `DB <- STORAGE (${datatype}) `;
-        const isNotChanged = await runWithLock("file:" + fullPath, false, async () => {
+        const isNotChanged = await runWithLock("file-" + fullPath, false, async () => {
             if (recentlyTouched(file)) {
                 return true;
             }
@@ -3081,7 +3100,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     async storeInternalFileToDatabase(file: InternalFileInfo, forceWrite = false) {
         const id = filename2idInternalMetadata(path2id(file.path));
         const contentBin = await this.app.vault.adapter.readBinary(file.path);
-        const content = await arrayBufferToBase64(contentBin);
+        let content: string[];
+        try {
+            content = await arrayBufferToBase64(contentBin);
+        } catch (ex) {
+            Logger(`The file ${file.path} could not be encoded`);
+            Logger(ex, LOG_LEVEL.VERBOSE);
+            return false;
+        }
         const mtime = file.mtime;
         return await runWithLock("file-" + id, false, async () => {
             try {
@@ -3275,16 +3301,36 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
     async resolveConflictOnInternalFiles() {
         // Scan all conflicted internal files
-        const docs = await this.localDatabase.localDatabase.allDocs({ startkey: ICHeader, endkey: ICHeaderEnd, conflicts: true, include_docs: true });
-        for (const row of docs.rows) {
-            const doc = row.doc;
+        const conflicted = this.localDatabase.findEntries(ICHeader, ICHeaderEnd, { conflicts: true });
+        for await (const doc of conflicted) {
             if (!("_conflicts" in doc)) continue;
-            if (isInternalMetadata(row.id)) {
-                await this.resolveConflictOnInternalFile(row.id);
+            if (isInternalMetadata(doc._id)) {
+                await this.resolveConflictOnInternalFile(doc._id);
             }
         }
     }
 
+    async resolveConflictByNewerEntry(id: string) {
+        const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true });
+        // If there is no conflict, return with false.
+        if (!("_conflicts" in doc)) return false;
+        if (doc._conflicts.length == 0) return false;
+        Logger(`Hidden file conflicted:${id2filenameInternalMetadata(id)}`);
+        const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
+        const revA = doc._rev;
+        const revB = conflicts[0];
+        const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
+        // determine which revision should been deleted.
+        // simply check modified time
+        const mtimeA = ("mtime" in doc && doc.mtime) || 0;
+        const mtimeB = ("mtime" in revBDoc && revBDoc.mtime) || 0;
+        const delRev = mtimeA < mtimeB ? revA : revB;
+        // delete older one.
+        await this.localDatabase.localDatabase.remove(id, delRev);
+        Logger(`Older one has been deleted:${id2filenameInternalMetadata(id)}`);
+        return true;
+    }
+
     async resolveConflictOnInternalFile(id: string): Promise<boolean> {
         try {
             // Retrieve data

updates.md

@@ -54,5 +54,26 @@
     - File names can now be made platform-appropriate.
   - Refactored:
     - Some redundant implementations have been sorted out.
+- 0.17.24
+  - New feature:
+    - If any conflicted files have been left, they will be reported.
+  - Fixed:
+    - Now the name of the conflicting file is shown on the conflict-resolving dialogue.
+    - Hidden files are now able to be merged again.
+    - No longer error caused at plug-in being loaded.
+  - Improved:
+    - Caching chunks are now limited in total size of cached chunks.
+- 0.17.25
+  - Fixed:
+    - Now reading error will be reported.
+- 0.17.26
+  - Fixed(Urgent):
+    - The modified document will be reflected in the storage now.
+- 0.17.27
+  - Improved:
+    - Now, the filename of the conflicted settings will be shown on the merging dialogue
+    - The plugin data can be resolved when conflicted.
+    - The semaphore status display has been changed to count only.
+    - Applying to the storage will be concurrent with a few files.
 
 ... To continue on to `updates_old.md`.