Fixed:

- Rewritten lock acquiring logic.
- Fixed plugin dialog's message.
- Fixed some error messages.
- Fixed action on replicating non-note entries.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.11.0",
+    "version": "0.11.2",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.11.0",
+    "version": "0.11.2",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.11.0",
+            "version": "0.11.2",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.11.0",
+    "version": "0.11.2",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/LocalPouchDB.ts

@@ -253,7 +253,7 @@ export class LocalPouchDB {
                     Logger("Conversion completed!", LOG_LEVEL.NOTICE);
                     old.destroy(); // delete the old database.
                     this.isReady = true;
-                    return nextSeq();
+                    return await nextSeq();
                 } else {
                     throw new Error("Conversion failed!");
                 }
@@ -265,7 +265,7 @@ export class LocalPouchDB {
                 return false;
             }
         } else {
-            return nextSeq();
+            return await nextSeq();
         }
     }
 
@@ -315,7 +315,7 @@ export class LocalPouchDB {
                 this.hashCaches.set(id, w.data);
                 return w.data;
             }
-            throw new Error(`Corrupted chunk detected.`);
+            throw new Error(`Corrupted chunk detected: ${id}`);
         } catch (ex) {
             if (ex.status && ex.status == 404) {
                 if (waitForReady) {
@@ -326,7 +326,7 @@ export class LocalPouchDB {
                     }
                     return this.getDBLeaf(id, false);
                 } else {
-                    throw new Error("Chunk was not found");
+                    throw new Error(`Chunk was not found: ${id}`);
                 }
             } else {
                 Logger(`Something went wrong on retriving chunk`);
@@ -437,7 +437,7 @@ export class LocalPouchDB {
                             Logger(childrens);
                         }
                     } catch (ex) {
-                        Logger(`Something went wrong on reading chunks of ${obj._id} from database:`, LOG_LEVEL.NOTICE);
+                        Logger(`Something went wrong on reading chunks of ${obj._id} from database, see verbose info for detail.`, LOG_LEVEL.NOTICE);
                         Logger(ex, LOG_LEVEL.VERBOSE);
                         this.corruptedEntries[obj._id] = obj;
                         return false;

src/ObsidianLiveSyncSettingTab.ts

@@ -710,6 +710,24 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 })
             );
 
+        new Setting(containerSyncSettingEl)
+            .setName("Skip old files on sync")
+            .setDesc("Skip old incoming if incoming changes older than storage.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.skipOlderFilesOnSync).onChange(async (value) => {
+                    this.plugin.settings.skipOlderFilesOnSync = value;
+                    await this.plugin.saveSettings();
+                })
+            );
+        new Setting(containerSyncSettingEl)
+            .setName("Check conflict only on opening file.")
+            .setDesc("Do not check conflict while replication")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.checkConflictOnlyOnOpen).onChange(async (value) => {
+                    this.plugin.settings.checkConflictOnlyOnOpen = value;
+                    await this.plugin.saveSettings();
+                })
+            );
         containerSyncSettingEl.createEl("div", {
             text: sanitizeHTMLToDom(`Advanced settings<br>
             If you reached the payload size limit when using IBM Cloudant, please set batch size and batch limit to a lower value.`),
@@ -1119,8 +1137,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
 
         const containerCorruptedDataEl = containerEl.createDiv();
 
-        containerCorruptedDataEl.createEl("h3", { text: "Corrupted data" });
-
+        containerCorruptedDataEl.createEl("h3", { text: "Corrupted or missing data" });
+        containerCorruptedDataEl.createEl("h4", { text: "Corrupted" });
         if (Object.keys(this.plugin.localDatabase.corruptedEntries).length > 0) {
             const cx = containerCorruptedDataEl.createEl("div", { text: "If you have copy of these items on any device, simply edit once or twice. Or not, delete this. sorry.." });
             for (const k in this.plugin.localDatabase.corruptedEntries) {
@@ -1149,6 +1167,38 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
         } else {
             containerCorruptedDataEl.createEl("div", { text: "There is no corrupted data." });
         }
+        containerCorruptedDataEl.createEl("h4", { text: "Missing or waiting" });
+        if (Object.keys(this.plugin.queuedFiles).length > 0) {
+            const cx = containerCorruptedDataEl.createEl("div", {
+                text: "These files have missing or waiting chunks. Perhaps almost chunks will be found in a while after replication. But if there're no chunk, you have to restore database entry from existed file by hitting the button below.",
+            });
+            const files = [...new Set([...this.plugin.queuedFiles.map((e) => e.entry._id)])];
+            for (const k of files) {
+                const xx = cx.createEl("div", { text: `${id2path(k)}` });
+
+                const ba = xx.createEl("button", { text: `Delete this` }, (e) => {
+                    e.addEventListener("click", async () => {
+                        await this.plugin.localDatabase.deleteDBEntry(k);
+                        xx.remove();
+                    });
+                });
+                ba.addClass("mod-warning");
+                xx.createEl("button", { text: `Restore from file` }, (e) => {
+                    e.addEventListener("click", async () => {
+                        const f = await this.app.vault.getFiles().filter((e) => path2id(e.path) == k);
+                        if (f.length == 0) {
+                            Logger("Not found in vault", LOG_LEVEL.NOTICE);
+                            return;
+                        }
+                        await this.plugin.updateIntoDB(f[0]);
+                        xx.remove();
+                    });
+                });
+                xx.addClass("mod-warning");
+            }
+        } else {
+            containerCorruptedDataEl.createEl("div", { text: "There is no missing or waiting chunk." });
+        }
         applyDisplayEnabled();
         addScreenElement("70", containerCorruptedDataEl);
         changeDisplay("0");

src/PluginPane.svelte

@@ -16,7 +16,7 @@
     export let plugin: ObsidianLiveSyncPlugin;
     let plugins: PluginDataEntry[] = [];
     let deviceAndPlugins: { [key: string]: PluginDataEntryDisp[] } = {};
-    let devicePluginList: [string, PluginDataEntryDisp[]][] = [];
+    let devicePluginList: [string, PluginDataEntryDisp[]][] = null;
     let ownPlugins: DevicePluginList = null;
     let showOwnPlugins = false;
     let targetList: { [key: string]: boolean } = {};
@@ -205,58 +205,63 @@
                 <th class="sls-plugins-tbl-device-head">Info</th>
                 <th class="sls-plugins-tbl-device-head">Target</th>
             </tr>
-            {#if devicePluginList.length == 0}
+            {#if !devicePluginList}
                 <tr>
                     <td colspan="3" class="sls-table-tail tcenter"> Retrieving... </td>
                 </tr>
-            {/if}
-            {#each devicePluginList as [deviceName, devicePlugins]}
+            {:else if devicePluginList.length == 0}
                 <tr>
-                    <th colspan="2" class="sls-plugins-tbl-device-head">{deviceName}</th>
-                    <th class="sls-plugins-tbl-device-head">
-                        <button class="mod-cta" on:click={() => toggleAll(deviceName)}>✔</button>
-                    </th>
+                    <td colspan="3" class="sls-table-tail tcenter"> No plugins found. </td>
                 </tr>
-                {#each devicePlugins as plugin}
+            {:else}
+                {#each devicePluginList as [deviceName, devicePlugins]}
                     <tr>
-                        <td class="sls-table-head">{plugin.manifest.name}</td>
-                        <td class="sls-table-tail tcenter">{plugin.versionInfo}{getDispString(plugin.versionFlag)}</td>
-                        <td class="sls-table-tail tcenter">
-                            {#if plugin.versionFlag === "EVEN" || plugin.versionFlag === ""}
-                                -
-                            {:else}
-                                <div class="wrapToggle">
-                                    <div
-                                        class="checkbox-container"
-                                        class:is-enabled={targetList[plugin.deviceVaultName + "---" + plugin.manifest.id + "---plugin"]}
-                                        on:click={() => toggleTarget(plugin.deviceVaultName + "---" + plugin.manifest.id + "---plugin")}
-                                    />
-                                </div>
-                            {/if}
-                        </td>
-                    </tr>
-                    <tr>
-                        <td class="sls-table-head">Settings</td>
-                        <td class="sls-table-tail tcenter">{plugin.mtimeInfo}{getDispString(plugin.mtimeFlag)}</td>
-                        <td class="sls-table-tail tcenter">
-                            {#if plugin.mtimeFlag === "EVEN" || plugin.mtimeFlag === ""}
-                                -
-                            {:else}
-                                <div class="wrapToggle">
-                                    <div
-                                        class="checkbox-container"
-                                        class:is-enabled={targetList[plugin.deviceVaultName + "---" + plugin.manifest.id + "---setting"]}
-                                        on:click={() => toggleTarget(plugin.deviceVaultName + "---" + plugin.manifest.id + "---setting")}
-                                    />
-                                </div>
-                            {/if}
-                        </td>
-                    </tr>
-                    <tr class="divider">
-                        <th colspan="3" />
+                        <th colspan="2" class="sls-plugins-tbl-device-head">{deviceName}</th>
+                        <th class="sls-plugins-tbl-device-head">
+                            <button class="mod-cta" on:click={() => toggleAll(deviceName)}>✔</button>
+                        </th>
                     </tr>
+                    {#each devicePlugins as plugin}
+                        <tr>
+                            <td class="sls-table-head">{plugin.manifest.name}</td>
+                            <td class="sls-table-tail tcenter">{plugin.versionInfo}{getDispString(plugin.versionFlag)}</td>
+                            <td class="sls-table-tail tcenter">
+                                {#if plugin.versionFlag === "EVEN" || plugin.versionFlag === ""}
+                                    -
+                                {:else}
+                                    <div class="wrapToggle">
+                                        <div
+                                            class="checkbox-container"
+                                            class:is-enabled={targetList[plugin.deviceVaultName + "---" + plugin.manifest.id + "---plugin"]}
+                                            on:click={() => toggleTarget(plugin.deviceVaultName + "---" + plugin.manifest.id + "---plugin")}
+                                        />
+                                    </div>
+                                {/if}
+                            </td>
+                        </tr>
+                        <tr>
+                            <td class="sls-table-head">Settings</td>
+                            <td class="sls-table-tail tcenter">{plugin.mtimeInfo}{getDispString(plugin.mtimeFlag)}</td>
+                            <td class="sls-table-tail tcenter">
+                                {#if plugin.mtimeFlag === "EVEN" || plugin.mtimeFlag === ""}
+                                    -
+                                {:else}
+                                    <div class="wrapToggle">
+                                        <div
+                                            class="checkbox-container"
+                                            class:is-enabled={targetList[plugin.deviceVaultName + "---" + plugin.manifest.id + "---setting"]}
+                                            on:click={() => toggleTarget(plugin.deviceVaultName + "---" + plugin.manifest.id + "---setting")}
+                                        />
+                                    </div>
+                                {/if}
+                            </td>
+                        </tr>
+                        <tr class="divider">
+                            <th colspan="3" />
+                        </tr>
+                    {/each}
                 {/each}
-            {/each}
+            {/if}
         </table>
     </div>
     <div class="ols-plugins-div-buttons">

src/main.ts

@@ -923,12 +923,120 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             const doc = change;
             const file = targetFile;
             await this.doc2storage_modify(doc, file);
-            this.queueConflictedCheck(file);
+            if (!this.settings.checkConflictOnlyOnOpen) {
+                this.queueConflictedCheck(file);
+            } else {
+                const af = app.workspace.getActiveFile();
+                if (af && af.path == file.path) {
+                    this.queueConflictedCheck(file);
+                }
+            }
         } else {
             Logger(`${id2path(change._id)} is already exist as the folder`);
         }
     }
 
+    queuedFiles: {
+        entry: EntryBody;
+        missingChildren: string[];
+        timeout?: number;
+        done?: boolean;
+        warned?: boolean;
+    }[] = [];
+    chunkWaitTimeout = 60000;
+
+    async saveQueuedFiles() {
+        const saveData = JSON.stringify(this.queuedFiles.filter((e) => !e.done).map((e) => e.entry._id));
+        const lsname = "obsidian-livesync-queuefiles-" + this.app.vault.getName();
+        localStorage.setItem(lsname, saveData);
+    }
+    async loadQueuedFiles() {
+        const lsname = "obsidian-livesync-queuefiles-" + this.app.vault.getName();
+        const ids = JSON.parse(localStorage.getItem(lsname) || "[]") as string[];
+        const ret = await this.localDatabase.localDatabase.allDocs({ keys: ids, include_docs: true });
+        for (const doc of ret.rows) {
+            if (doc.doc && !this.queuedFiles.some((e) => e.entry._id == doc.doc._id)) {
+                await this.parseIncomingDoc(doc.doc as PouchDB.Core.ExistingDocument<EntryBody & PouchDB.Core.AllDocsMeta>);
+            }
+        }
+    }
+    async procQueuedFiles() {
+        await runWithLock("procQueue", true, async () => {
+            this.saveQueuedFiles();
+            for (const queue of this.queuedFiles) {
+                if (queue.done) continue;
+                const now = new Date().getTime();
+                if (queue.missingChildren.length == 0) {
+                    queue.done = true;
+                    if (isValidPath(id2path(queue.entry._id))) {
+                        Logger(`Applying ${queue.entry._id} (${queue.entry._rev}) change...`);
+                        await this.handleDBChanged(queue.entry);
+                    }
+                }
+                if (now > queue.timeout) {
+                    if (!queue.warned) Logger(`Timed out: ${queue.entry._id} could not collect ${queue.missingChildren.length} chunks. plugin keeps watching, but you have to check the file after the replication.`, LOG_LEVEL.NOTICE);
+                    queue.warned = true;
+                    continue;
+                }
+            }
+            this.queuedFiles = this.queuedFiles.filter((e) => !e.done);
+            this.saveQueuedFiles();
+        });
+    }
+    parseIncomingChunk(chunk: PouchDB.Core.ExistingDocument<EntryDoc>) {
+        const now = new Date().getTime();
+        let isNewFileCompleted = false;
+
+        for (const queue of this.queuedFiles) {
+            if (queue.done) continue;
+            if (queue.missingChildren.indexOf(chunk._id) !== -1) {
+                queue.missingChildren = queue.missingChildren.filter((e) => e != chunk._id);
+                queue.timeout = now + this.chunkWaitTimeout;
+            }
+            if (queue.missingChildren.length == 0) {
+                for (const e of this.queuedFiles) {
+                    if (e.entry._id == queue.entry._id && e.entry.mtime < queue.entry.mtime) {
+                        e.done = true;
+                    }
+                }
+                isNewFileCompleted = true;
+            }
+        }
+        if (isNewFileCompleted) this.procQueuedFiles();
+    }
+    async parseIncomingDoc(doc: PouchDB.Core.ExistingDocument<EntryBody>) {
+        const skipOldFile = this.settings.skipOlderFilesOnSync;
+        if (skipOldFile) {
+            const info = this.app.vault.getAbstractFileByPath(id2path(doc._id));
+
+            if (info && info instanceof TFile) {
+                const localMtime = ~~((info as TFile).stat.mtime / 1000);
+                const docMtime = ~~(doc.mtime / 1000);
+                if (localMtime >= docMtime) {
+                    Logger(`${doc._id} Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
+                    return;
+                }
+            }
+        }
+        const now = new Date().getTime();
+        const newQueue = {
+            entry: doc,
+            missingChildren: [] as string[],
+            timeout: now + this.chunkWaitTimeout,
+        };
+        if ("children" in doc) {
+            const c = await this.localDatabase.localDatabase.allDocs({ keys: doc.children, include_docs: false });
+            const missing = c.rows.filter((e) => "error" in e).map((e) => e.key);
+            if (missing.length) Logger(`${doc._id}(${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
+            newQueue.missingChildren = missing;
+            this.queuedFiles.push(newQueue);
+            this.saveQueuedFiles();
+        } else {
+            this.queuedFiles.push(newQueue);
+            this.saveQueuedFiles();
+            this.procQueuedFiles();
+        }
+    }
     periodicSyncHandler: number = null;
 
     //---> Sync
@@ -942,14 +1050,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 continue;
             }
             if (change._id.startsWith("h:")) {
+                await this.parseIncomingChunk(change);
                 continue;
             }
             if (change._id == SYNCINFO_ID) {
                 continue;
             }
             if (change.type != "leaf" && change.type != "versioninfo" && change.type != "milestoneinfo" && change.type != "nodeinfo") {
-                Logger("replication change arrived", LOG_LEVEL.VERBOSE);
-                await this.handleDBChanged(change);
+                await this.parseIncomingDoc(change);
+                continue;
             }
             if (change.type == "versioninfo") {
                 if (change.version > VER) {
@@ -1086,9 +1195,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             waiting = " " + this.batchFileChange.map((e) => "🛫").join("");
             waiting = waiting.replace(/(🛫){10}/g, "🚀");
         }
+        let queued = "";
+        const queue = Object.entries(this.queuedFiles).filter((e) => !e[1].warned);
+        const queuedCount = queue.length;
+
+        if (queuedCount) {
+            const pieces = queue.map((e) => e[1].missingChildren).reduce((prev, cur) => prev + cur.length, 0);
+            queued = ` 🧩 ${queuedCount} (${pieces})`;
+        }
         const procs = getProcessingCounts();
         const procsDisp = procs == 0 ? "" : ` ⏳${procs}`;
-        const message = `Sync:${w} ↑${sent} ↓${arrived}${waiting}${procsDisp}`;
+        const message = `Sync:${w} ↑${sent} ↓${arrived}${waiting}${procsDisp}${queued}`;
         const locks = getLocks();
         const pendingTask = locks.pending.length ? `\nPending:${locks.pending.join(", ")}` : "";
         const runningTask = locks.running.length ? `\nRunning:${locks.running.join(", ")}` : "";
@@ -1129,6 +1246,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (this.settings.autoSweepPlugins) {
             await this.sweepPlugin(false);
         }
+        await this.loadQueuedFiles();
         this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
     }
 
@@ -1168,6 +1286,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (showingNotice) {
             notice = NewNotice("Initializing", 0);
         }
+
         const filesStorage = this.app.vault.getFiles();
         const filesStorageName = filesStorage.map((e) => e.path);
         const wf = await this.localDatabase.localDatabase.allDocs();
@@ -1555,6 +1674,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         });
         if (isNotChanged) return;
         await this.localDatabase.putDBEntry(d);
+        this.queuedFiles = this.queuedFiles.map((e) => ({ ...e, ...(e.entry._id == d._id ? { done: true } : {}) }));
 
         Logger("put database:" + fullpath + "(" + datatype + ") ");
         if (this.settings.syncOnSave && !this.suspended) {
@@ -1615,7 +1735,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
     async sweepPlugin(showMessage = false) {
         if (!this.settings.usePluginSync) return;
-        await runWithLock("sweepplugin", false, async () => {
+        if (!this.localDatabase.isReady) return;
+        await runWithLock("sweepplugin", true, async () => {
             const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
             if (!this.settings.encrypt) {
                 Logger("You have to encrypt the database to use plugin setting sync.", LOG_LEVEL.NOTICE);
@@ -1632,7 +1753,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                 endkey: `ps:${this.deviceAndVaultName}.`,
                 include_docs: true,
             });
-            Logger("OLD DOCS.", LOG_LEVEL.VERBOSE);
+            // Logger("OLD DOCS.", LOG_LEVEL.VERBOSE);
             // sweep current plugin.
             // @ts-ignore
             const pl = this.app.plugins;

src/utils_couchdb.ts

@@ -134,6 +134,7 @@ const connectRemoteCouchDB = async (uri: string, auth: { username: string; passw
             // return await fetch(url, opts);
         },
     };
+
     const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
     if (passphrase && typeof passphrase === "string") {
         enableEncryption(db, passphrase);