bump

- Some messages have been refined.
- Boot sequence has been speeded up.
- Opening the local database multiple times in a short duration has been suppressed.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.17.26",
+    "version": "0.17.28",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.17.26",
+    "version": "0.17.28",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.17.26",
+            "version": "0.17.28",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.17.26",
+    "version": "0.17.28",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/JsonResolvePane.svelte

@@ -3,7 +3,7 @@
     import type { LoadedEntry } from "./lib/src/types";
     import { base64ToString } from "./lib/src/strbin";
     import { getDocData } from "./lib/src/utils";
-    import { mergeObject } from "./utils";
+    import { id2path, mergeObject } from "./utils";
 
     export let docs: LoadedEntry[] = [];
     export let callback: (keepRev: string, mergedStr?: string) => Promise<void> = async (_, __) => {
@@ -93,9 +93,11 @@
         diffs = getJsonDiff(objA, selectedObj);
         console.dir(selectedObj);
     }
+    $: filename = id2path(docA?._id ?? "");
 </script>
 
-<h1>File Conflicted</h1>
+<h1>Conflicted settings</h1>
+<div><span>{filename}</span></div>
 {#if !docA || !docB}
     <div class="message">Just for a minute, please!</div>
     <div class="buttons">

src/main.ts

@@ -64,7 +64,6 @@ function filename2idInternalMetadata(str: string): string {
 }
 
 const CHeader = "h:";
-const CHeaderEnd = "h;";
 // const CHeaderLength = CHeader.length;
 function isChunk(str: string): boolean {
     return str.startsWith(CHeader);
@@ -221,40 +220,32 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         }
         const target = await askSelectString(this.app, "File to view History", notesList);
         if (target) {
-            if (isInternalMetadata(target)) {
-                //NOP
-                await this.resolveConflictOnInternalFile(target);
-            } else {
-                await this.showIfConflicted(target);
-            }
+            await this.resolveConflicted(target);
+        }
+    }
+    async resolveConflicted(target: string) {
+        if (isInternalMetadata(target)) {
+            await this.resolveConflictOnInternalFile(target);
+        } else if (isPluginMetadata(target)) {
+            await this.resolveConflictByNewerEntry(target);
+        } else {
+            await this.showIfConflicted(target);
         }
     }
 
     async collectDeletedFiles() {
-        const pageLimit = 1000;
-        let nextKey = "";
         const limitDays = this.settings.automaticallyDeleteMetadataOfDeletedFiles;
         if (limitDays <= 0) return;
         Logger(`Checking expired file history`);
         const limit = Date.now() - (86400 * 1000 * limitDays);
         const notes: { path: string, mtime: number, ttl: number, doc: PouchDB.Core.ExistingDocument<EntryDoc & PouchDB.Core.AllDocsMeta> }[] = [];
-        do {
-            const docs = await this.localDatabase.localDatabase.allDocs({ limit: pageLimit, startkey: nextKey, conflicts: true, include_docs: true });
-            nextKey = "";
-            for (const row of docs.rows) {
-                const doc = row.doc;
-                nextKey = `${row.id}\u{10ffff}`;
-                if (doc.type == "newnote" || doc.type == "plain") {
-                    if (doc.deleted && (doc.mtime - limit) < 0) {
-                        notes.push({ path: id2path(doc._id), mtime: doc.mtime, ttl: (doc.mtime - limit) / 1000 / 86400, doc: doc });
-                    }
-                }
-                if (isChunk(nextKey)) {
-                    // skip the chunk zone.
-                    nextKey = CHeaderEnd;
+        for await (const doc of this.localDatabase.findAllDocs({ conflicts: true })) {
+            if (doc.type == "newnote" || doc.type == "plain") {
+                if (doc.deleted && (doc.mtime - limit) < 0) {
+                    notes.push({ path: id2path(doc._id), mtime: doc.mtime, ttl: (doc.mtime - limit) / 1000 / 86400, doc: doc });
                 }
             }
-        } while (nextKey != "");
+        }
         if (notes.length == 0) {
             Logger("There are no old documents");
             Logger(`Checking expired file history done`);
@@ -327,7 +318,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                     if (this.settings.suspendFileWatching) {
                         Logger("'Suspend file watching' turned on. Are you sure this is what you intended? Every modification on the vault will be ignored.", LOG_LEVEL.NOTICE);
                     }
-                    const isInitialized = await this.initializeDatabase();
+                    const isInitialized = await this.initializeDatabase(false, false);
                     if (!isInitialized) {
                         //TODO:stop all sync.
                         return false;
@@ -431,7 +422,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                         this.settings = newSettingW;
                         this.usedPassphrase = "";
                         await this.saveSettings();
-                        await this.resetLocalOldDatabase();
                         await this.resetLocalDatabase();
                         await this.localDatabase.initializeDatabase();
                         await this.markRemoteResolved();
@@ -444,7 +434,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                         this.settings = newSettingW;
                         this.usedPassphrase = "";
                         await this.saveSettings();
-                        await this.resetLocalOldDatabase();
                         await this.resetLocalDatabase();
                         await this.localDatabase.initializeDatabase();
                         await this.initializeDatabase(true);
@@ -482,7 +471,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                         this.usedPassphrase = "";
                         await this.saveSettings();
                         if (keepLocalDB == "no") {
-                            this.resetLocalOldDatabase();
                             this.resetLocalDatabase();
                             this.localDatabase.initializeDatabase();
                             const rebuild = await askYesNo(this.app, "Rebuild the database?");
@@ -531,6 +519,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         const lsKey = "obsidian-live-sync-ver" + this.getVaultName();
         const last_version = localStorage.getItem(lsKey);
         await this.loadSettings();
+
         const lastVersion = ~~(versionNumberString2Number(manifestVersion) / 1000);
         if (lastVersion > this.settings.lastReadUpdates) {
             Logger("Self-hosted LiveSync has undergone a major upgrade. Please open the setting dialog, and check the information pane.", LOG_LEVEL.NOTICE);
@@ -606,7 +595,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         });
 
         this.addSettingTab(new ObsidianLiveSyncSettingTab(this.app, this));
-
         this.app.workspace.onLayoutReady(this.onLayoutReady.bind(this));
         this.registerObsidianProtocolHandler("setuplivesync", async (conf: any) => await this.setupWizard(conf.settings));
 
@@ -782,7 +770,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             await this.localDatabase.close();
         }
         const vaultName = this.getVaultName();
-        Logger("Open Database...");
+        Logger("Waiting for ready...");
         //@ts-ignore
         const isMobile = this.app.isMobile;
         this.localDatabase = new LocalPouchDB(this.settings, vaultName, isMobile);
@@ -1315,7 +1303,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             this.app.vault.adapter.append(normalizePath(logDate), vaultName + ":" + newMessage + "\n");
         }
         logMessageStore.apply(e => [...e, newMessage].slice(-100));
-        this.setStatusBarText(null, messageContent.substring(0, 30));
+        this.setStatusBarText(null, messageContent);
 
         if (level >= LOG_LEVEL.NOTICE) {
             if (!key) key = messageContent;
@@ -1457,33 +1445,40 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
 
     queuedEntries: EntryBody[] = [];
+    dbChangeProcRunning = false;
     handleDBChanged(change: EntryBody) {
-        // If queued same file, cancel previous one.
-        this.queuedEntries.remove(this.queuedEntries.find(e => e._id == change._id));
         // If the file is opened, we have to apply immediately
         const af = app.workspace.getActiveFile();
         if (af && af.path == id2path(change._id)) {
+            this.queuedEntries = this.queuedEntries.filter(e => e._id != change._id);
             return this.handleDBChangedAsync(change);
         }
         this.queuedEntries.push(change);
-        if (this.queuedEntries.length > 50) {
-            clearTrigger("dbchanged");
-            this.execDBchanged();
-        }
-        setTrigger("dbchanged", 500, () => this.execDBchanged());
+        this.execDBchanged();
     }
     async execDBchanged() {
-        await runWithLock("dbchanged", false, async () => {
-            const w = [...this.queuedEntries];
-            this.queuedEntries = [];
-            Logger(`Applying ${w.length} files`);
-            for (const entry of w) {
-                Logger(`Applying ${entry._id} (${entry._rev}) change...`, LOG_LEVEL.VERBOSE);
-                await this.handleDBChangedAsync(entry);
-                Logger(`Applied ${entry._id} (${entry._rev}) change...`);
-            }
+        if (this.dbChangeProcRunning) return false;
+        this.dbChangeProcRunning = true;
+        const semaphore = Semaphore(4);
+        try {
+            do {
+                const entry = this.queuedEntries.shift();
+                // If the same file is to be manipulated, leave it to the last process.
+                if (this.queuedEntries.some(e => e._id == entry._id)) continue;
+                try {
+                    const releaser = await semaphore.acquire(1);
+                    runWithLock(`dbchanged-${entry._id}`, false, async () => {
+                        Logger(`Applying ${entry._id} (${entry._rev}) change...`, LOG_LEVEL.VERBOSE);
+                        await this.handleDBChangedAsync(entry);
+                        Logger(`Applied ${entry._id} (${entry._rev}) change...`);
+                    }).finally(() => { releaser(); });
+                } catch (ex) {
+                    Logger(`Failed to apply the change of ${entry._id} (${entry._rev})`);
+                }
+            } while (this.queuedEntries.length > 0);
+        } finally {
+            this.dbChangeProcRunning = false;
         }
-        );
     }
     async handleDBChangedAsync(change: EntryBody) {
 
@@ -1843,17 +1838,23 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             const processes = e.count;
             const processesDisp = processes == 0 ? "" : ` ⏳${processes}`;
             const message = `Sync: ${w} ↑${sent}${pushLast} ↓${arrived}${pullLast}${waiting}${processesDisp}${queued}`;
-            // const locks = getLocks();
+            function getProcKind(proc: string) {
+                const p = proc.indexOf("-");
+                if (p == -1) {
+                    return proc;
+                }
+                return proc.substring(0, p);
+            }
             const pendingTask = e.pending.length
                 ? "\nPending: " +
-                Object.entries(e.pending.reduce((p, c) => ({ ...p, [c]: (p[c] ?? 0) + 1 }), {} as { [key: string]: number }))
+                Object.entries(e.pending.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
                     .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
                     .join(", ")
                 : "";
 
             const runningTask = e.running.length
                 ? "\nRunning: " +
-                Object.entries(e.running.reduce((p, c) => ({ ...p, [c]: (p[c] ?? 0) + 1 }), {} as { [key: string]: number }))
+                Object.entries(e.running.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
                     .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
                     .join(", ")
                 : "";
@@ -1909,9 +1910,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         await this.localDatabase.openReplication(this.settings, false, showMessage, this.parseReplicationResult);
     }
 
-    async initializeDatabase(showingNotice?: boolean) {
+    async initializeDatabase(showingNotice?: boolean, reopenDatabase = true) {
         this.isReady = false;
-        if (await this.openDatabase()) {
+        if ((!reopenDatabase) || await this.openDatabase()) {
             if (this.localDatabase.isReady) {
                 await this.syncAllFiles(showingNotice);
             }
@@ -1973,18 +1974,22 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             Logger("Initializing", LOG_LEVEL.NOTICE, "syncAll");
         }
 
+        Logger("Initialize and checking database files");
+        Logger("Checking deleted files");
         await this.collectDeletedFiles();
 
+        Logger("Collecting local files on the storage", LOG_LEVEL.VERBOSE);
         const filesStorage = this.app.vault.getFiles().filter(e => this.isTargetFile(e));
         const filesStorageName = filesStorage.map((e) => e.path);
-        const wf = await this.localDatabase.localDatabase.allDocs();
-        const filesDatabase = wf.rows.filter((e) =>
-            !isChunk(e.id) &&
-            !isPluginMetadata(e.id) &&
-            e.id != "obsydian_livesync_version" &&
-            e.id != "_design/replicate"
-        )
-            .filter(e => isValidPath(e.id)).map((e) => id2path(e.id)).filter(e => this.isTargetFile(e));
+        Logger("Collecting local files on the DB", LOG_LEVEL.VERBOSE);
+        const filesDatabase = [] as string[]
+        for await (const doc of this.localDatabase.findAllDocs()) {
+            const path = id2path(doc._id);
+            if (isValidPath(doc._id) && this.isTargetFile(path)) {
+                filesDatabase.push(path);
+            }
+        }
+        Logger("Opening the key-value database", LOG_LEVEL.VERBOSE);
         const isInitialized = await (this.localDatabase.kvDB.get<boolean>("initialized")) || false;
         // Make chunk bigger if it is the initial scan. There must be non-active docs.
         if (filesDatabase.length == 0 && !isInitialized) {
@@ -1997,7 +2002,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         const onlyInStorageNames = onlyInStorage.map((e) => e.path);
 
         const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1);
-        Logger("Initialize and checking database files");
         Logger("Updating database by new files");
         this.setStatusBarText(`UPDATE DATABASE`);
 
@@ -2755,7 +2759,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         };
         //upsert should locked
         const msg = `DB <- STORAGE (${datatype}) `;
-        const isNotChanged = await runWithLock("file:" + fullPath, false, async () => {
+        const isNotChanged = await runWithLock("file-" + fullPath, false, async () => {
             if (recentlyTouched(file)) {
                 return true;
             }
@@ -2811,11 +2815,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     async resetLocalDatabase() {
         clearTouched();
         await this.localDatabase.resetDatabase();
-        await this.localDatabase.resetLocalOldDatabase();
-    }
-    async resetLocalOldDatabase() {
-        clearTouched();
-        await this.localDatabase.resetLocalOldDatabase();
     }
 
     async tryResetRemoteDatabase() {
@@ -3285,16 +3284,36 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
 
     async resolveConflictOnInternalFiles() {
         // Scan all conflicted internal files
-        const docs = await this.localDatabase.localDatabase.allDocs({ startkey: ICHeader, endkey: ICHeaderEnd, conflicts: true, include_docs: true });
-        for (const row of docs.rows) {
-            const doc = row.doc;
+        const conflicted = this.localDatabase.findEntries(ICHeader, ICHeaderEnd, { conflicts: true });
+        for await (const doc of conflicted) {
             if (!("_conflicts" in doc)) continue;
-            if (isInternalMetadata(row.id)) {
-                await this.resolveConflictOnInternalFile(row.id);
+            if (isInternalMetadata(doc._id)) {
+                await this.resolveConflictOnInternalFile(doc._id);
             }
         }
     }
 
+    async resolveConflictByNewerEntry(id: string) {
+        const doc = await this.localDatabase.localDatabase.get(id, { conflicts: true });
+        // If there is no conflict, return with false.
+        if (!("_conflicts" in doc)) return false;
+        if (doc._conflicts.length == 0) return false;
+        Logger(`Hidden file conflicted:${id2filenameInternalMetadata(id)}`);
+        const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
+        const revA = doc._rev;
+        const revB = conflicts[0];
+        const revBDoc = await this.localDatabase.localDatabase.get(id, { rev: revB });
+        // determine which revision should been deleted.
+        // simply check modified time
+        const mtimeA = ("mtime" in doc && doc.mtime) || 0;
+        const mtimeB = ("mtime" in revBDoc && revBDoc.mtime) || 0;
+        const delRev = mtimeA < mtimeB ? revA : revB;
+        // delete older one.
+        await this.localDatabase.localDatabase.remove(id, delRev);
+        Logger(`Older one has been deleted:${id2filenameInternalMetadata(id)}`);
+        return true;
+    }
+
     async resolveConflictOnInternalFile(id: string): Promise<boolean> {
         try {
             // Retrieve data

updates.md

@@ -69,5 +69,18 @@
 - 0.17.26
   - Fixed(Urgent):
     - The modified document will be reflected in the storage now.
+- 0.17.27
+  - Improved:
+    - Now, the filename of the conflicted settings will be shown on the merging dialogue
+    - The plugin data can be resolved when conflicted.
+    - The semaphore status display has been changed to count only.
+    - Applying to the storage will be concurrent with a few files.
+- 0.17.28
+  -Fixed:
+    - Some messages have been refined.
+    - Boot sequence has been speeded up.
+    - Opening the local database multiple times in a short duration has been suppressed.
+    - Older migration logic. 
+      - Note: If you have used 0.10.0 or lower and have not upgraded, you will need to run 0.17.27 or earlier once or reinstall Obsidian.
 
 ... To continue on to `updates_old.md`.