bump

- No longer missing tasks which have queued as the same key (e.g., for the same operation to the same file).
- Some trivial issues have been fixed.
New feature:
- Reloading Obsidian can be scheduled until that file and database operations are stable.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.23.6",
+    "version": "0.23.7",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.23.6",
+    "version": "0.23.7",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.23.6",
+            "version": "0.23.7",
             "license": "MIT",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.556.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.23.6",
+    "version": "0.23.7",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/features/CmdConfigSync.ts

@@ -10,7 +10,7 @@ import { readString, decodeBinary, arrayBufferToBase64, digestHash } from "../li
 import { serialized } from "../lib/src/concurrency/lock.ts";
 import { LiveSyncCommands } from "./LiveSyncCommands.ts";
 import { stripAllPrefixes } from "../lib/src/string_and_binary/path.ts";
-import { PeriodicProcessor, askYesNo, disposeMemoObject, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask } from "../common/utils.ts";
+import { PeriodicProcessor, disposeMemoObject, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask } from "../common/utils.ts";
 import { PluginDialogModal } from "../common/dialogs.ts";
 import { JsonResolveModal } from "../ui/JsonResolveModal.ts";
 import { QueueProcessor } from '../lib/src/concurrency/processor.ts';
@@ -466,12 +466,7 @@ export class ConfigSync extends LiveSyncCommands {
                     Logger(`Plugin reloaded: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id);
                 }
             } else if (data.category == "CONFIG") {
-                scheduleTask("configReload", 250, async () => {
+                this.plugin.askReload();
-                    if (await askYesNo(this.app, "Do you want to restart and reload Obsidian now?") == "yes") {
-                        // @ts-ignore
-                        this.app.commands.executeCommandById("app:reload")
-                    }
-                })
             }
             return true;
         } catch (ex) {

src/features/CmdHiddenFileSync.ts

@@ -432,13 +432,14 @@ export class HiddenFileSync extends LiveSyncCommands {
 
                 // If something changes left, notify for reloading Obsidian.
                 if (updatedCount != 0) {
-                    this.plugin.askInPopup(`updated-any-hidden`, `Hidden files have been synchronized, Press  {HERE} to reload Obsidian, or press elsewhere to dismiss this message.`, (anchor) => {
+                    if (!this.plugin.isReloadingScheduled) {
-                        anchor.text = "HERE";
+                        this.plugin.askInPopup(`updated-any-hidden`, `Hidden files have been synchronised, Press {HERE} to schedule a reload of Obsidian, or press elsewhere to dismiss this message.`, (anchor) => {
-                        anchor.addEventListener("click", () => {
+                            anchor.text = "HERE";
-                            // @ts-ignore
+                            anchor.addEventListener("click", () => {
-                            this.app.commands.executeCommandById("app:reload");
+                                this.plugin.scheduleAppReload();
+                            });
                         });
-                    });
+                    }
                 }
             }
         }
@@ -471,6 +472,7 @@ export class HiddenFileSync extends LiveSyncCommands {
                         children: [],
                         deleted: false,
                         type: "newnote",
+                        eden: {},
                     };
                 } else {
                     if (await isDocContentSame(readAsBlob(old), content) && !forceWrite) {
@@ -521,6 +523,7 @@ export class HiddenFileSync extends LiveSyncCommands {
                         children: [],
                         deleted: true,
                         type: "newnote",
+                        eden: {}
                     };
                 } else {
                     // Remove all conflicted before deleting.

src/main.ts

@@ -32,7 +32,7 @@ import { LogPaneView, VIEW_TYPE_LOG } from "./ui/LogPaneView.ts";
 import { LRUCache } from "./lib/src/memory/LRUCache.ts";
 import { SerializedFileAccess } from "./storages/SerializedFileAccess.js";
 import { QueueProcessor } from "./lib/src/concurrency/processor.js";
-import { reactive, reactiveSource } from "./lib/src/dataobject/reactive.js";
+import { reactive, reactiveSource, type ReactiveValue } from "./lib/src/dataobject/reactive.js";
 import { initializeStores } from "./common/stores.js";
 import { JournalSyncMinio } from "./lib/src/replication/journal/objectstore/JournalSyncMinio.js";
 import { LiveSyncJournalReplicator, type LiveSyncJournalReplicatorEnv } from "./lib/src/replication/journal/LiveSyncJournalReplicator.js";
@@ -1422,55 +1422,62 @@ We can perform a command in this file.
     }
     async handleFileEvent(queue: FileEventItem): Promise<any> {
         const file = queue.args.file;
-        const key = `file-last-proc-${queue.type}-${file.path}`;
+        const lockKey = `handleFile:${file.path}`;
-        const last = Number(await this.kvDB.get(key) || 0);
+        return await serialized(lockKey, async () => {
-        let mtime = file.mtime;
+            const key = `file-last-proc-${queue.type}-${file.path}`;
-        if (queue.type == "DELETE") {
+            const last = Number(await this.kvDB.get(key) || 0);
-            await this.deleteFromDBbyPath(file.path);
+            let mtime = file.mtime;
-            mtime = file.mtime - 1;
+            if (queue.type == "DELETE") {
-            const keyD1 = `file-last-proc-CREATE-${file.path}`;
+                await this.deleteFromDBbyPath(file.path);
-            const keyD2 = `file-last-proc-CHANGED-${file.path}`;
+                mtime = file.mtime - 1;
-            await this.kvDB.set(keyD1, mtime);
+                const keyD1 = `file-last-proc-CREATE-${file.path}`;
-            await this.kvDB.set(keyD2, mtime);
+                const keyD2 = `file-last-proc-CHANGED-${file.path}`;
-        } else if (queue.type == "INTERNAL") {
-            await this.addOnHiddenFileSync.watchVaultRawEventsAsync(file.path);
-            await this.addOnConfigSync.watchVaultRawEventsAsync(file.path);
-        } else {
-            const targetFile = this.vaultAccess.getAbstractFileByPath(file.path);
-            if (!(targetFile instanceof TFile)) {
-                Logger(`Target file was not found: ${file.path}`, LOG_LEVEL_INFO);
-                return;
-            }
-            if (file.mtime == last) {
-                Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL_VERBOSE);
-                return;
-            }
-
-            // const cache = queue.args.cache;
-            if (queue.type == "CREATE" || queue.type == "CHANGED") {
-                fireAndForget(() => this.checkAndApplySettingFromMarkdown(queue.args.file.path, true));
-                const keyD1 = `file-last-proc-DELETED-${file.path}`;
                 await this.kvDB.set(keyD1, mtime);
-                if (!await this.updateIntoDB(targetFile, undefined)) {
+                await this.kvDB.set(keyD2, mtime);
-                    Logger(`STORAGE -> DB: failed, cancel the relative operations: ${targetFile.path}`, LOG_LEVEL_INFO);
+            } else if (queue.type == "INTERNAL") {
-                    // cancel running queues and remove one of atomic operation
+                await this.addOnHiddenFileSync.watchVaultRawEventsAsync(file.path);
-                    this.cancelRelativeEvent(queue);
+                await this.addOnConfigSync.watchVaultRawEventsAsync(file.path);
+            } else {
+                const targetFile = this.vaultAccess.getAbstractFileByPath(file.path);
+                if (!(targetFile instanceof TFile)) {
+                    Logger(`Target file was not found: ${file.path}`, LOG_LEVEL_INFO);
                     return;
                 }
+                if (file.mtime == last) {
+                    Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL_VERBOSE);
+                    return;
+                }
+
+                // const cache = queue.args.cache;
+                if (queue.type == "CREATE" || queue.type == "CHANGED") {
+                    fireAndForget(() => this.checkAndApplySettingFromMarkdown(queue.args.file.path, true));
+                    const keyD1 = `file-last-proc-DELETED-${file.path}`;
+                    await this.kvDB.set(keyD1, mtime);
+                    if (!await this.updateIntoDB(targetFile, undefined)) {
+                        Logger(`STORAGE -> DB: failed, cancel the relative operations: ${targetFile.path}`, LOG_LEVEL_INFO);
+                        // cancel running queues and remove one of atomic operation
+                        this.cancelRelativeEvent(queue);
+                        return;
+                    }
+                }
+                if (queue.type == "RENAME") {
+                    // Obsolete
+                    await this.watchVaultRenameAsync(targetFile, queue.args.oldPath);
+                }
             }
-            if (queue.type == "RENAME") {
+            await this.kvDB.set(key, mtime);
-                // Obsolete
+        });
-                await this.watchVaultRenameAsync(targetFile, queue.args.oldPath);
-            }
-        }
-        await this.kvDB.set(key, mtime);
     }
 
     pendingFileEventCount = reactiveSource(0);
     processingFileEventCount = reactiveSource(0);
     fileEventQueue =
         new QueueProcessor(
-            (items: FileEventItem[]) => this.handleFileEvent(items[0]),
+            async (items: FileEventItem[]) => {
+                await this.handleFileEvent(items[0]);
+                return []
+            }
+            ,
             { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 100, yieldThreshold: FileWatchEventQueueMax, totalRemainingReactiveSource: this.pendingFileEventCount, processingEntitiesReactiveSource: this.processingFileEventCount }
         ).replaceEnqueueProcessor((items, newItem) => this.queueNextFileEvent(items, newItem));
 
@@ -1894,7 +1901,7 @@ We can perform a command in this file.
     observeForLogs() {
         const padSpaces = `\u{2007}`.repeat(10);
         // const emptyMark = `\u{2003}`;
-        const rerenderTimer = new Map<string, [ReturnType<typeof setTimeout>, number]>;
+        const rerenderTimer = new Map<string, [ReturnType<typeof setTimeout>, number]>();
         const tick = reactiveSource(0);
         function padLeftSp(num: number, mark: string) {
             const numLen = `${num}`.length + 1;
@@ -2005,9 +2012,9 @@ We can perform a command in this file.
             };
         })
         const statusBarLabels = reactive(() => {
-
+            const scheduleMessage = this.isReloadingScheduled ? `WARNING! RESTARTING OBSIDIAN IS SCHEDULED!\n` : "";
             const { message } = statusLineLabel.value;
-            const status = this.statusLog.value;
+            const status = scheduleMessage + this.statusLog.value;
             return {
                 message, status
             }
@@ -3174,5 +3181,61 @@ Or if you are sure know what had been happened, we can unlock the database from
         // @ts-ignore
         this.app.commands.executeCommandById(id)
     }
+
+    _totalProcessingCount?: ReactiveValue<number>;
+    get isReloadingScheduled() {
+        return this._totalProcessingCount !== undefined;
+    }
+    askReload(message?: string) {
+        if (this.isReloadingScheduled) {
+            Logger(`Reloading is already scheduled`, LOG_LEVEL_VERBOSE);
+            return;
+        }
+        scheduleTask("configReload", 250, async () => {
+            const RESTART_NOW = "Yes, restart immediately";
+            const RESTART_AFTER_STABLE = "Yes, schedule a restart after stabilisation";
+            const RETRY_LATER = "No, Leave it to me";
+            const ret = await askSelectString(this.app, message || "Do you want to restart and reload Obsidian now?", [RESTART_AFTER_STABLE, RESTART_NOW, RETRY_LATER]);
+            if (ret == RESTART_NOW) {
+                this.performAppReload();
+            } else if (ret == RESTART_AFTER_STABLE) {
+                this.scheduleAppReload();
+            }
+        })
+    }
+    scheduleAppReload() {
+        if (!this._totalProcessingCount) {
+            const __tick = reactiveSource(0);
+            this._totalProcessingCount = reactive(() => {
+                const dbCount = this.databaseQueueCount.value;
+                const replicationCount = this.replicationResultCount.value;
+                const storageApplyingCount = this.storageApplyingCount.value;
+                const chunkCount = collectingChunks.value;
+                const pluginScanCount = pluginScanningCount.value;
+                const hiddenFilesCount = hiddenFilesEventCount.value + hiddenFilesProcessingCount.value;
+                const conflictProcessCount = this.conflictProcessQueueCount.value;
+                const e = this.pendingFileEventCount.value;
+                const proc = this.processingFileEventCount.value;
+                // eslint-disable-next-line @typescript-eslint/no-unused-vars
+                const __ = __tick.value;
+                return dbCount + replicationCount + storageApplyingCount + chunkCount + pluginScanCount + hiddenFilesCount + conflictProcessCount + e + proc;
+            })
+            this.registerInterval(setInterval(() => {
+                __tick.value++;
+            }, 1000) as unknown as number);
+
+            let stableCheck = 3;
+            this._totalProcessingCount.onChanged(e => {
+                if (e.value == 0) {
+                    if (stableCheck-- <= 0) {
+                        this.performAppReload();
+                    }
+                    Logger(`Obsidian will be restarted soon! (Within ${stableCheck} seconds)`, LOG_LEVEL_NOTICE, "restart-notice");
+                } else {
+                    stableCheck = 3;
+                }
+            })
+        }
+    }
 }
 

src/ui/ObsidianLiveSyncSettingTab.ts

@@ -28,7 +28,7 @@ import { Logger } from "../lib/src/common/logger.ts";
 import { checkSyncInfo, isCloudantURI } from "../lib/src/pouchdb/utils_couchdb.ts";
 import { testCrypt } from "../lib/src/encryption/e2ee_v2.ts";
 import ObsidianLiveSyncPlugin from "../main.ts";
-import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "../common/utils.ts";
+import { askYesNo, performRebuildDB, requestToCouchDB } from "../common/utils.ts";
 import { request, type ButtonComponent, TFile } from "obsidian";
 import { shouldBeIgnored } from "../lib/src/string_and_binary/path.ts";
 import MultipleRegExpControl from './components/MultipleRegExpControl.svelte';
@@ -51,15 +51,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
         await replicator.tryConnectRemote(trialSetting);
     }
 
-    askReload(message?: string) {
-        scheduleTask("configReload", 250, async () => {
-            if (await askYesNo(this.app, message || "Do you want to restart and reload Obsidian now?") == "yes") {
-                // @ts-ignore
-                this.app.commands.executeCommandById("app:reload")
-            }
-        })
-    }
-
     closeSetting() {
         // @ts-ignore
         this.plugin.app.setting.close()
@@ -217,7 +208,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     text.setButtonText("Enable").onClick(async () => {
                         this.plugin.settings.isConfigured = true;
                         await this.plugin.saveSettings();
-                        this.askReload();
+                        this.plugin.askReload();
                     })
                 })
         }
@@ -231,7 +222,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                             await this.plugin.saveSettingData();
                             await this.plugin.resetLocalDatabase();
                             // await this.plugin.initializeDatabase();
-                            this.askReload();
+                            this.plugin.askReload();
                         }
                     }).setWarning()
                 })
@@ -1317,7 +1308,7 @@ However, your report is needed to stabilise this. I appreciate you for your grea
                                 Logger("All done! Please set up subsequent devices with 'Copy current settings as a new setup URI' and 'Use the copied setup URI'.", LOG_LEVEL_NOTICE);
                                 await this.plugin.addOnSetup.command_copySetupURI();
                             } else {
-                                this.askReload();
+                                this.plugin.askReload();
                             }
                         }
                     })
@@ -1976,7 +1967,7 @@ ${stringifyYaml(pluginConfig)}`;
                     .onClick(async () => {
                         this.plugin.settings.isConfigured = false;
                         await this.plugin.saveSettings();
-                        this.askReload();
+                        this.plugin.askReload();
                     }));
         const hatchWarn = containerHatchEl.createEl("div", { text: `To stop the boot up sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
         hatchWarn.addClass("op-warn-info");
@@ -2167,7 +2158,7 @@ ${stringifyYaml(pluginConfig)}`;
                 toggle.setValue(this.plugin.settings.suspendFileWatching).onChange(async (value) => {
                     this.plugin.settings.suspendFileWatching = value;
                     await this.plugin.saveSettings();
-                    this.askReload();
+                    this.plugin.askReload();
                 })
             );
         new Setting(containerHatchEl)
@@ -2177,7 +2168,7 @@ ${stringifyYaml(pluginConfig)}`;
                 toggle.setValue(this.plugin.settings.suspendParseReplicationResult).onChange(async (value) => {
                     this.plugin.settings.suspendParseReplicationResult = value;
                     await this.plugin.saveSettings();
-                    this.askReload();
+                    this.plugin.askReload();
                 })
             );
         new Setting(containerHatchEl)

updates.md

@@ -18,6 +18,13 @@ I have a lot of respect for that plugin, even though it is sometimes treated as
 Hooray for open source, and generous licences, and the sharing of knowledge by experts.
 
 #### Version history
+- 0.23.7
+  - Fixed:
+    - No longer missing tasks which have queued as the same key (e.g., for the same operation to the same file).
+      - This occurs, for example, with hidden files that have been changed multiple times in a very short period of time, such as `appearance.json`. Thanks for the report!
+    - Some trivial issues have been fixed.
+  - New feature:
+    - Reloading Obsidian can be scheduled until that file and database operations are stable.
 - 0.23.6:
   - Fixed:
     - Now the remote chunks could be decrypted even if we are using `Incubate chunks in Document`. (The note of 0.23.6 has been fixed).