update a dependency.

- Reduced remote database checking to improve speed and reduce bandwidth.

Fixed:
- Chunks which previously misinterpreted are now interpreted correctly.
- Deleted file detection on hidden file synchronising now works fine.
- Now the Customisation sync is surely quiet while it has been disabled.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.19.1",
+    "version": "0.19.4",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.1",
+    "version": "0.19.4",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.19.1",
+            "version": "0.19.4",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",
@@ -4087,9 +4087,9 @@
             "dev": true
         },
         "node_modules/yaml": {
-            "version": "2.1.3",
-            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz",
-            "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==",
+            "version": "2.2.2",
+            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.2.tgz",
+            "integrity": "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA==",
             "dev": true,
             "engines": {
                 "node": ">= 14"
@@ -7047,9 +7047,9 @@
             "dev": true
         },
         "yaml": {
-            "version": "2.1.3",
-            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz",
-            "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==",
+            "version": "2.2.2",
+            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.2.tgz",
+            "integrity": "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA==",
             "dev": true
         },
         "yocto-queue": {

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.1",
+    "version": "0.19.4",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/CmdConfigSync.ts

@@ -145,7 +145,7 @@ export class ConfigSync extends LiveSyncCommands {
         if (this.plugin.suspended) {
             return;
         }
-        if (this.settings.autoSweepPlugins) {
+        if (this.settings.autoSweepPlugins && this.settings.usePluginSync) {
             await this.scanAllConfigFiles(false);
         }
         this.periodicPluginSweepProcessor.enable(this.settings.autoSweepPluginsPeriodic && !this.settings.watchInternalFileChanges ? (PERIODIC_PLUGIN_SWEEP * 1000) : 0);
@@ -567,6 +567,7 @@ export class ConfigSync extends LiveSyncCommands {
 
     }
     async watchVaultRawEventsAsync(path: FilePath) {
+        if (!this.settings.usePluginSync) return false;
         if (!this.isTargetPath(path)) return false;
         const stat = await this.app.vault.adapter.stat(path);
         // Make sure that target is a file.

src/CmdHiddenFileSync.ts

@@ -495,7 +495,7 @@ export class HiddenFileSync extends LiveSyncCommands {
         const mtime = new Date().getTime();
         await runWithLock("file-" + prefixedFileName, false, async () => {
             try {
-                const old = await this.localDatabase.getDBEntry(prefixedFileName, null, false, false) as InternalFileEntry | false;
+                const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, null, true) as InternalFileEntry | false;
                 let saveData: InternalFileEntry;
                 if (old === false) {
                     saveData = {
@@ -541,7 +541,7 @@ export class HiddenFileSync extends LiveSyncCommands {
             try {
                 // Check conflicted status 
                 //TODO option
-                const fileOnDB = await this.localDatabase.getDBEntry(prefixedFileName, { conflicts: true }, false, false);
+                const fileOnDB = await this.localDatabase.getDBEntry(prefixedFileName, { conflicts: true }, false, true);
                 if (fileOnDB === false)
                     throw new Error(`File not found on database.:${filename}`);
                 // Prevent overwrite for Prevent overwriting while some conflicted revision exists.

src/ObsidianLiveSyncSettingTab.ts

@@ -19,7 +19,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
         this.plugin = plugin;
     }
     async testConnection(): Promise<void> {
-        const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.isMobile);
+        const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.isMobile, true);
         if (typeof db === "string") {
             this.plugin.addLog(`could not connect to ${this.plugin.settings.couchDB_URI} : ${this.plugin.settings.couchDB_DBNAME} \n(${db})`, LOG_LEVEL.NOTICE);
             return;
@@ -376,7 +376,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 useDynamicIterationCount: useDynamicIterationCount,
             };
             console.dir(settingForCheck);
-            const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(settingForCheck, this.plugin.isMobile);
+            const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(settingForCheck, this.plugin.isMobile, true);
             if (typeof db === "string") {
                 Logger("Could not connect to the database.", LOG_LEVEL.NOTICE);
                 return false;
@@ -1599,6 +1599,16 @@ ${stringifyYaml(pluginConfig)}`;
                 })
             );
 
+        new Setting(containerHatchEl)
+            .setName("Do not pace synchronization")
+            .setDesc("If this toggle enabled, synchronisation will not be paced by queued entries. If synchronisation has been deadlocked, please make this enabled once.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.doNotPaceReplication).onChange(async (value) => {
+                    this.plugin.settings.doNotPaceReplication = value;
+                    await this.plugin.saveSettings();
+                })
+            );
+
 
 
 

src/main.ts

@@ -87,7 +87,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     }
 
     processReplication = (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => this.parseReplicationResult(e);
-    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
+    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean, performSetup: boolean, skipInfo: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
         if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid";
         if (uri.toLowerCase() != uri) return "Remote URI and database name could not contain capital letters.";
         if (uri.indexOf(" ") !== -1) return "Remote URI and database name could not contain spaces.";
@@ -104,6 +104,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         const conf: PouchDB.HttpAdapter.HttpAdapterConfiguration = {
             adapter: "http",
             auth,
+            skip_setup: !performSetup,
             fetch: async (url: string | Request, opts: RequestInit) => {
                 let size = "";
                 const localURL = url.toString().substring(uri.length);
@@ -192,6 +193,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         if (passphrase !== "false" && typeof passphrase === "string") {
             enableEncryption(db, passphrase, useDynamicIterationCount);
         }
+        if (skipInfo) {
+            return { db: db, info: {} };
+        }
         try {
             const info = await db.info();
             return { db: db, info: info };
@@ -1350,7 +1354,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                 const docMtime = ~~(doc.mtime / 1000);
                 //TODO: some margin required.
                 if (localMtime >= docMtime) {
-                    Logger(`${doc._id} Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
+                    Logger(`${path} (${doc._id}, ${doc._rev}) Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
                     return;
                 }
             }
@@ -1361,12 +1365,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin
             missingChildren: [] as string[],
             timeout: now + this.chunkWaitTimeout,
         };
-        // If `Read chunks online` is enabled, retrieve chunks from the remote CouchDB directly.
+        // If `Read chunks online` is disabled, chunks should be transferred before here.
+        // However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
         if ((!this.settings.readChunksOnline) && "children" in doc) {
-            const c = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: doc.children, include_docs: false });
-            const missing = c.rows.filter((e) => "error" in e).map((e) => e.key);
-            // fetch from remote
-            if (missing.length > 0) Logger(`${doc._id}(${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
+            const c = await this.localDatabase.collectChunksWithCache(doc.children);
+            const missing = c.filter((e) => e.chunk === false).map((e) => e.id);
+            if (missing.length > 0) Logger(`${path} (${doc._id}, ${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
             newQueue.missingChildren = missing;
             this.queuedFiles.push(newQueue);
         } else {
@@ -1381,15 +1385,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         const docsSorted = docs.sort((a, b) => b.mtime - a.mtime);
         L1:
         for (const change of docsSorted) {
+            if (isChunk(change._id)) {
+                await this.parseIncomingChunk(change);
+                continue;
+            }
             for (const proc of this.addOns) {
                 if (await proc.parseReplicationResultItem(change)) {
                     continue L1;
                 }
             }
-            if (isChunk(change._id)) {
-                await this.parseIncomingChunk(change);
-                continue;
-            }
             if (change._id == SYNCINFO_ID) {
                 continue;
             }
@@ -1509,27 +1513,25 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         return;
     }
 
-    logHideTimer: NodeJS.Timeout = null;
     setStatusBarText(message: string = null, log: string = null) {
         if (!this.statusBar) return;
         const newMsg = typeof message == "string" ? message : this.lastMessage;
         const newLog = typeof log == "string" ? log : this.lastLog;
         if (`${this.lastMessage}-${this.lastLog}` != `${newMsg}-${newLog}`) {
-            this.statusBar.setText(newMsg.split("\n")[0]);
+            scheduleTask("update-display", 50, () => {
+                this.statusBar.setText(newMsg.split("\n")[0]);
 
-            if (this.settings.showStatusOnEditor) {
-                const root = activeDocument.documentElement;
-                const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
-                q.forEach(e => e.setAttr("data-log", '' + (newMsg + "\n" + newLog) + ''))
-            } else {
-                const root = activeDocument.documentElement;
-                const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
-                q.forEach(e => e.setAttr("data-log", ''))
-            }
-            if (this.logHideTimer != null) {
-                clearTimeout(this.logHideTimer);
-            }
-            this.logHideTimer = setTimeout(() => this.setStatusBarText(null, ""), 3000);
+                if (this.settings.showStatusOnEditor) {
+                    const root = activeDocument.documentElement;
+                    const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+                    q.forEach(e => e.setAttr("data-log", '' + (newMsg + "\n" + newLog) + ''))
+                } else {
+                    const root = activeDocument.documentElement;
+                    const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+                    q.forEach(e => e.setAttr("data-log", ''))
+                }
+            }, true);
+            scheduleTask("log-hide", 3000, () => this.setStatusBarText(null, ""));
             this.lastMessage = newMsg;
             this.lastLog = newLog;
         }
@@ -2137,16 +2139,10 @@ Or if you are sure know what had been happened, we can unlock the database from
     conflictedCheckFiles: FilePath[] = [];
 
     // queueing the conflicted file check
-    conflictedCheckTimer: number;
-
     queueConflictedCheck(file: TFile) {
         this.conflictedCheckFiles = this.conflictedCheckFiles.filter((e) => e != file.path);
         this.conflictedCheckFiles.push(getPathFromTFile(file));
-        if (this.conflictedCheckTimer != null) {
-            window.clearTimeout(this.conflictedCheckTimer);
-        }
-        this.conflictedCheckTimer = window.setTimeout(async () => {
-            this.conflictedCheckTimer = null;
+        scheduleTask("check-conflict", 100, async () => {
             const checkFiles = JSON.parse(JSON.stringify(this.conflictedCheckFiles)) as FilePath[];
             for (const filename of checkFiles) {
                 try {
@@ -2158,7 +2154,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                     Logger(ex);
                 }
             }
-        }, 100);
+        });
     }
 
     async showIfConflicted(filename: FilePathWithPrefix) {

src/utils.ts

@@ -44,7 +44,10 @@ export function getPathFromTFile(file: TAbstractFile) {
 }
 
 const tasks: { [key: string]: ReturnType<typeof setTimeout> } = {};
-export function scheduleTask(key: string, timeout: number, proc: (() => Promise<any> | void)) {
+export function scheduleTask(key: string, timeout: number, proc: (() => Promise<any> | void), skipIfTaskExist?: boolean) {
+    if (skipIfTaskExist && key in tasks) {
+        return;
+    }
     cancelTask(key);
     tasks[key] = setTimeout(async () => {
         delete tasks[key];
@@ -663,6 +666,14 @@ export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryR
         return Number.parseInt((info as any)?.sizes?.[key] ?? 0);
     }
     await runWithLock("clean-up:remote", true, async () => {
+        const CHUNK_SIZE = 100;
+        function makeChunkedArrayFromArray<T>(items: T[]): T[][] {
+            const chunked = [];
+            for (let i = 0; i < items.length; i += CHUNK_SIZE) {
+                chunked.push(items.slice(i, i + CHUNK_SIZE));
+            }
+            return chunked;
+        }
         try {
             const ret = await plugin.replicator.connectRemoteCouchDBWithSetting(plugin.settings, plugin.isMobile);
             if (typeof ret === "string") {
@@ -701,14 +712,17 @@ export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryR
                 return;
             }
             Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
-            const rets = await _requestToCouchDBFetch(
-                `${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
-                plugin.settings.couchDB_USER,
-                plugin.settings.couchDB_PASSWORD,
-                "_purge",
-                payload, "POST");
-            // const result = await rets();
-            Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
+            const buffer = makeChunkedArrayFromArray(Object.entries(payload));
+            for (const chunkedPayload of buffer) {
+                const rets = await _requestToCouchDBFetch(
+                    `${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
+                    plugin.settings.couchDB_USER,
+                    plugin.settings.couchDB_PASSWORD,
+                    "_purge",
+                    chunkedPayload.reduce((p, c) => ({ ...p, [c[0]]: c[1] }), {}), "POST");
+                // const result = await rets();
+                Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
+            }
             Logger(`Compacting database...`, LOG_LEVEL.NOTICE, "clean-up-db");
             await db.compact();
             const endInfo = await db.info();

updates.md

@@ -17,5 +17,23 @@ I hope you will give it a try.
 - 0.19.1
   - Fixed: Fixed hidden file handling on Linux
   - Improved: Now customization sync works more smoothly.
+- 0.19.2
+  - Fixed:
+    - Fixed garbage collection error while unreferenced chunks exist many.
+    - Fixed filename validation on Linux.
+  - Improved:
+    - Showing status is now thinned for performance.
+    - Enhance caching while collecting chunks.
+- 0.19.3
+  - Improved:
+    - Now replication will be paced by collecting chunks. If synchronisation has been deadlocked, please enable `Do not pace synchronization` once.
+- 0.19.4
+  - Improved:
+    - Reduced remote database checking to improve speed and reduce bandwidth.
+  - Fixed:
+    - Chunks which previously misinterpreted are now interpreted correctly.
+      - No more missing chunks which not be found forever, except if it has been actually missing.
+    - Deleted file detection on hidden file synchronising now works fine.
+    - Now the Customisation sync is surely quiet while it has been disabled.
 
 ... To continue on to `updates_old.md`.