bump

- Now hidden file synchronisation would not be hanged, even if so many files exist.

Improved:
- Customisation sync works more smoothly.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.19.0",
+    "version": "0.19.5",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.0",
+    "version": "0.19.5",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.19.0",
+            "version": "0.19.5",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",
@@ -4087,9 +4087,9 @@
             "dev": true
         },
         "node_modules/yaml": {
-            "version": "2.1.3",
-            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz",
-            "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==",
+            "version": "2.2.2",
+            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.2.tgz",
+            "integrity": "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA==",
             "dev": true,
             "engines": {
                 "node": ">= 14"
@@ -7047,9 +7047,9 @@
             "dev": true
         },
         "yaml": {
-            "version": "2.1.3",
-            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz",
-            "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==",
+            "version": "2.2.2",
+            "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.2.tgz",
+            "integrity": "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA==",
             "dev": true
         },
         "yocto-queue": {

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.0",
+    "version": "0.19.5",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/CmdConfigSync.ts

@@ -1,29 +1,45 @@
 import { writable } from 'svelte/store';
-import { Notice, PluginManifest, stringifyYaml, parseYaml } from "./deps";
+import { Notice, PluginManifest, parseYaml } from "./deps";
 
 import { EntryDoc, LoadedEntry, LOG_LEVEL, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID } from "./lib/src/types";
 import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
-import { delay, getDocData } from "./lib/src/utils";
+import { Parallels, delay, getDocData } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { WrappedNotice } from "./lib/src/wrapper";
-import { base64ToArrayBuffer, arrayBufferToBase64, readString, writeString, uint8ArrayToHexString } from "./lib/src/strbin";
+import { base64ToArrayBuffer, arrayBufferToBase64, readString, uint8ArrayToHexString } from "./lib/src/strbin";
 import { runWithLock } from "./lib/src/lock";
 import { LiveSyncCommands } from "./LiveSyncCommands";
-import { addPrefix, stripAllPrefixes } from "./lib/src/path";
+import { stripAllPrefixes } from "./lib/src/path";
 import { PeriodicProcessor, askYesNo, disposeMemoObject, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask } from "./utils";
 import { Semaphore } from "./lib/src/semaphore";
 import { PluginDialogModal } from "./dialogs";
 import { JsonResolveModal } from "./JsonResolveModal";
 
 
+function serialize<T>(obj: T): string {
+    return JSON.stringify(obj, null, 1);
+}
+function deserialize<T>(str: string, def: T) {
+    try {
+        return JSON.parse(str) as T;
+    } catch (ex) {
+        try {
+            return parseYaml(str);
+        } catch (ex) {
+            return def;
+        }
+    }
+}
 
 
 export const pluginList = writable([] as PluginDataExDisplay[]);
 export const pluginIsEnumerating = writable(false);
 
+const encoder = new TextEncoder();
 const hashString = (async (key: string) => {
-    const buff = writeString(key);
+    // const buff = writeString(key);
+    const buff = encoder.encode(key);
     const digest = await crypto.subtle.digest('SHA-256', buff);
     return uint8ArrayToHexString(new Uint8Array(digest));
 })
@@ -67,6 +83,7 @@ export class ConfigSync extends LiveSyncCommands {
     pluginDialog: PluginDialogModal = null;
     periodicPluginSweepProcessor = new PeriodicProcessor(this.plugin, async () => await this.scanAllConfigFiles(false));
 
+    pluginList: PluginDataExDisplay[] = [];
     showPluginSyncModal() {
         if (!this.settings.usePluginSync) {
             return;
@@ -141,49 +158,59 @@ export class ConfigSync extends LiveSyncCommands {
         }
     }
     async onResume() {
-        if (this.plugin.suspended)
+        if (this.plugin.suspended) {
             return;
-        if (this.settings.autoSweepPlugins) {
-            await this.scanAllConfigFiles(true);
+        }
+        if (this.settings.autoSweepPlugins && this.settings.usePluginSync) {
+            await this.scanAllConfigFiles(false);
         }
         this.periodicPluginSweepProcessor.enable(this.settings.autoSweepPluginsPeriodic && !this.settings.watchInternalFileChanges ? (PERIODIC_PLUGIN_SWEEP * 1000) : 0);
-    }
 
-    async reloadPluginList() {
-        pluginList.set([])
-        await this.updatePluginList();
     }
-    async updatePluginList(updatedDocumentPath?: FilePathWithPrefix): Promise<void> {
+    async reloadPluginList(showMessage: boolean) {
+        this.pluginList = [];
+        pluginList.set(this.pluginList)
+        await this.updatePluginList(showMessage);
+    }
+    async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise<void> {
+        const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
         // pluginList.set([]);
         if (!this.settings.usePluginSync) {
-            pluginList.set([]);
+            this.pluginList = [];
+            pluginList.set(this.pluginList)
             return;
         }
 
         await runWithLock("update-plugin-list", false, async () => {
-            if (updatedDocumentPath != "") pluginList.update(e => e.filter(ee => ee.documentPath != updatedDocumentPath));
+            // if (updatedDocumentPath != "") pluginList.update(e => e.filter(ee => ee.documentPath != updatedDocumentPath));
             // const work: Record<string, Record<string, Record<string, Record<string, PluginDataEntryEx>>>> = {};
             const entries = [] as PluginDataExDisplay[]
             const plugins = this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true });
             const semaphore = Semaphore(4);
-            const processes = [] as Promise<void>[];
+            const para = Parallels();
             let count = 0;
             pluginIsEnumerating.set(true);
+            let processed = false;
             try {
                 for await (const plugin of plugins) {
                     const path = plugin.path || this.getPath(plugin);
                     if (updatedDocumentPath && updatedDocumentPath != path) {
                         continue;
                     }
-                    processes.push((async (v) => {
+                    processed = true;
+                    const oldEntry = (this.pluginList.find(e => e.documentPath == path));
+                    if (oldEntry && oldEntry.mtime == plugin.mtime) continue;
+                    await para.wait(5);
+                    para.add((async (v) => {
+
                         const release = await semaphore.acquire(1);
                         try {
-                            Logger(`Enumerating files... ${count++}`, LOG_LEVEL.NOTICE, "get-plugins");
+                            Logger(`Enumerating files... ${count++}`, logLevel, "get-plugins");
 
                             Logger(`plugin-${path}`, LOG_LEVEL.VERBOSE);
                             const wx = await this.localDatabase.getDBEntry(path, null, false, false);
                             if (wx) {
-                                const data = parseYaml(getDocData(wx.data)) as PluginDataEx;
+                                const data = deserialize(getDocData(wx.data), {}) as PluginDataEx;
                                 const xFiles = [] as PluginDataExFile[];
                                 for (const file of data.files) {
                                     const work = { ...file };
@@ -199,7 +226,7 @@ export class ConfigSync extends LiveSyncCommands {
                             }
                         } catch (ex) {
                             //TODO
-                            Logger(`Something happened at enumerating customization :${v.path}`);
+                            Logger(`Something happened at enumerating customization :${v.path}`, LOG_LEVEL.NOTICE);
                             console.warn(ex);
                         } finally {
                             release();
@@ -207,17 +234,19 @@ export class ConfigSync extends LiveSyncCommands {
                     }
                     )(plugin));
                 }
-                await Promise.all(processes);
-                pluginList.update(e => {
-                    let newList = [...e];
-                    for (const item of entries) {
-                        console.log(item.documentPath);
-                        newList = newList.filter(x => x.documentPath != item.documentPath);
-                        newList.push(item)
-                    }
-                    return newList;
-                })
-                Logger(`All files enumerated`, LOG_LEVEL.NOTICE, "get-plugins");
+                await para.all();
+                let newList = [...this.pluginList];
+                for (const item of entries) {
+                    newList = newList.filter(x => x.documentPath != item.documentPath);
+                    newList.push(item)
+                }
+                if (updatedDocumentPath != "" && !processed) newList = newList.filter(e => e.documentPath != updatedDocumentPath);
+
+                this.pluginList = newList;
+                pluginList.set(newList);
+
+
+                Logger(`All files enumerated`, logLevel, "get-plugins");
             } finally {
                 pluginIsEnumerating.set(false);
             }
@@ -229,9 +258,9 @@ export class ConfigSync extends LiveSyncCommands {
         const docB = await this.localDatabase.getDBEntry(dataB.documentPath);
 
         if (docA && docB) {
-            const pluginDataA = parseYaml(getDocData(docA.data)) as PluginDataEx;
+            const pluginDataA = deserialize(getDocData(docA.data), {}) as PluginDataEx;
             pluginDataA.documentPath = dataA.documentPath;
-            const pluginDataB = parseYaml(getDocData(docB.data)) as PluginDataEx;
+            const pluginDataB = deserialize(getDocData(docB.data), {}) as PluginDataEx;
             pluginDataB.documentPath = dataB.documentPath;
 
             // Use outer structure to wrap each data.
@@ -270,7 +299,7 @@ export class ConfigSync extends LiveSyncCommands {
             if (dx == false) {
                 throw "Not found on database"
             }
-            const loadedData = parseYaml(getDocData(dx.data)) as PluginDataEx;
+            const loadedData = deserialize(getDocData(dx.data), {}) as PluginDataEx;
             for (const f of loadedData.files) {
                 Logger(`Applying ${f.filename} of ${data.displayName || data.name}..`);
                 try {
@@ -293,7 +322,7 @@ export class ConfigSync extends LiveSyncCommands {
             }
             const uPath = `${baseDir}/${loadedData.files[0].filename}` as FilePath;
             await this.storeCustomizationFiles(uPath);
-            await this.updatePluginList(uPath);
+            await this.updatePluginList(true, uPath);
             await delay(100);
             Logger(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL.NOTICE);
             if (data.category == "PLUGIN_DATA" || data.category == "PLUGIN_MAIN") {
@@ -329,6 +358,7 @@ export class ConfigSync extends LiveSyncCommands {
         try {
             if (data.documentPath) {
                 await this.deleteConfigOnDatabase(data.documentPath);
+                await this.updatePluginList(false, data.documentPath);
                 Logger(`Delete: ${data.documentPath}`, LOG_LEVEL.NOTICE);
             }
             return true;
@@ -338,8 +368,11 @@ export class ConfigSync extends LiveSyncCommands {
 
         }
     }
-    parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
+    async parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
         if (docs._id.startsWith(ICXHeader)) {
+            if (this.plugin.settings.usePluginSync) {
+                await this.updatePluginList(false, docs.path ? docs.path : this.getPath(docs));
+            }
             if (this.plugin.settings.usePluginSync && this.plugin.settings.notifyPluginOrSettingUpdated) {
                 if (!this.pluginDialog || (this.pluginDialog && !this.pluginDialog.isOpened())) {
                     const fragment = createFragment((doc) => {
@@ -347,7 +380,7 @@ export class ConfigSync extends LiveSyncCommands {
                             a.appendText(`Some configuration has been arrived, Press `);
                             a.appendChild(a.createEl("a", null, (anchor) => {
                                 anchor.text = "HERE";
-                                anchor.addEventListener("click", async () => {
+                                anchor.addEventListener("click", () => {
                                     this.showPluginSyncModal();
                                 });
                             }));
@@ -375,10 +408,7 @@ export class ConfigSync extends LiveSyncCommands {
                             disposeMemoObject(updatedPluginKey);
                         });
                     });
-                } else {
-                    this.updatePluginList(docs.path ? docs.path : this.getPath(docs));
                 }
-
             }
             return true;
         }
@@ -502,10 +532,12 @@ export class ConfigSync extends LiveSyncCommands {
             // Logger(`Configuration saving: ${prefixedFileName}`);
             if (dt.files.length == 0) {
                 Logger(`Nothing left: deleting.. ${path}`);
-                return await this.deleteConfigOnDatabase(prefixedFileName);
+                await this.deleteConfigOnDatabase(prefixedFileName);
+                await this.updatePluginList(false, prefixedFileName);
+                return
             }
 
-            const content = stringifyYaml(dt);
+            const content = serialize(dt);
             try {
                 const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, null, false);
                 let saveData: LoadedEntry;
@@ -540,6 +572,7 @@ export class ConfigSync extends LiveSyncCommands {
                     };
                 }
                 const ret = await this.localDatabase.putDBEntry(saveData);
+                await this.updatePluginList(false, saveData.path);
                 Logger(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
                 return ret;
             } catch (ex) {
@@ -548,10 +581,10 @@ export class ConfigSync extends LiveSyncCommands {
                 return false;
             }
         })
-        // })
 
     }
     async watchVaultRawEventsAsync(path: FilePath) {
+        if (!this.settings.usePluginSync) return false;
         if (!this.isTargetPath(path)) return false;
         const stat = await this.app.vault.adapter.stat(path);
         // Make sure that target is a file.
@@ -591,6 +624,7 @@ export class ConfigSync extends LiveSyncCommands {
         for (const vp of deleteCandidate) {
             await this.deleteConfigOnDatabase(vp);
         }
+        this.updatePluginList(false).then(/* fire and forget */);
     }
     async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false) {
 
@@ -618,6 +652,7 @@ export class ConfigSync extends LiveSyncCommands {
                     };
                 }
                 await this.localDatabase.putRaw(saveData);
+                await this.updatePluginList(false, prefixedFileName);
                 Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Done`);
             } catch (ex) {
                 Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`);

src/CmdHiddenFileSync.ts

@@ -1,14 +1,13 @@
 import { Notice, normalizePath, PluginManifest } from "./deps";
 import { EntryDoc, LoadedEntry, LOG_LEVEL, InternalFileEntry, FilePathWithPrefix, FilePath } from "./lib/src/types";
 import { InternalFileInfo, ICHeader, ICHeaderEnd } from "./types";
-import { delay, isDocContentSame } from "./lib/src/utils";
+import { Parallels, delay, isDocContentSame } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { disposeMemoObject, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask, isInternalMetadata, PeriodicProcessor } from "./utils";
 import { WrappedNotice } from "./lib/src/wrapper";
 import { base64ToArrayBuffer, arrayBufferToBase64 } from "./lib/src/strbin";
 import { runWithLock } from "./lib/src/lock";
-import { Semaphore } from "./lib/src/semaphore";
 import { JsonResolveModal } from "./JsonResolveModal";
 import { LiveSyncCommands } from "./LiveSyncCommands";
 import { addPrefix, stripAllPrefixes } from "./lib/src/path";
@@ -254,37 +253,35 @@ export class HiddenFileSync extends LiveSyncCommands {
                 c = pieces.shift();
             }
         };
-        const p = [] as Promise<void>[];
-        const semaphore = Semaphore(10);
         // Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content)
         let caches: { [key: string]: { storageMtime: number; docMtime: number; }; } = {};
         caches = await this.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number; }; }>("diff-caches-internal") || {};
+        const filesMap = files.reduce((acc, cur) => {
+            acc[cur.path] = cur;
+            return acc;
+        }, {} as { [key: string]: InternalFileInfo; });
+        const filesOnDBMap = filesOnDB.reduce((acc, cur) => {
+            acc[stripAllPrefixes(this.getPath(cur))] = cur;
+            return acc;
+        }, {} as { [key: string]: InternalFileEntry; });
+        const para = Parallels();
         for (const filename of allFileNames) {
-            if (!filename) continue;
             processed++;
-            if (processed % 100 == 0)
+            if (processed % 100 == 0) {
                 Logger(`Hidden file: ${processed}/${fileCount}`, logLevel, "sync_internal");
+            }
+            if (!filename) continue;
             if (ignorePatterns.some(e => filename.match(e)))
                 continue;
 
-            const fileOnStorage = files.find(e => e.path == filename);
-            const fileOnDatabase = filesOnDB.find(e => stripAllPrefixes(this.getPath(e)) == filename);
-            const addProc = async (p: () => Promise<void>): Promise<void> => {
-                const releaser = await semaphore.acquire(1);
-                try {
-                    return p();
-                } catch (ex) {
-                    Logger("Some process failed", logLevel);
-                    Logger(ex);
-                } finally {
-                    releaser();
-                }
-            };
+            const fileOnStorage = filename in filesMap ? filesMap[filename] : undefined;
+            const fileOnDatabase = filename in filesOnDBMap ? filesOnDBMap[filename] : undefined;
+
             const cache = filename in caches ? caches[filename] : { storageMtime: 0, docMtime: 0 };
 
-            p.push(addProc(async () => {
-                const xFileOnStorage = fileOnStorage;
-                const xFileOnDatabase = fileOnDatabase;
+            await para.wait(5);
+            const proc = (async (xFileOnStorage: InternalFileInfo, xFileOnDatabase: InternalFileEntry) => {
+
                 if (xFileOnStorage && xFileOnDatabase) {
                     // Both => Synchronize
                     if ((direction != "pullForce" && direction != "pushForce") && xFileOnDatabase.mtime == cache.docMtime && xFileOnStorage.mtime == cache.storageMtime) {
@@ -326,9 +323,11 @@ export class HiddenFileSync extends LiveSyncCommands {
                     throw new Error("Invalid state on hidden file sync");
                     // Something corrupted?
                 }
-            }));
+
+            });
+            para.add(proc(fileOnStorage, fileOnDatabase))
         }
-        await Promise.all(p);
+        await para.all();
         await this.kvDB.set("diff-caches-internal", caches);
 
         // When files has been retrieved from the database. they must be reloaded.
@@ -495,7 +494,7 @@ export class HiddenFileSync extends LiveSyncCommands {
         const mtime = new Date().getTime();
         await runWithLock("file-" + prefixedFileName, false, async () => {
             try {
-                const old = await this.localDatabase.getDBEntry(prefixedFileName, null, false, false) as InternalFileEntry | false;
+                const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, null, true) as InternalFileEntry | false;
                 let saveData: InternalFileEntry;
                 if (old === false) {
                     saveData = {
@@ -541,7 +540,7 @@ export class HiddenFileSync extends LiveSyncCommands {
             try {
                 // Check conflicted status 
                 //TODO option
-                const fileOnDB = await this.localDatabase.getDBEntry(prefixedFileName, { conflicts: true }, false, false);
+                const fileOnDB = await this.localDatabase.getDBEntry(prefixedFileName, { conflicts: true }, false, true);
                 if (fileOnDB === false)
                     throw new Error(`File not found on database.:${filename}`);
                 // Prevent overwrite for Prevent overwriting while some conflicted revision exists.

src/ObsidianLiveSyncSettingTab.ts

@@ -19,7 +19,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
         this.plugin = plugin;
     }
     async testConnection(): Promise<void> {
-        const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.isMobile);
+        const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.isMobile, true);
         if (typeof db === "string") {
             this.plugin.addLog(`could not connect to ${this.plugin.settings.couchDB_URI} : ${this.plugin.settings.couchDB_DBNAME} \n(${db})`, LOG_LEVEL.NOTICE);
             return;
@@ -376,7 +376,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                 useDynamicIterationCount: useDynamicIterationCount,
             };
             console.dir(settingForCheck);
-            const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(settingForCheck, this.plugin.isMobile);
+            const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(settingForCheck, this.plugin.isMobile, true);
             if (typeof db === "string") {
                 Logger("Could not connect to the database.", LOG_LEVEL.NOTICE);
                 return false;
@@ -1599,6 +1599,16 @@ ${stringifyYaml(pluginConfig)}`;
                 })
             );
 
+        new Setting(containerHatchEl)
+            .setName("Do not pace synchronization")
+            .setDesc("If this toggle enabled, synchronisation will not be paced by queued entries. If synchronisation has been deadlocked, please make this enabled once.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.doNotPaceReplication).onChange(async (value) => {
+                    this.plugin.settings.doNotPaceReplication = value;
+                    await this.plugin.saveSettings();
+                })
+            );
+
 
 
 

src/PluginCombo.svelte

@@ -207,21 +207,21 @@
         const local = list.find((e) => e.term == thisTerm);
         const selectedItem = list.find((e) => e.term == selected);
         if (selectedItem && (await applyData(selectedItem))) {
-            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(local.documentPath));
+            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(true, local.documentPath));
         }
     }
     async function compareSelected() {
         const local = list.find((e) => e.term == thisTerm);
         const selectedItem = list.find((e) => e.term == selected);
         if (local && selectedItem && (await compareData(local, selectedItem))) {
-            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(local.documentPath));
+            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(true, local.documentPath));
         }
     }
     async function deleteSelected() {
         const selectedItem = list.find((e) => e.term == selected);
-        const deletedPath = selectedItem.documentPath;
+        // const deletedPath = selectedItem.documentPath;
         if (selectedItem && (await deleteData(selectedItem))) {
-            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(deletedPath));
+            scheduleTask("update-plugin-list", 250, () => addOn.reloadPluginList(true));
         }
     }
     async function duplicateItem() {
@@ -234,7 +234,7 @@
             }
             const key = `${plugin.app.vault.configDir}/${local.files[0].filename}`;
             await addOn.storeCustomizationFiles(key as FilePath, duplicateTermName);
-            await addOn.updatePluginList(addOn.filenameToUnifiedKey(key, duplicateTermName));
+            await addOn.updatePluginList(false, addOn.filenameToUnifiedKey(key, duplicateTermName));
         }
     }
 </script>
@@ -281,6 +281,7 @@
 
 <style>
     .spacer {
+        min-width: 1px;
         flex-grow: 1;
     }
     button {
@@ -311,4 +312,7 @@
         flex-direction: column;
         align-items: center;
     }
+    :global(.is-mobile) .spacer {
+        margin-left: auto;
+    }
 </style>

src/PluginPane.svelte

@@ -18,10 +18,10 @@
     let applyAllPluse = 0;
     let isMaintenanceMode = false;
     async function requestUpdate() {
-        await addOn.updatePluginList();
+        await addOn.updatePluginList(true);
     }
     async function requestReload() {
-        await addOn.reloadPluginList();
+        await addOn.reloadPluginList(true);
     }
     pluginList.subscribe((e) => {
         list = e;
@@ -169,6 +169,7 @@
         align-items: center;
         border-top: 1px solid var(--background-modifier-border);
         padding: 4px;
+        flex-wrap: wrap;
     }
     .filerow {
         margin-left: 1.25em;
@@ -176,6 +177,7 @@
         justify-content: flex-start;
         align-items: center;
         padding-right: 4px;
+        flex-wrap: wrap;
     }
     .filerow.hideeven:has(.even),
     .labelrow.hideeven:has(.even) {
@@ -199,9 +201,11 @@
         flex-direction: row;
         justify-content: flex-end;
         margin-top: 8px;
+        flex-wrap: wrap;
     }
     .buttons > button {
         margin-left: 4px;
+        width: auto;
     }
 
     label {
@@ -212,6 +216,11 @@
     label > span {
         margin-right: 0.25em;
     }
+    :global(.is-mobile) .title,
+    :global(.is-mobile) .filetitle {
+        width: 100%;
+    }
+
     .center {
         display: flex;
         justify-content: center;

src/main.ts

@@ -4,7 +4,7 @@ import { Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "di
 import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, RequestUrlParam, RequestUrlResponse, requestUrl } from "./deps";
 import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, ConfigPassphraseStore, CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, DatabaseConnectingStatus, EntryHasPath, DocumentID, FilePathWithPrefix, FilePath, AnyEntry } from "./lib/src/types";
 import { InternalFileInfo, queueItem, CacheData, FileEventItem, FileWatchEventQueueMax } from "./types";
-import { getDocData, isDocContentSame } from "./lib/src/utils";
+import { getDocData, isDocContentSame, Parallels } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { LogDisplayModal } from "./LogDisplayModal";
@@ -87,7 +87,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     }
 
     processReplication = (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => this.parseReplicationResult(e);
-    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
+    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean, performSetup: boolean, skipInfo: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
         if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid";
         if (uri.toLowerCase() != uri) return "Remote URI and database name could not contain capital letters.";
         if (uri.indexOf(" ") !== -1) return "Remote URI and database name could not contain spaces.";
@@ -104,6 +104,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         const conf: PouchDB.HttpAdapter.HttpAdapterConfiguration = {
             adapter: "http",
             auth,
+            skip_setup: !performSetup,
             fetch: async (url: string | Request, opts: RequestInit) => {
                 let size = "";
                 const localURL = url.toString().substring(uri.length);
@@ -192,6 +193,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         if (passphrase !== "false" && typeof passphrase === "string") {
             enableEncryption(db, passphrase, useDynamicIterationCount);
         }
+        if (skipInfo) {
+            return { db: db, info: {} };
+        }
         try {
             const info = await db.info();
             return { db: db, info: info };
@@ -1350,7 +1354,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                 const docMtime = ~~(doc.mtime / 1000);
                 //TODO: some margin required.
                 if (localMtime >= docMtime) {
-                    Logger(`${doc._id} Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
+                    Logger(`${path} (${doc._id}, ${doc._rev}) Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
                     return;
                 }
             }
@@ -1361,12 +1365,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin
             missingChildren: [] as string[],
             timeout: now + this.chunkWaitTimeout,
         };
-        // If `Read chunks online` is enabled, retrieve chunks from the remote CouchDB directly.
+        // If `Read chunks online` is disabled, chunks should be transferred before here.
+        // However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
         if ((!this.settings.readChunksOnline) && "children" in doc) {
-            const c = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: doc.children, include_docs: false });
-            const missing = c.rows.filter((e) => "error" in e).map((e) => e.key);
-            // fetch from remote
-            if (missing.length > 0) Logger(`${doc._id}(${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
+            const c = await this.localDatabase.collectChunksWithCache(doc.children);
+            const missing = c.filter((e) => e.chunk === false).map((e) => e.id);
+            if (missing.length > 0) Logger(`${path} (${doc._id}, ${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
             newQueue.missingChildren = missing;
             this.queuedFiles.push(newQueue);
         } else {
@@ -1381,15 +1385,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         const docsSorted = docs.sort((a, b) => b.mtime - a.mtime);
         L1:
         for (const change of docsSorted) {
+            if (isChunk(change._id)) {
+                await this.parseIncomingChunk(change);
+                continue;
+            }
             for (const proc of this.addOns) {
                 if (await proc.parseReplicationResultItem(change)) {
                     continue L1;
                 }
             }
-            if (isChunk(change._id)) {
-                await this.parseIncomingChunk(change);
-                continue;
-            }
             if (change._id == SYNCINFO_ID) {
                 continue;
             }
@@ -1488,19 +1492,20 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                 }
                 return proc.substring(0, p);
             }
+
             const pendingTask = e.pending.length
-                ? "\nPending: " +
-                Object.entries(e.pending.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
-                    .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
-                    .join(", ")
-                : "";
+                ? e.pending.length < 10 ? ("\nPending: " +
+                    Object.entries(e.pending.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
+                        .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
+                        .join(", ")
+                ) : `\n Pending: ${e.pending.length}` : "";
 
             const runningTask = e.running.length
-                ? "\nRunning: " +
-                Object.entries(e.running.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
-                    .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
-                    .join(", ")
-                : "";
+                ? e.running.length < 10 ? ("\nRunning: " +
+                    Object.entries(e.running.reduce((p, c) => ({ ...p, [getProcKind(c)]: (p[getProcKind(c)] ?? 0) + 1 }), {} as { [key: string]: number }))
+                        .map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
+                        .join(", ")
+                ) : `\n Running: ${e.running.length}` : "";
             this.setStatusBarText(message + pendingTask + runningTask);
         })
     }
@@ -1509,27 +1514,25 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         return;
     }
 
-    logHideTimer: NodeJS.Timeout = null;
     setStatusBarText(message: string = null, log: string = null) {
         if (!this.statusBar) return;
         const newMsg = typeof message == "string" ? message : this.lastMessage;
         const newLog = typeof log == "string" ? log : this.lastLog;
         if (`${this.lastMessage}-${this.lastLog}` != `${newMsg}-${newLog}`) {
-            this.statusBar.setText(newMsg.split("\n")[0]);
+            scheduleTask("update-display", 50, () => {
+                this.statusBar.setText(newMsg.split("\n")[0]);
 
-            if (this.settings.showStatusOnEditor) {
-                const root = activeDocument.documentElement;
-                const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
-                q.forEach(e => e.setAttr("data-log", '' + (newMsg + "\n" + newLog) + ''))
-            } else {
-                const root = activeDocument.documentElement;
-                const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
-                q.forEach(e => e.setAttr("data-log", ''))
-            }
-            if (this.logHideTimer != null) {
-                clearTimeout(this.logHideTimer);
-            }
-            this.logHideTimer = setTimeout(() => this.setStatusBarText(null, ""), 3000);
+                if (this.settings.showStatusOnEditor) {
+                    const root = activeDocument.documentElement;
+                    const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+                    q.forEach(e => e.setAttr("data-log", '' + (newMsg + "\n" + newLog) + ''))
+                } else {
+                    const root = activeDocument.documentElement;
+                    const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+                    q.forEach(e => e.setAttr("data-log", ''))
+                }
+            }, true);
+            scheduleTask("log-hide", 3000, () => this.setStatusBarText(null, ""));
             this.lastMessage = newMsg;
             this.lastLog = newLog;
         }
@@ -1665,25 +1668,23 @@ Or if you are sure know what had been happened, we can unlock the database from
 
         const runAll = async<T>(procedureName: string, objects: T[], callback: (arg: T) => Promise<void>) => {
             Logger(procedureName);
-            const semaphore = Semaphore(25);
             if (!this.localDatabase.isReady) throw Error("Database is not ready!");
-            const processes = objects.map(e => (async (v) => {
-                const releaser = await semaphore.acquire(1, procedureName);
+            const para = Parallels();
+            for (const v of objects) {
+                await para.wait(10);
+                para.add((async (v) => {
+                    try {
+                        await callback(v);
+                    } catch (ex) {
+                        Logger(`Error while ${procedureName}`, LOG_LEVEL.NOTICE);
+                        Logger(ex);
+                    }
+                })(v));
 
-                try {
-                    await callback(v);
-                } catch (ex) {
-                    Logger(`Error while ${procedureName}`, LOG_LEVEL.NOTICE);
-                    Logger(ex);
-                } finally {
-                    releaser();
-                }
             }
-            )(e));
-            await Promise.all(processes);
-
+            await para.all();
             Logger(`${procedureName} done.`);
-        };
+        }
 
         await runAll("UPDATE DATABASE", onlyInStorage, async (e) => {
             Logger(`UPDATE DATABASE ${e.path}`);
@@ -2137,16 +2138,10 @@ Or if you are sure know what had been happened, we can unlock the database from
     conflictedCheckFiles: FilePath[] = [];
 
     // queueing the conflicted file check
-    conflictedCheckTimer: number;
-
     queueConflictedCheck(file: TFile) {
         this.conflictedCheckFiles = this.conflictedCheckFiles.filter((e) => e != file.path);
         this.conflictedCheckFiles.push(getPathFromTFile(file));
-        if (this.conflictedCheckTimer != null) {
-            window.clearTimeout(this.conflictedCheckTimer);
-        }
-        this.conflictedCheckTimer = window.setTimeout(async () => {
-            this.conflictedCheckTimer = null;
+        scheduleTask("check-conflict", 100, async () => {
             const checkFiles = JSON.parse(JSON.stringify(this.conflictedCheckFiles)) as FilePath[];
             for (const filename of checkFiles) {
                 try {
@@ -2158,7 +2153,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                     Logger(ex);
                 }
             }
-        }, 100);
+        });
     }
 
     async showIfConflicted(filename: FilePathWithPrefix) {

src/utils.ts

@@ -44,7 +44,10 @@ export function getPathFromTFile(file: TAbstractFile) {
 }
 
 const tasks: { [key: string]: ReturnType<typeof setTimeout> } = {};
-export function scheduleTask(key: string, timeout: number, proc: (() => Promise<any> | void)) {
+export function scheduleTask(key: string, timeout: number, proc: (() => Promise<any> | void), skipIfTaskExist?: boolean) {
+    if (skipIfTaskExist && key in tasks) {
+        return;
+    }
     cancelTask(key);
     tasks[key] = setTimeout(async () => {
         delete tasks[key];
@@ -663,6 +666,14 @@ export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryR
         return Number.parseInt((info as any)?.sizes?.[key] ?? 0);
     }
     await runWithLock("clean-up:remote", true, async () => {
+        const CHUNK_SIZE = 100;
+        function makeChunkedArrayFromArray<T>(items: T[]): T[][] {
+            const chunked = [];
+            for (let i = 0; i < items.length; i += CHUNK_SIZE) {
+                chunked.push(items.slice(i, i + CHUNK_SIZE));
+            }
+            return chunked;
+        }
         try {
             const ret = await plugin.replicator.connectRemoteCouchDBWithSetting(plugin.settings, plugin.isMobile);
             if (typeof ret === "string") {
@@ -701,14 +712,17 @@ export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryR
                 return;
             }
             Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
-            const rets = await _requestToCouchDBFetch(
-                `${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
-                plugin.settings.couchDB_USER,
-                plugin.settings.couchDB_PASSWORD,
-                "_purge",
-                payload, "POST");
-            // const result = await rets();
-            Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
+            const buffer = makeChunkedArrayFromArray(Object.entries(payload));
+            for (const chunkedPayload of buffer) {
+                const rets = await _requestToCouchDBFetch(
+                    `${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
+                    plugin.settings.couchDB_USER,
+                    plugin.settings.couchDB_PASSWORD,
+                    "_purge",
+                    chunkedPayload.reduce((p, c) => ({ ...p, [c[0]]: c[1] }), {}), "POST");
+                // const result = await rets();
+                Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
+            }
             Logger(`Compacting database...`, LOG_LEVEL.NOTICE, "clean-up-db");
             await db.compact();
             const endInfo = await db.info();

updates.md

@@ -14,4 +14,32 @@ I hope you will give it a try.
 
 #### Minors
 
+- 0.19.1
+  - Fixed: Fixed hidden file handling on Linux
+  - Improved: Now customization sync works more smoothly.
+- 0.19.2
+  - Fixed:
+    - Fixed garbage collection error while unreferenced chunks exist many.
+    - Fixed filename validation on Linux.
+  - Improved:
+    - Showing status is now thinned for performance.
+    - Enhance caching while collecting chunks.
+- 0.19.3
+  - Improved:
+    - Now replication will be paced by collecting chunks. If synchronisation has been deadlocked, please enable `Do not pace synchronization` once.
+- 0.19.4
+  - Improved:
+    - Reduced remote database checking to improve speed and reduce bandwidth.
+  - Fixed:
+    - Chunks which previously misinterpreted are now interpreted correctly.
+      - No more missing chunks which not be found forever, except if it has been actually missing.
+    - Deleted file detection on hidden file synchronising now works fine.
+    - Now the Customisation sync is surely quiet while it has been disabled.
+- 0.19.5
+  - Fixed:
+    - Now hidden file synchronisation would not be hanged, even if so many files exist.
+  - Improved:
+    - Customisation sync works more smoothly.
+  - Note: Concurrent processing has been rollbacked into the original implementation. As a result, the total number of processes is no longer shown next to the hourglass icon. However, only the processes that are running concurrently are shown.
+
 ... To continue on to `updates_old.md`.