bump

- Now replication will be paced by collecting chunks.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.19.0",
+    "version": "0.19.3",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.0",
+    "version": "0.19.3",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.19.0",
+            "version": "0.19.3",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.0",
+    "version": "0.19.3",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/CmdConfigSync.ts

@@ -10,7 +10,7 @@ import { WrappedNotice } from "./lib/src/wrapper";
 import { base64ToArrayBuffer, arrayBufferToBase64, readString, writeString, uint8ArrayToHexString } from "./lib/src/strbin";
 import { runWithLock } from "./lib/src/lock";
 import { LiveSyncCommands } from "./LiveSyncCommands";
-import { addPrefix, stripAllPrefixes } from "./lib/src/path";
+import { stripAllPrefixes } from "./lib/src/path";
 import { PeriodicProcessor, askYesNo, disposeMemoObject, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask } from "./utils";
 import { Semaphore } from "./lib/src/semaphore";
 import { PluginDialogModal } from "./dialogs";
@@ -67,6 +67,7 @@ export class ConfigSync extends LiveSyncCommands {
     pluginDialog: PluginDialogModal = null;
     periodicPluginSweepProcessor = new PeriodicProcessor(this.plugin, async () => await this.scanAllConfigFiles(false));
 
+    pluginList: PluginDataExDisplay[] = [];
     showPluginSyncModal() {
         if (!this.settings.usePluginSync) {
             return;
@@ -141,27 +142,31 @@ export class ConfigSync extends LiveSyncCommands {
         }
     }
     async onResume() {
-        if (this.plugin.suspended)
+        if (this.plugin.suspended) {
             return;
+        }
         if (this.settings.autoSweepPlugins) {
-            await this.scanAllConfigFiles(true);
+            await this.scanAllConfigFiles(false);
         }
         this.periodicPluginSweepProcessor.enable(this.settings.autoSweepPluginsPeriodic && !this.settings.watchInternalFileChanges ? (PERIODIC_PLUGIN_SWEEP * 1000) : 0);
-    }
 
-    async reloadPluginList() {
-        pluginList.set([])
-        await this.updatePluginList();
     }
-    async updatePluginList(updatedDocumentPath?: FilePathWithPrefix): Promise<void> {
+    async reloadPluginList(showMessage: boolean) {
+        this.pluginList = [];
+        pluginList.set(this.pluginList)
+        await this.updatePluginList(showMessage);
+    }
+    async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise<void> {
+        const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
         // pluginList.set([]);
         if (!this.settings.usePluginSync) {
-            pluginList.set([]);
+            this.pluginList = [];
+            pluginList.set(this.pluginList)
             return;
         }
 
         await runWithLock("update-plugin-list", false, async () => {
-            if (updatedDocumentPath != "") pluginList.update(e => e.filter(ee => ee.documentPath != updatedDocumentPath));
+            // if (updatedDocumentPath != "") pluginList.update(e => e.filter(ee => ee.documentPath != updatedDocumentPath));
             // const work: Record<string, Record<string, Record<string, Record<string, PluginDataEntryEx>>>> = {};
             const entries = [] as PluginDataExDisplay[]
             const plugins = this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true });
@@ -169,16 +174,21 @@ export class ConfigSync extends LiveSyncCommands {
             const processes = [] as Promise<void>[];
             let count = 0;
             pluginIsEnumerating.set(true);
+            let processed = false;
             try {
                 for await (const plugin of plugins) {
                     const path = plugin.path || this.getPath(plugin);
                     if (updatedDocumentPath && updatedDocumentPath != path) {
                         continue;
                     }
+                    processed = true;
+                    const oldEntry = (this.pluginList.find(e => e.documentPath == path));
+                    if (oldEntry && oldEntry.mtime == plugin.mtime) continue;
                     processes.push((async (v) => {
+
                         const release = await semaphore.acquire(1);
                         try {
-                            Logger(`Enumerating files... ${count++}`, LOG_LEVEL.NOTICE, "get-plugins");
+                            Logger(`Enumerating files... ${count++}`, logLevel, "get-plugins");
 
                             Logger(`plugin-${path}`, LOG_LEVEL.VERBOSE);
                             const wx = await this.localDatabase.getDBEntry(path, null, false, false);
@@ -199,7 +209,7 @@ export class ConfigSync extends LiveSyncCommands {
                             }
                         } catch (ex) {
                             //TODO
-                            Logger(`Something happened at enumerating customization :${v.path}`);
+                            Logger(`Something happened at enumerating customization :${v.path}`, LOG_LEVEL.NOTICE);
                             console.warn(ex);
                         } finally {
                             release();
@@ -208,16 +218,18 @@ export class ConfigSync extends LiveSyncCommands {
                     )(plugin));
                 }
                 await Promise.all(processes);
-                pluginList.update(e => {
-                    let newList = [...e];
-                    for (const item of entries) {
-                        console.log(item.documentPath);
-                        newList = newList.filter(x => x.documentPath != item.documentPath);
-                        newList.push(item)
-                    }
-                    return newList;
-                })
-                Logger(`All files enumerated`, LOG_LEVEL.NOTICE, "get-plugins");
+                let newList = [...this.pluginList];
+                for (const item of entries) {
+                    newList = newList.filter(x => x.documentPath != item.documentPath);
+                    newList.push(item)
+                }
+                if (updatedDocumentPath != "" && !processed) newList = newList.filter(e => e.documentPath != updatedDocumentPath);
+
+                this.pluginList = newList;
+                pluginList.set(newList);
+
+
+                Logger(`All files enumerated`, logLevel, "get-plugins");
             } finally {
                 pluginIsEnumerating.set(false);
             }
@@ -293,7 +305,7 @@ export class ConfigSync extends LiveSyncCommands {
             }
             const uPath = `${baseDir}/${loadedData.files[0].filename}` as FilePath;
             await this.storeCustomizationFiles(uPath);
-            await this.updatePluginList(uPath);
+            await this.updatePluginList(true, uPath);
             await delay(100);
             Logger(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL.NOTICE);
             if (data.category == "PLUGIN_DATA" || data.category == "PLUGIN_MAIN") {
@@ -329,6 +341,7 @@ export class ConfigSync extends LiveSyncCommands {
         try {
             if (data.documentPath) {
                 await this.deleteConfigOnDatabase(data.documentPath);
+                await this.updatePluginList(false, data.documentPath);
                 Logger(`Delete: ${data.documentPath}`, LOG_LEVEL.NOTICE);
             }
             return true;
@@ -338,8 +351,11 @@ export class ConfigSync extends LiveSyncCommands {
 
         }
     }
-    parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
+    async parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
         if (docs._id.startsWith(ICXHeader)) {
+            if (this.plugin.settings.usePluginSync) {
+                await this.updatePluginList(false, docs.path ? docs.path : this.getPath(docs));
+            }
             if (this.plugin.settings.usePluginSync && this.plugin.settings.notifyPluginOrSettingUpdated) {
                 if (!this.pluginDialog || (this.pluginDialog && !this.pluginDialog.isOpened())) {
                     const fragment = createFragment((doc) => {
@@ -347,7 +363,7 @@ export class ConfigSync extends LiveSyncCommands {
                             a.appendText(`Some configuration has been arrived, Press `);
                             a.appendChild(a.createEl("a", null, (anchor) => {
                                 anchor.text = "HERE";
-                                anchor.addEventListener("click", async () => {
+                                anchor.addEventListener("click", () => {
                                     this.showPluginSyncModal();
                                 });
                             }));
@@ -375,10 +391,7 @@ export class ConfigSync extends LiveSyncCommands {
                             disposeMemoObject(updatedPluginKey);
                         });
                     });
-                } else {
-                    this.updatePluginList(docs.path ? docs.path : this.getPath(docs));
                 }
-
             }
             return true;
         }
@@ -502,7 +515,9 @@ export class ConfigSync extends LiveSyncCommands {
             // Logger(`Configuration saving: ${prefixedFileName}`);
             if (dt.files.length == 0) {
                 Logger(`Nothing left: deleting.. ${path}`);
-                return await this.deleteConfigOnDatabase(prefixedFileName);
+                await this.deleteConfigOnDatabase(prefixedFileName);
+                await this.updatePluginList(false, prefixedFileName);
+                return
             }
 
             const content = stringifyYaml(dt);
@@ -540,6 +555,7 @@ export class ConfigSync extends LiveSyncCommands {
                     };
                 }
                 const ret = await this.localDatabase.putDBEntry(saveData);
+                await this.updatePluginList(false, saveData.path);
                 Logger(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
                 return ret;
             } catch (ex) {
@@ -548,7 +564,6 @@ export class ConfigSync extends LiveSyncCommands {
                 return false;
             }
         })
-        // })
 
     }
     async watchVaultRawEventsAsync(path: FilePath) {
@@ -591,6 +606,7 @@ export class ConfigSync extends LiveSyncCommands {
         for (const vp of deleteCandidate) {
             await this.deleteConfigOnDatabase(vp);
         }
+        this.updatePluginList(false).then(/* fire and forget */);
     }
     async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false) {
 
@@ -618,6 +634,7 @@ export class ConfigSync extends LiveSyncCommands {
                     };
                 }
                 await this.localDatabase.putRaw(saveData);
+                await this.updatePluginList(false, prefixedFileName);
                 Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Done`);
             } catch (ex) {
                 Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`);

src/ObsidianLiveSyncSettingTab.ts

@@ -1599,6 +1599,16 @@ ${stringifyYaml(pluginConfig)}`;
                 })
             );
 
+        new Setting(containerHatchEl)
+            .setName("Do not pace synchronization")
+            .setDesc("If this toggle enabled, synchronisation will not be paced by queued entries. If synchronisation has been deadlocked, please make this enabled once.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.doNotPaceReplication).onChange(async (value) => {
+                    this.plugin.settings.doNotPaceReplication = value;
+                    await this.plugin.saveSettings();
+                })
+            );
+
 
 
 

src/PluginCombo.svelte

@@ -207,21 +207,21 @@
         const local = list.find((e) => e.term == thisTerm);
         const selectedItem = list.find((e) => e.term == selected);
         if (selectedItem && (await applyData(selectedItem))) {
-            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(local.documentPath));
+            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(true, local.documentPath));
         }
     }
     async function compareSelected() {
         const local = list.find((e) => e.term == thisTerm);
         const selectedItem = list.find((e) => e.term == selected);
         if (local && selectedItem && (await compareData(local, selectedItem))) {
-            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(local.documentPath));
+            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(true, local.documentPath));
         }
     }
     async function deleteSelected() {
         const selectedItem = list.find((e) => e.term == selected);
-        const deletedPath = selectedItem.documentPath;
+        // const deletedPath = selectedItem.documentPath;
         if (selectedItem && (await deleteData(selectedItem))) {
-            scheduleTask("update-plugin-list", 250, () => addOn.updatePluginList(deletedPath));
+            scheduleTask("update-plugin-list", 250, () => addOn.reloadPluginList(true));
         }
     }
     async function duplicateItem() {
@@ -234,7 +234,7 @@
             }
             const key = `${plugin.app.vault.configDir}/${local.files[0].filename}`;
             await addOn.storeCustomizationFiles(key as FilePath, duplicateTermName);
-            await addOn.updatePluginList(addOn.filenameToUnifiedKey(key, duplicateTermName));
+            await addOn.updatePluginList(false, addOn.filenameToUnifiedKey(key, duplicateTermName));
         }
     }
 </script>
@@ -281,6 +281,7 @@
 
 <style>
     .spacer {
+        min-width: 1px;
         flex-grow: 1;
     }
     button {
@@ -311,4 +312,7 @@
         flex-direction: column;
         align-items: center;
     }
+    :global(.is-mobile) .spacer {
+        margin-left: auto;
+    }
 </style>

src/PluginPane.svelte

@@ -18,10 +18,10 @@
     let applyAllPluse = 0;
     let isMaintenanceMode = false;
     async function requestUpdate() {
-        await addOn.updatePluginList();
+        await addOn.updatePluginList(true);
     }
     async function requestReload() {
-        await addOn.reloadPluginList();
+        await addOn.reloadPluginList(true);
     }
     pluginList.subscribe((e) => {
         list = e;
@@ -169,6 +169,7 @@
         align-items: center;
         border-top: 1px solid var(--background-modifier-border);
         padding: 4px;
+        flex-wrap: wrap;
     }
     .filerow {
         margin-left: 1.25em;
@@ -176,6 +177,7 @@
         justify-content: flex-start;
         align-items: center;
         padding-right: 4px;
+        flex-wrap: wrap;
     }
     .filerow.hideeven:has(.even),
     .labelrow.hideeven:has(.even) {
@@ -199,9 +201,11 @@
         flex-direction: row;
         justify-content: flex-end;
         margin-top: 8px;
+        flex-wrap: wrap;
     }
     .buttons > button {
         margin-left: 4px;
+        width: auto;
     }
 
     label {
@@ -212,6 +216,11 @@
     label > span {
         margin-right: 0.25em;
     }
+    :global(.is-mobile) .title,
+    :global(.is-mobile) .filetitle {
+        width: 100%;
+    }
+
     .center {
         display: flex;
         justify-content: center;

src/main.ts

@@ -1350,7 +1350,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
                 const docMtime = ~~(doc.mtime / 1000);
                 //TODO: some margin required.
                 if (localMtime >= docMtime) {
-                    Logger(`${doc._id} Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
+                    Logger(`${path} (${doc._id}, ${doc._rev}) Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
                     return;
                 }
             }
@@ -1361,12 +1361,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin
             missingChildren: [] as string[],
             timeout: now + this.chunkWaitTimeout,
         };
-        // If `Read chunks online` is enabled, retrieve chunks from the remote CouchDB directly.
+        // If `Read chunks online` is disabled, chunks should be transferred before here.
+        // However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
         if ((!this.settings.readChunksOnline) && "children" in doc) {
-            const c = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: doc.children, include_docs: false });
-            const missing = c.rows.filter((e) => "error" in e).map((e) => e.key);
-            // fetch from remote
-            if (missing.length > 0) Logger(`${doc._id}(${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
+            const c = await this.localDatabase.collectChunksWithCache(doc.children)
+            const missing = c.filter((e) => !e.chunk).map((e) => e.id);
+            if (missing.length > 0) Logger(`${path} (${doc._id}, ${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
             newQueue.missingChildren = missing;
             this.queuedFiles.push(newQueue);
         } else {
@@ -1509,27 +1509,25 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         return;
     }
 
-    logHideTimer: NodeJS.Timeout = null;
     setStatusBarText(message: string = null, log: string = null) {
         if (!this.statusBar) return;
         const newMsg = typeof message == "string" ? message : this.lastMessage;
         const newLog = typeof log == "string" ? log : this.lastLog;
         if (`${this.lastMessage}-${this.lastLog}` != `${newMsg}-${newLog}`) {
-            this.statusBar.setText(newMsg.split("\n")[0]);
+            scheduleTask("update-display", 50, () => {
+                this.statusBar.setText(newMsg.split("\n")[0]);
 
-            if (this.settings.showStatusOnEditor) {
-                const root = activeDocument.documentElement;
-                const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
-                q.forEach(e => e.setAttr("data-log", '' + (newMsg + "\n" + newLog) + ''))
-            } else {
-                const root = activeDocument.documentElement;
-                const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
-                q.forEach(e => e.setAttr("data-log", ''))
-            }
-            if (this.logHideTimer != null) {
-                clearTimeout(this.logHideTimer);
-            }
-            this.logHideTimer = setTimeout(() => this.setStatusBarText(null, ""), 3000);
+                if (this.settings.showStatusOnEditor) {
+                    const root = activeDocument.documentElement;
+                    const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+                    q.forEach(e => e.setAttr("data-log", '' + (newMsg + "\n" + newLog) + ''))
+                } else {
+                    const root = activeDocument.documentElement;
+                    const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+                    q.forEach(e => e.setAttr("data-log", ''))
+                }
+            }, true);
+            scheduleTask("log-hide", 3000, () => this.setStatusBarText(null, ""));
             this.lastMessage = newMsg;
             this.lastLog = newLog;
         }
@@ -2137,16 +2135,10 @@ Or if you are sure know what had been happened, we can unlock the database from
     conflictedCheckFiles: FilePath[] = [];
 
     // queueing the conflicted file check
-    conflictedCheckTimer: number;
-
     queueConflictedCheck(file: TFile) {
         this.conflictedCheckFiles = this.conflictedCheckFiles.filter((e) => e != file.path);
         this.conflictedCheckFiles.push(getPathFromTFile(file));
-        if (this.conflictedCheckTimer != null) {
-            window.clearTimeout(this.conflictedCheckTimer);
-        }
-        this.conflictedCheckTimer = window.setTimeout(async () => {
-            this.conflictedCheckTimer = null;
+        scheduleTask("check-conflict", 100, async () => {
             const checkFiles = JSON.parse(JSON.stringify(this.conflictedCheckFiles)) as FilePath[];
             for (const filename of checkFiles) {
                 try {
@@ -2158,7 +2150,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                     Logger(ex);
                 }
             }
-        }, 100);
+        });
     }
 
     async showIfConflicted(filename: FilePathWithPrefix) {

src/utils.ts

@@ -44,7 +44,10 @@ export function getPathFromTFile(file: TAbstractFile) {
 }
 
 const tasks: { [key: string]: ReturnType<typeof setTimeout> } = {};
-export function scheduleTask(key: string, timeout: number, proc: (() => Promise<any> | void)) {
+export function scheduleTask(key: string, timeout: number, proc: (() => Promise<any> | void), skipIfTaskExist?: boolean) {
+    if (skipIfTaskExist && key in tasks) {
+        return;
+    }
     cancelTask(key);
     tasks[key] = setTimeout(async () => {
         delete tasks[key];
@@ -663,6 +666,14 @@ export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryR
         return Number.parseInt((info as any)?.sizes?.[key] ?? 0);
     }
     await runWithLock("clean-up:remote", true, async () => {
+        const CHUNK_SIZE = 100;
+        function makeChunkedArrayFromArray<T>(items: T[]): T[][] {
+            const chunked = [];
+            for (let i = 0; i < items.length; i += CHUNK_SIZE) {
+                chunked.push(items.slice(i, i + CHUNK_SIZE));
+            }
+            return chunked;
+        }
         try {
             const ret = await plugin.replicator.connectRemoteCouchDBWithSetting(plugin.settings, plugin.isMobile);
             if (typeof ret === "string") {
@@ -701,14 +712,17 @@ export const remoteDatabaseCleanup = async (plugin: ObsidianLiveSyncPlugin, dryR
                 return;
             }
             Logger(`Deleting unreferenced chunks: ${removeItems}`, LOG_LEVEL.NOTICE, "clean-up-db");
-            const rets = await _requestToCouchDBFetch(
-                `${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
-                plugin.settings.couchDB_USER,
-                plugin.settings.couchDB_PASSWORD,
-                "_purge",
-                payload, "POST");
-            // const result = await rets();
-            Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
+            const buffer = makeChunkedArrayFromArray(Object.entries(payload));
+            for (const chunkedPayload of buffer) {
+                const rets = await _requestToCouchDBFetch(
+                    `${plugin.settings.couchDB_URI}/${plugin.settings.couchDB_DBNAME}`,
+                    plugin.settings.couchDB_USER,
+                    plugin.settings.couchDB_PASSWORD,
+                    "_purge",
+                    chunkedPayload.reduce((p, c) => ({ ...p, [c[0]]: c[1] }), {}), "POST");
+                // const result = await rets();
+                Logger(JSON.stringify(await rets.json()), LOG_LEVEL.VERBOSE);
+            }
             Logger(`Compacting database...`, LOG_LEVEL.NOTICE, "clean-up-db");
             await db.compact();
             const endInfo = await db.info();

updates.md

@@ -14,4 +14,18 @@ I hope you will give it a try.
 
 #### Minors
 
+- 0.19.1
+  - Fixed: Fixed hidden file handling on Linux
+  - Improved: Now customization sync works more smoothly.
+- 0.19.2
+  - Fixed:
+    - Fixed garbage collection error while unreferenced chunks exist many.
+    - Fixed filename validation on Linux.
+  - Improved:
+    - Showing status is now thinned for performance.
+    - Enhance caching while collecting chunks.
+- 0.19.3
+  - Improved:
+    - Now replication will be paced by collecting chunks. If synchronisation has been deadlocked, please enable `Do not pace synchronization` once.
+
 ... To continue on to `updates_old.md`.