bump

- Auto chunk size adjusting.
  Now our large files are processed more efficiently
- These configuration has been removed.

Improved
- Remote chunk retrieving logic has been speeded up.

Fixed
- Fixed process handling of boot sequence

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.14.2",
+    "version": "0.14.5",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.14.2",
+    "version": "0.14.5",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.14.2",
+            "version": "0.14.5",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.14.2",
+    "version": "0.14.5",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/LocalPouchDB.ts

@@ -615,7 +615,8 @@ export class LocalPouchDB {
         let processed = 0;
         let made = 0;
         let skiped = 0;
-        let pieceSize = MAX_DOC_SIZE_BIN * Math.max(this.settings.customChunkSize, 1);
+        const maxChunkSize = MAX_DOC_SIZE_BIN * Math.max(this.settings.customChunkSize, 1);
+        let pieceSize = maxChunkSize;
         let plainSplit = false;
         let cacheUsed = 0;
         const userpasswordHash = this.h32Raw(new TextEncoder().encode(this.settings.passphrase));
@@ -624,20 +625,11 @@ export class LocalPouchDB {
             plainSplit = true;
         }
 
+        const minimumChunkSize = Math.min(Math.max(40, ~~(note.data.length / 100)), maxChunkSize);
+        if (pieceSize < minimumChunkSize) pieceSize = minimumChunkSize;
         const newLeafs: EntryLeaf[] = [];
-        // To keep low bandwith and database size,
-        // Dedup pieces on database.
-        // from 0.1.10, for best performance. we use markdown delimiters
-        // 1. \n[^\n]{longLineThreshold}[^\n]*\n -> long sentence shuld break.
-        // 2. \n\n shold break
-        // 3. \r\n\r\n should break
-        // 4. \n# should break.
-        let minimumChunkSize = this.settings.minimumChunkSize;
-        if (minimumChunkSize < 10) minimumChunkSize = 10;
-        let longLineThreshold = this.settings.longLineThreshold;
-        if (longLineThreshold < 100) longLineThreshold = 100;
 
-        const pieces = splitPieces2(note.data, pieceSize, plainSplit, minimumChunkSize, longLineThreshold);
+        const pieces = splitPieces2(note.data, pieceSize, plainSplit, minimumChunkSize, 0);
         for (const piece of pieces()) {
             processed++;
             let leafid = "";
@@ -1380,8 +1372,21 @@ export class LocalPouchDB {
         if (remoteChunks.rows.some(e => "error" in e)) {
             return false;
         }
+
+        const remoteChunkItems = remoteChunks.rows.map(e => e.doc);
+        const max = remoteChunkItems.length;
+        let last = 0;
+        // Chunks should be ordered by as we requested.
+        function findChunk(key: string) {
+            const offset = last;
+            for (let i = 0; i < max; i++) {
+                const idx = (offset + i) % max;
+                last = i;
+                if (remoteChunkItems[idx]._id == key) return remoteChunkItems[idx];
+            }
+            throw Error("Chunk collecting error");
+        }
         // Merge them
-        const chunkMap: { [key: string]: EntryDoc } = remoteChunks.rows.reduce((p, c) => ({ ...p, [c.key]: c.doc }), {})
-        return localChunks.rows.map(e => ("error" in e) ? (chunkMap[e.key]) : e.doc);
+        return localChunks.rows.map(e => ("error" in e) ? (findChunk(e.key)) : e.doc);
     }
 }

src/ObsidianLiveSyncSettingTab.ts

@@ -469,6 +469,22 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                                 } else {
                                     addResult("✔ httpd.enable_cors is ok.");
                                 }
+                                // If the server is not cloudant, configure request size
+                                if (!this.plugin.settings.couchDB_URI.contains(".cloudantnosqldb.")) {
+                                    // REQUEST SIZE
+                                    if (Number(responseConfig?.chttpd?.max_http_request_size ?? 0) < 4294967296) {
+                                        addResult("❗ chttpd.max_http_request_size is low)");
+                                        addConfigFixButton("Set chttpd.max_http_request_size", "chttpd/max_http_request_size", "4294967296");
+                                    } else {
+                                        addResult("✔ chttpd.max_http_request_size is ok.");
+                                    }
+                                    if (Number(responseConfig?.couchdb?.max_document_size ?? 0) < 50000000) {
+                                        addResult("❗ couchdb.max_document_size is low)");
+                                        addConfigFixButton("Set couchdb.max_document_size", "couchdb/max_document_size", "50000000");
+                                    } else {
+                                        addResult("✔ couchdb.max_document_size is ok.");
+                                    }
+                                }
                                 // CORS check
                                 //  checking connectivity for mobile
                                 if (responseConfig?.cors?.credentials != "true") {
@@ -583,43 +599,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
                     })
             )
 
-        containerLocalDatabaseEl.createEl("div", {
-            text: sanitizeHTMLToDom(`Advanced settings<br>
-                Configuration of how LiveSync makes chunks from the file.`),
-        });
-        new Setting(containerLocalDatabaseEl)
-            .setName("Minimum chunk size")
-            .setDesc("(letters), minimum chunk size.")
-            .addText((text) => {
-                text.setPlaceholder("")
-                    .setValue(this.plugin.settings.minimumChunkSize + "")
-                    .onChange(async (value) => {
-                        let v = Number(value);
-                        if (isNaN(v) || v < 10 || v > 1000) {
-                            v = 10;
-                        }
-                        this.plugin.settings.minimumChunkSize = v;
-                        await this.plugin.saveSettings();
-                    });
-                text.inputEl.setAttribute("type", "number");
-            });
 
-        new Setting(containerLocalDatabaseEl)
-            .setName("LongLine Threshold")
-            .setDesc("(letters), If the line is longer than this, make the line to chunk")
-            .addText((text) => {
-                text.setPlaceholder("")
-                    .setValue(this.plugin.settings.longLineThreshold + "")
-                    .onChange(async (value) => {
-                        let v = Number(value);
-                        if (isNaN(v) || v < 10 || v > 1000) {
-                            v = 10;
-                        }
-                        this.plugin.settings.longLineThreshold = v;
-                        await this.plugin.saveSettings();
-                    });
-                text.inputEl.setAttribute("type", "number");
-            });
         let newDatabaseName = this.plugin.settings.additionalSuffixOfDatabaseName + "";
         new Setting(containerLocalDatabaseEl)
             .setName("Database suffix")

src/main.ts

@@ -402,6 +402,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                             }
                         }
                         let initDB;
+                        this.settings = newSettingW;
                         await this.saveSettings();
                         if (keepLocalDB == "no") {
                             this.resetLocalOldDatabase();
@@ -1617,18 +1618,20 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             const count = objects.length;
             Logger(procedurename);
             let i = 0;
-            // let lastTicks = performance.now() + 2000;
-            // let workProcs = 0;
             const p = Parallels();
             const limit = 10;
 
             Logger(`${procedurename} exec.`);
             for (const v of objects) {
-                // workProcs++;
                 if (!this.localDatabase.isReady) throw Error("Database is not ready!");
-                p.add(callback(v).then(() => {
+                const addProc = (p: () => Promise<void>): Promise<unknown> => {
+                    return p();
+                }
+                p.add(addProc(async () => {
+                    try {
+                        await callback(v);
                     i++;
-                    if (i % 100 == 0) {
+                        if (i % 50 == 0) {
                         const notify = `${procedurename} : ${i}/${count}`;
                         if (showingNotice) {
                             Logger(notify, LOG_LEVEL.NOTICE, "syncAll");
@@ -1637,13 +1640,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                         }
                         this.setStatusBarText(notify);
                     }
-                }).catch(ex => {
+                    } catch (ex) {
                     Logger(`Error while ${procedurename}`, LOG_LEVEL.NOTICE);
                     Logger(ex);
-                }).finally(() => {
-                    // workProcs--;
-                })
-                );
+                    }
+                }));
                 await p.wait(limit);
             }
             await p.all();
@@ -1659,6 +1660,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             await runAll("UPDATE STORAGE", onlyInDatabase, async (e) => {
                 Logger(`Check or pull from db:${e}`);
                 await this.pullFile(e, filesStorage, false, null, false);
+                Logger(`Check or pull from db:${e} OK`);
             });
         }
         if (!initialScan) {

updates.md

@@ -11,8 +11,13 @@
 
 #### Recommended configuration for Self-hosted CouchDB
 - Set chunk size to around 100 to 250 (10MB - 25MB per chunk)
+- *Set batch size to 100 and batch limit to 20 (0.14.2)*
 - Be sure to `Read chunks online` checked.
 
+#### Minors
+- 0.14.2 Fixed issue about retrieving files if synchronisation has been interrupted or failed
+- 0.14.3 New test items have been added to `Check database configuration`.
+- 0.14.4 Fixed issue of importing configurations.
 
 ### 0.13.0