Improved:

- Now notes will be split into chunks in the background thread to improve smoothness.
  - Default enabled, to disable, toggle `Do not split chunks in the foreground` on `Hatch` -> `Compatibility`.
  - If you want to process very small notes in the foreground, please enable `Process small files in the foreground` on `Hatch` -> `Compatibility`.
- We can use a `splitting-limit-capped chunk splitter`; which performs more simple and make less amount of chunks.
  - Default disabled, to enable, toggle `Use splitting-limit-capped chunk splitter` on `Sync settings` -> `Performance tweaks`
Tidied
  - Some files have been separated into multiple files to make them more explicit in what they are responsible for.

src/common/utils.ts

@@ -6,7 +6,7 @@ import { LOG_LEVEL_VERBOSE, type AnyEntry, type DocumentID, type EntryHasPath, t
 import { CHeader, ICHeader, ICHeaderLength, ICXHeader, PSCHeader } from "./types.ts";
 import { InputStringDialog, PopoverSelectString } from "./dialogs.ts";
 import type ObsidianLiveSyncPlugin from "../main.ts";
-import { writeString } from "../lib/src/string_and_binary/strbin.ts";
+import { writeString } from "../lib/src/string_and_binary/convert.ts";
 import { fireAndForget } from "../lib/src/common/utils.ts";
 import { sameChangePairs } from "./stores.ts";
 

src/features/CmdConfigSync.ts

@@ -6,7 +6,8 @@ import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } f
 import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "../common/types.ts";
 import { createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocDataAsArray, isDocContentSame } from "../lib/src/common/utils.ts";
 import { Logger } from "../lib/src/common/logger.ts";
-import { readString, decodeBinary, arrayBufferToBase64, digestHash } from "../lib/src/string_and_binary/strbin.ts";
+import { digestHash } from "../lib/src/string_and_binary/hash.ts";
+import { arrayBufferToBase64, decodeBinary, readString } from 'src/lib/src/string_and_binary/convert.ts';
 import { serialized, shareRunningResult } from "../lib/src/concurrency/lock.ts";
 import { LiveSyncCommands } from "./LiveSyncCommands.ts";
 import { stripAllPrefixes } from "../lib/src/string_and_binary/path.ts";

src/main.ts

@@ -15,7 +15,7 @@ import { encrypt, tryDecrypt } from "./lib/src/encryption/e2ee_v2.ts";
 import { balanceChunkPurgedDBs, enableCompression, enableEncryption, isCloudantURI, isErrorOfMissingDoc, isValidRemoteCouchDBURI, purgeUnreferencedChunks } from "./lib/src/pouchdb/utils_couchdb.ts";
 import { logStore, type LogEntry, collectingChunks, pluginScanningCount, hiddenFilesProcessingCount, hiddenFilesEventCount, logMessages } from "./lib/src/mock_and_interop/stores.ts";
 import { setNoticeClass } from "./lib/src/mock_and_interop/wrapper.ts";
-import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/string_and_binary/strbin.ts";
+import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/string_and_binary/convert.ts";
 import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/string_and_binary/path.ts";
 import { isLockAcquired, serialized, shareRunningResult, skipIfDuplicated } from "./lib/src/concurrency/lock.ts";
 import { StorageEventManager, StorageEventManagerObsidian } from "./storages/StorageEventManager.ts";
@@ -41,6 +41,8 @@ import type { CheckPointInfo } from "./lib/src/replication/journal/JournalSyncTy
 import { ObsHttpHandler } from "./common/ObsHttpHandler.js";
 import { TestPaneView, VIEW_TYPE_TEST } from "./tests/TestPaneView.js"
 import { $f, __onMissingTranslation, setLang } from "./lib/src/common/i18n.ts";
+import { enableTestFunction } from "./tests/testUtils.ts";
+import { terminateWorker } from "./lib/src/worker/splitWorker.ts";
 
 
 setNoticeClass(Notice);
@@ -855,6 +857,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
         );
         // eslint-disable-next-line no-unused-labels
         TEST: {
+            enableTestFunction(this);
             this.registerView(
                 VIEW_TYPE_TEST,
                 (leaf) => new TestPaneView(leaf, this)
@@ -870,9 +873,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
                     this.showView(VIEW_TYPE_TEST);
                 }
             });
-
         }
-
     }
 
     async onload() {
@@ -982,6 +983,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
     }
 
     onunload() {
+        terminateWorker();
         cancelAllPeriodicTask();
         cancelAllTasks();
         stopAllRunningProcessors();

src/tests/testUtils.ts

@@ -0,0 +1,45 @@
+import { fireAndForget } from "src/lib/src/common/utils";
+import { serialized } from "src/lib/src/concurrency/lock";
+import type ObsidianLiveSyncPlugin from "src/main";
+
+let plugin: ObsidianLiveSyncPlugin;
+export function enableTestFunction(plugin_: ObsidianLiveSyncPlugin) {
+    plugin = plugin_;
+}
+export function addDebugFileLog(message: any, stackLog = false) {
+    fireAndForget(serialized("debug-log", async () => {
+        const now = new Date();
+        const filename = `debug-log`
+        const time = now.toISOString().split("T")[0];
+        const outFile = `${filename}${time}.jsonl`;
+        // const messageContent = typeof message == "string" ? message : message instanceof Error ? `${message.name}:${message.message}` : JSON.stringify(message, null, 2);
+        const timestamp = now.toLocaleString();
+        const timestampEpoch = now;
+        let out = { "timestamp": timestamp, epoch: timestampEpoch, } as Record<string, any>;
+        if (message instanceof Error) {
+            // debugger;
+            // console.dir(message.stack);
+            out = { ...out, message };
+        } else if (stackLog) {
+            if (stackLog) {
+                const stackE = new Error();
+                const stack = stackE.stack;
+                out = { ...out, stack }
+            }
+        }
+        if (typeof message == "object") {
+            out = { ...out, ...message, }
+        } else {
+            out = {
+                result: message
+            }
+        }
+        // const out = "--" + timestamp + "--\n" + messageContent + " " + (stack || "");
+        // const out
+        try {
+            await plugin.vaultAccess.adapterAppend(plugin.app.vault.configDir + "/ls-debug/" + outFile, JSON.stringify(out) + "\n")
+        } catch (ex) {
+            //NO OP
+        }
+    }));
+}

src/ui/ConflictResolveModal.ts

@@ -1,7 +1,7 @@
 import { App, Modal } from "../deps.ts";
 import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT } from "diff-match-patch";
 import { CANCELLED, LEAVE_TO_SUBSEQUENT, RESULT_TIMED_OUT, type diff_result } from "../lib/src/common/types.ts";
-import { escapeStringToHTML } from "../lib/src/string_and_binary/strbin.ts";
+import { escapeStringToHTML } from "../lib/src/string_and_binary/convert.ts";
 import { delay, sendValue, waitForValue } from "../lib/src/common/utils.ts";
 
 export type MergeDialogResult = typeof LEAVE_TO_SUBSEQUENT | typeof CANCELLED | string;

src/ui/DocumentHistoryModal.ts

@@ -1,6 +1,6 @@
 import { TFile, Modal, App, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "../deps.ts";
 import { getPathFromTFile, isValidPath } from "../common/utils.ts";
-import { decodeBinary, escapeStringToHTML, readString } from "../lib/src/string_and_binary/strbin.ts";
+import { decodeBinary, escapeStringToHTML, readString } from "../lib/src/string_and_binary/convert.ts";
 import ObsidianLiveSyncPlugin from "../main.ts";
 import { type DocumentID, type FilePathWithPrefix, type LoadedEntry, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "../lib/src/common/types.ts";
 import { Logger } from "../lib/src/common/logger.ts";

src/ui/JsonResolvePane.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
     import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "../deps";
     import type { FilePath, LoadedEntry } from "../lib/src/common/types";
-    import { decodeBinary, readString } from "../lib/src/string_and_binary/strbin";
+    import { decodeBinary, readString } from "../lib/src/string_and_binary/convert";
     import { getDocData } from "../lib/src/common/utils";
     import { mergeObject } from "../common/utils";
 

src/ui/ObsidianLiveSyncSettingTab.ts

@@ -22,7 +22,7 @@ import {
     type ConfigurationItem
 } from "../lib/src/common/types.ts";
 import { createBlob, delay, isDocContentSame, isObjectDifferent, readAsBlob, unique } from "../lib/src/common/utils.ts";
-import { versionNumberString2Number } from "../lib/src/string_and_binary/strbin.ts";
+import { versionNumberString2Number } from "../lib/src/string_and_binary/convert.ts";
 import { Logger } from "../lib/src/common/logger.ts";
 import { checkSyncInfo, isCloudantURI } from "../lib/src/pouchdb/utils_couchdb.ts";
 import { testCrypt } from "../lib/src/encryption/e2ee_v2.ts";
@@ -1748,6 +1748,10 @@ However, your report is needed to stabilise this. I appreciate you for your grea
             .setClass("wizardHidden")
             .autoWireToggle("readChunksOnline", { onUpdate: onlyOnCouchDB })
 
+        new Setting(containerSyncSettingEl)
+            .setClass("wizardHidden")
+            .autoWireToggle("enableChunkSplitterV2")
+
         this.createEl(containerSyncSettingEl, "h4", {
             text: sanitizeHTMLToDom(`Targets`),
         }).addClass("wizardHidden");
@@ -2184,6 +2188,14 @@ ${stringifyYaml(pluginConfig)}`;
         new Setting(containerHatchEl)
             .autoWireToggle("disableCheckingConfigMismatch")
 
+        new Setting(containerHatchEl)
+            .autoWireToggle("disableWorkerForGeneratingChunks")
+
+        new Setting(containerHatchEl)
+            .autoWireToggle("processSmallFilesInUIThread", {
+                onUpdate: visibleOnly(() => this.isConfiguredAs("disableWorkerForGeneratingChunks", false))
+            })
+
         addScreenElement("50", containerHatchEl);
 
 

src/ui/components/PluginCombo.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
     import type { PluginDataExDisplay } from "../../features/CmdConfigSync";
     import { Logger } from "../../lib/src/common/logger";
-    import { versionNumberString2Number } from "../../lib/src/string_and_binary/strbin";
+    import { versionNumberString2Number } from "../../lib/src/string_and_binary/convert";
     import { type FilePath, LOG_LEVEL_NOTICE } from "../../lib/src/common/types";
     import { getDocData } from "../../lib/src/common/utils";
     import type ObsidianLiveSyncPlugin from "../../main";

src/ui/settingConstants.ts

@@ -300,6 +300,18 @@ export const SettingInformation: Partial<Record<keyof AllSettings, Configuration
     "displayLanguage": {
         "name": "Display Language",
         "desc": "Not all messages have been translated. And, please revert to \"Default\" when reporting errors."
+    },
+    enableChunkSplitterV2: {
+        name: "Use splitting-limit-capped chunk splitter",
+        desc: "If enabled, chunks will be split into no more than 100 items. However, dedupe is slightly weaker."
+    },
+    disableWorkerForGeneratingChunks: {
+        name: "Do not split chunks in the foreground",
+        desc: "If disabled(toggled), chunks will be split on the UI thread (Previous behaviour)."
+    },
+    processSmallFilesInUIThread: {
+        name: "Process small files in the foreground",
+        desc: "If enabled, the file under 1kb will be processed in the UI thread."
     }
 }
 function translateInfo(infoSrc: ConfigurationItem | undefined | false) {