### Fixed

- Storage scanning no longer occurs when `Suspend file watching` is enabled (including boot-sequence).

### Improved
- Saving notes and files now consumes less memory.
- Chunk caching is now more efficient.
- Both of them (may) are effective for #692, #680, and some more.

### Changed
- `Incubate Chunks in Document` (also known as `Eden`) is now fully sunset.
- The `Compute revisions for chunks` setting has also been removed.
- As mentioned, `Memory cache size (by total characters)` has been removed.

### Refactored
- A significant refactoring of the core codebase is underway (please refer the release-note).

package.json

@@ -23,7 +23,8 @@
         "pretty": "npm run prettyNoWrite -- --write --log-level error",
         "prettyCheck": "npm run prettyNoWrite -- --check",
         "prettyNoWrite": "prettier --config ./.prettierrc \"**/*.js\" \"**/*.ts\" \"**/*.json\" ",
-        "check": "npm run lint && npm run svelte-check && npm run tsc-check"
+        "check": "npm run lint && npm run svelte-check && npm run tsc-check",
+        "unittest": "deno test -A --no-check --coverage=cov_profile  --v8-flags=--expose-gc  --trace-leaks  ./src/"
     },
     "keywords": [],
     "author": "vorotamoroz",
@@ -60,6 +61,7 @@
         "pouchdb-adapter-http": "^9.0.0",
         "pouchdb-adapter-idb": "^9.0.0",
         "pouchdb-adapter-indexeddb": "^9.0.0",
+        "pouchdb-adapter-memory": "^9.0.0",
         "pouchdb-core": "^9.0.0",
         "pouchdb-errors": "^9.0.0",
         "pouchdb-find": "^9.0.0",
@@ -75,7 +77,8 @@
         "tslib": "^2.8.1",
         "tsx": "^4.19.4",
         "typescript": "5.7.3",
-        "yaml": "^2.8.0"
+        "yaml": "^2.8.0",
+        "@types/deno": "^2.3.0"
     },
     "dependencies": {
         "@aws-sdk/client-s3": "^3.808.0",

src/common/KeyValueDB.ts

@@ -1,13 +1,5 @@
 import { deleteDB, type IDBPDatabase, openDB } from "idb";
-export interface KeyValueDatabase {
+import type { KeyValueDatabase } from "../lib/src/interfaces/KeyValueDatabase.ts";
-    get<T>(key: IDBValidKey): Promise<T>;
-    set<T>(key: IDBValidKey, value: T): Promise<IDBValidKey>;
-    del(key: IDBValidKey): Promise<void>;
-    clear(): Promise<void>;
-    keys(query?: IDBValidKey | IDBKeyRange, count?: number): Promise<IDBValidKey[]>;
-    close(): void;
-    destroy(): Promise<void>;
-}
 const databaseCache: { [key: string]: IDBPDatabase<any> } = {};
 export const OpenKeyValueDatabase = async (dbKey: string): Promise<KeyValueDatabase> => {
     if (dbKey in databaseCache) {

src/common/utils.ts

@@ -28,11 +28,12 @@ import type ObsidianLiveSyncPlugin from "../main.ts";
 import { writeString } from "../lib/src/string_and_binary/convert.ts";
 import { fireAndForget } from "../lib/src/common/utils.ts";
 import { sameChangePairs } from "./stores.ts";
-import type { KeyValueDatabase } from "./KeyValueDB.ts";
+
 import { scheduleTask } from "octagonal-wheels/concurrency/task";
 import { EVENT_PLUGIN_UNLOADED, eventHub } from "./events.ts";
 import { promiseWithResolver, type PromiseWithResolvers } from "octagonal-wheels/promises";
 import { AuthorizationHeaderGenerator } from "../lib/src/replication/httplib.ts";
+import type { KeyValueDatabase } from "../lib/src/interfaces/KeyValueDatabase.ts";
 
 export { scheduleTask, cancelTask, cancelAllTasks } from "../lib/src/concurrency/task.ts";
 

src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts

@@ -28,7 +28,7 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands implements IObsid
         return this.localDatabase.localDatabase;
     }
     clearHash() {
-        this.localDatabase.hashCaches.clear();
+        this.localDatabase.clearCaches();
     }
 
     async confirm(title: string, message: string, affirmative = "Yes", negative = "No") {

src/main.ts

@@ -27,7 +27,7 @@ import {
     LiveSyncAbstractReplicator,
     type LiveSyncReplicatorEnv,
 } from "./lib/src/replication/LiveSyncAbstractReplicator.js";
-import { type KeyValueDatabase } from "./common/KeyValueDB.ts";
+import { type KeyValueDatabase } from "./lib/src/interfaces/KeyValueDatabase.ts";
 import { LiveSyncCommands } from "./features/LiveSyncCommands.ts";
 import { HiddenFileSync } from "./features/HiddenFileSync/CmdHiddenFileSync.ts";
 import { ConfigSync } from "./features/ConfigSync/CmdConfigSync.ts";
@@ -591,7 +591,11 @@ export default class ObsidianLiveSyncPlugin
         throwShouldBeOverridden();
     }
 
-    $$initializeDatabase(showingNotice: boolean = false, reopenDatabase = true): Promise<boolean> {
+    $$initializeDatabase(
+        showingNotice: boolean = false,
+        reopenDatabase = true,
+        ignoreSuspending: boolean = false
+    ): Promise<boolean> {
         throwShouldBeOverridden();
     }
 
@@ -628,7 +632,7 @@ export default class ObsidianLiveSyncPlugin
         throwShouldBeOverridden();
     }
 
-    $$performFullScan(showingNotice?: boolean): Promise<void> {
+    $$performFullScan(showingNotice?: boolean, ignoreSuspending?: boolean): Promise<void> {
         throwShouldBeOverridden();
     }
 

src/modules/core/ModuleDatabaseFileAccess.ts

@@ -53,7 +53,7 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidia
             async () => await this.storeContent("autoTest.md" as FilePathWithPrefix, testString)
         );
         // For test, we need to clear the caches.
-        await this.localDatabase.hashCaches.clear();
+        this.localDatabase.clearCaches();
         await this._test("readContent", async () => {
             const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
             if (!content) return "File not found";

src/modules/core/ModuleRebuilder.ts

@@ -73,7 +73,7 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
         await this.core.$$realizeSettingSyncMode();
         await this.resetLocalDatabase();
         await delay(1000);
-        await this.core.$$initializeDatabase(true);
+        await this.core.$$initializeDatabase(true, true, true);
         await this.core.$$markRemoteLocked();
         await this.core.$$tryResetRemoteDatabase();
         await this.core.$$markRemoteLocked();
@@ -190,7 +190,7 @@ export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebu
         if (makeLocalChunkBeforeSync) {
             await this.core.fileHandler.createAllChunks(true);
         } else if (!preventMakeLocalFilesBeforeSync) {
-            await this.core.$$initializeDatabase(true);
+            await this.core.$$initializeDatabase(true, true, true);
         } else {
             // Do not create local file entries before sync (Means use remote information)
         }

src/modules/core/ModuleReplicator.ts

@@ -30,7 +30,7 @@ import {
 import { isAnyNote } from "../../lib/src/common/utils";
 import { EVENT_FILE_SAVED, EVENT_SETTING_SAVED, eventHub } from "../../common/events";
 import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
-import { globalSlipBoard } from "../../lib/src/bureau/bureau";
+
 import { $msg } from "../../lib/src/common/i18n";
 import { clearHandlers } from "../../lib/src/replication/SyncParamsHandler";
 
@@ -150,12 +150,12 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
                 }
 
                 await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
-                this.localDatabase.hashCaches.clear();
+                this.localDatabase.clearCaches();
                 // Perform the synchronisation once.
                 if (await this.core.replicator.openReplication(this.settings, false, showMessage, true)) {
                     await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
                     await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
-                    this.localDatabase.hashCaches.clear();
+                    this.localDatabase.clearCaches();
                     await this.core.$$getReplicator().markRemoteResolved(this.settings);
                     Logger("The local database has been cleaned up.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
                 } else {
@@ -310,7 +310,7 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
             const change = docs[0];
             if (!change) return;
             if (isChunk(change._id)) {
-                globalSlipBoard.submit("read-chunk", change._id, change as EntryLeaf);
+                this.localDatabase.onNewLeaf(change as EntryLeaf);
                 return;
             }
             if (await this.core.$anyModuleParsedReplicationResultItem(change)) return;

src/modules/essential/ModuleInitializerFile.ts

@@ -20,7 +20,7 @@ import { AbstractModule } from "../AbstractModule.ts";
 import type { ICoreModule } from "../ModuleTypes.ts";
 import { withConcurrency } from "octagonal-wheels/iterable/map";
 export class ModuleInitializerFile extends AbstractModule implements ICoreModule {
-    async $$performFullScan(showingNotice?: boolean): Promise<void> {
+    async $$performFullScan(showingNotice?: boolean, ignoreSuspending: boolean = false): Promise<void> {
         this._log("Opening the key-value database", LOG_LEVEL_VERBOSE);
         const isInitialized = (await this.core.kvDB.get<boolean>("initialized")) || false;
         // synchronize all files between database and storage.
@@ -34,6 +34,16 @@ export class ModuleInitializerFile extends AbstractModule implements ICoreModule
             }
             return;
         }
+        if (!ignoreSuspending && this.settings.suspendFileWatching) {
+            if (showingNotice) {
+                this._log(
+                    "Now suspending file watching. Synchronising between the storage and the local database is now prevented.",
+                    LOG_LEVEL_NOTICE,
+                    "syncAll"
+                );
+            }
+            return;
+        }
 
         if (showingNotice) {
             this._log("Initializing", LOG_LEVEL_NOTICE, "syncAll");
@@ -355,11 +365,15 @@ export class ModuleInitializerFile extends AbstractModule implements ICoreModule
         this._log(`Checking expired file history done`);
     }
 
-    async $$initializeDatabase(showingNotice: boolean = false, reopenDatabase = true): Promise<boolean> {
+    async $$initializeDatabase(
+        showingNotice: boolean = false,
+        reopenDatabase = true,
+        ignoreSuspending: boolean = false
+    ): Promise<boolean> {
         this.core.$$resetIsReady();
         if (!reopenDatabase || (await this.core.$$openDatabase())) {
             if (this.localDatabase.isReady) {
-                await this.core.$$performFullScan(showingNotice);
+                await this.core.$$performFullScan(showingNotice, ignoreSuspending);
             }
             if (!(await this.core.$everyOnDatabaseInitialized(showingNotice))) {
                 this._log(`Initializing database has been failed on some module`, LOG_LEVEL_NOTICE);

src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts

@@ -875,7 +875,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             }
             await purgeUnreferencedChunks(remoteDBConn.db, true, this.plugin.settings, false);
             await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, true);
-            this.plugin.localDatabase.hashCaches.clear();
+            this.plugin.localDatabase.clearCaches();
         });
     }
 
@@ -895,7 +895,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             }
             await purgeUnreferencedChunks(remoteDBConnection.db, false, this.plugin.settings, true);
             await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, false);
-            this.plugin.localDatabase.hashCaches.clear();
+            this.plugin.localDatabase.clearCaches();
             await balanceChunkPurgedDBs(this.plugin.localDatabase.localDatabase, remoteDBConnection.db);
             this.plugin.localDatabase.refreshSettings();
             Logger(

src/modules/features/SettingDialogue/PaneAdvanced.ts

@@ -6,7 +6,7 @@ import type { PageFunctions } from "./SettingPane.ts";
 export function paneAdvanced(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, { addPanel }: PageFunctions): void {
     void addPanel(paneEl, "Memory cache").then((paneEl) => {
         new Setting(paneEl).autoWireNumeric("hashCacheMaxCount", { clampMin: 10 });
-        new Setting(paneEl).autoWireNumeric("hashCacheMaxAmount", { clampMin: 1 });
+        // new Setting(paneEl).autoWireNumeric("hashCacheMaxAmount", { clampMin: 1 });
     });
     void addPanel(paneEl, "Local Database Tweak").then((paneEl) => {
         paneEl.addClass("wizardHidden");

src/modules/features/SettingDialogue/PaneHatch.ts

@@ -335,7 +335,7 @@ ${stringifyYaml({
                         Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify");
                         const ignorePatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesIgnorePatterns");
                         const targetPatterns = getFileRegExp(this.plugin.settings, "syncInternalFilesTargetPatterns");
-                        this.plugin.localDatabase.hashCaches.clear();
+                        this.plugin.localDatabase.clearCaches();
                         Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify");
                         const files = this.plugin.settings.syncInternalFiles
                             ? await this.plugin.storageAccess.getFilesIncludeHidden("/", targetPatterns, ignorePatterns)

src/modules/features/SettingDialogue/PanePatches.ts

@@ -28,9 +28,9 @@ export function panePatches(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElemen
     void addPanel(paneEl, "Compatibility (Database structure)").then((paneEl) => {
         new Setting(paneEl).autoWireToggle("useIndexedDBAdapter", { invert: true, holdValue: true });
 
-        new Setting(paneEl)
+        // new Setting(paneEl)
-            .autoWireToggle("doNotUseFixedRevisionForChunks", { holdValue: true })
+        //     .autoWireToggle("doNotUseFixedRevisionForChunks", { holdValue: true })
-            .setClass("wizardHidden");
+        //     .setClass("wizardHidden");
         new Setting(paneEl).autoWireToggle("handleFilenameCaseSensitive", { holdValue: true }).setClass("wizardHidden");
 
         this.addOnSaved("useIndexedDBAdapter", async () => {
@@ -99,13 +99,13 @@ export function panePatches(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElemen
     });
 
     void addPanel(paneEl, "Remote Database Tweak (In sunset)").then((paneEl) => {
-        new Setting(paneEl).autoWireToggle("useEden").setClass("wizardHidden");
+        // new Setting(paneEl).autoWireToggle("useEden").setClass("wizardHidden");
-        const onlyUsingEden = visibleOnly(() => this.isConfiguredAs("useEden", true));
+        // const onlyUsingEden = visibleOnly(() => this.isConfiguredAs("useEden", true));
-        new Setting(paneEl).autoWireNumeric("maxChunksInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
+        // new Setting(paneEl).autoWireNumeric("maxChunksInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
-        new Setting(paneEl)
+        // new Setting(paneEl)
-            .autoWireNumeric("maxTotalLengthInEden", { onUpdate: onlyUsingEden })
+        //     .autoWireNumeric("maxTotalLengthInEden", { onUpdate: onlyUsingEden })
-            .setClass("wizardHidden");
+        //     .setClass("wizardHidden");
-        new Setting(paneEl).autoWireNumeric("maxAgeInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
+        // new Setting(paneEl).autoWireNumeric("maxAgeInEden", { onUpdate: onlyUsingEden }).setClass("wizardHidden");
 
         new Setting(paneEl).autoWireToggle("enableCompression").setClass("wizardHidden");
     });

tsconfig.json

@@ -23,5 +23,5 @@
         }
     },
     "include": ["**/*.ts"],
-    "exclude": ["pouchdb-browser-webpack", "utils", "src/lib/apps"]
+    "exclude": ["pouchdb-browser-webpack", "utils", "src/lib/apps", "**/*.test.ts"]
 }