### New feature

- We can analyse the local database with the `Analyse database usage` command.
- We can reset the notification threshold and check the remote usage at once with the `Reset notification threshold and check the remote database usage` command.

### Fixed

- Now the plug-in resets the remote size notification threshold after rebuild.

src/common/events.ts

@@ -23,6 +23,8 @@ export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor";
 export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete";
 export const EVENT_ON_UNRESOLVED_ERROR = "on-unresolved-error";
 
+export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage";
+export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size";
 // export const EVENT_FILE_CHANGED = "file-changed";
 
 declare global {
@@ -42,6 +44,8 @@ declare global {
         [EVENT_REQUEST_RUN_DOCTOR]: string;
         [EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined;
         [EVENT_ON_UNRESOLVED_ERROR]: undefined;
+        [EVENT_ANALYSE_DB_USAGE]: undefined;
+        [EVENT_REQUEST_CHECK_REMOTE_SIZE]: undefined;
     }
 }
 

src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts

@@ -7,12 +7,14 @@ import {
     type DocumentID,
     type EntryDoc,
     type EntryLeaf,
+    type FilePathWithPrefix,
     type MetaEntry,
 } from "../../lib/src/common/types";
 import { getNoFromRev } from "../../lib/src/pouchdb/LiveSyncLocalDB";
 import { LiveSyncCommands } from "../LiveSyncCommands";
 import { serialized } from "octagonal-wheels/concurrency/lock_v2";
 import { arrayToChunkedArray } from "octagonal-wheels/collection";
+import { EVENT_ANALYSE_DB_USAGE, eventHub } from "@/common/events";
 const DB_KEY_SEQ = "gc-seq";
 const DB_KEY_CHUNK_SET = "chunk-set";
 const DB_KEY_DOC_USAGE_MAP = "doc-usage-map";
@@ -27,6 +29,15 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands {
     }
     onload(): void | Promise<void> {
         // NO OP.
+        this.plugin.addCommand({
+            id: "analyse-database",
+            name: "Analyse Database Usage (advanced)",
+            icon: "database-search",
+            callback: async () => {
+                await this.analyseDatabase();
+            },
+        });
+        eventHub.onEvent(EVENT_ANALYSE_DB_USAGE, () => this.analyseDatabase());
     }
     async allChunks(includeDeleted: boolean = false) {
         const p = this._progress("", LOG_LEVEL_NOTICE);
@@ -485,4 +496,216 @@ Success: ${successCount}, Errored: ${errored}`;
         const kvDB = this.plugin.kvDB;
         await kvDB.set(DB_KEY_CHUNK_SET, chunkSet);
     }
+
+    // Analyse the database and report chunk usage.
+    async analyseDatabase() {
+        if (!this.isAvailable()) return;
+        const db = this.localDatabase.localDatabase;
+        // Map of chunk ID to its info
+        type ChunkInfo = {
+            id: DocumentID;
+            refCount: number;
+            length: number;
+        };
+        const chunkMap = new Map<DocumentID, Set<ChunkInfo>>();
+        // Map of document ID to its info
+        type DocumentInfo = {
+            id: DocumentID;
+            rev: Rev;
+            chunks: Set<ChunkID>;
+            uniqueChunks: Set<ChunkID>;
+            sharedChunks: Set<ChunkID>;
+            path: FilePathWithPrefix;
+        };
+        const docMap = new Map<DocumentID, Set<DocumentInfo>>();
+        const info = await db.info();
+        // Total number of revisions to process (approximate)
+        const maxSeq = new Number(info.update_seq);
+        let processed = 0;
+        let read = 0;
+        let errored = 0;
+        // Fetch Tasks
+        const ft = [] as ReturnType<typeof fetchRevision>[];
+        // Fetch a specific revision of a document and make note of its chunks, or add chunk info.
+        const fetchRevision = async (id: DocumentID, rev: Rev, seq: string | number) => {
+            try {
+                processed++;
+                const doc = await db.get(id, { rev: rev });
+                if (doc) {
+                    if ("children" in doc) {
+                        const id = doc._id;
+                        const rev = doc._rev;
+                        const children = (doc.children || []) as DocumentID[];
+                        const set = docMap.get(id) || new Set();
+                        set.add({
+                            id,
+                            rev,
+                            chunks: new Set(children),
+                            uniqueChunks: new Set(),
+                            sharedChunks: new Set(),
+                            path: doc.path,
+                        });
+                        docMap.set(id, set);
+                    } else if (doc.type === EntryTypes.CHUNK) {
+                        const id = doc._id as DocumentID;
+                        if (chunkMap.has(id)) {
+                            return;
+                        }
+                        if (doc._deleted) {
+                            // Deleted chunk, skip (possibly resurrected later)
+                            return;
+                        }
+                        const length = doc.data.length;
+                        const set = chunkMap.get(id) || new Set();
+                        set.add({ id, length, refCount: 0 });
+                        chunkMap.set(id, set);
+                    }
+                    read++;
+                } else {
+                    this._log(`Analysing Database: not found: ${id} / ${rev}`);
+                    errored++;
+                }
+            } catch (error) {
+                this._log(`Error fetching document ${id} / ${rev}: $`, LOG_LEVEL_NOTICE);
+                this._log(error, LOG_LEVEL_VERBOSE);
+                errored++;
+            }
+            if (processed % 100 == 0) {
+                this._log(`Analysing database: ${read} (${errored}) / ${maxSeq} `, LOG_LEVEL_NOTICE, "db-analyse");
+            }
+        };
+
+        // Enumerate all documents and their revisions.
+        const IDs = this.localDatabase.findEntryNames("", "", {});
+        for await (const id of IDs) {
+            const revList = await this.localDatabase.getRaw(id as DocumentID, {
+                revs: true,
+                revs_info: true,
+                conflicts: true,
+            });
+            const revInfos = revList._revs_info || [];
+            for (const revInfo of revInfos) {
+                // All available revisions should be processed.
+                // If the revision is not available, it means the revision is already tombstoned.
+                if (revInfo.status == "available") {
+                    // Schedule fetch task
+                    ft.push(fetchRevision(id as DocumentID, revInfo.rev, 0));
+                }
+            }
+        }
+        // Wait for all fetch tasks to complete.
+        await Promise.all(ft);
+        // Reference count marking and unique/shared chunk classification.
+        for (const [, docRevs] of docMap) {
+            for (const docRev of docRevs) {
+                for (const chunkId of docRev.chunks) {
+                    const chunkInfos = chunkMap.get(chunkId);
+                    if (chunkInfos) {
+                        for (const chunkInfo of chunkInfos) {
+                            if (chunkInfo.refCount === 0) {
+                                docRev.uniqueChunks.add(chunkId);
+                            } else {
+                                docRev.sharedChunks.add(chunkId);
+                            }
+                            chunkInfo.refCount++;
+                        }
+                    }
+                }
+            }
+        }
+        // Prepare results
+        const result = [];
+        // Calculate total size of chunks in the given set.
+        const getTotalSize = (ids: Set<DocumentID>) => {
+            return [...ids].reduce((acc, chunkId) => {
+                const chunkInfos = chunkMap.get(chunkId);
+                if (chunkInfos) {
+                    for (const chunkInfo of chunkInfos) {
+                        acc += chunkInfo.length;
+                    }
+                }
+                return acc;
+            }, 0);
+        };
+
+        // Compile results for each document revision
+        for (const doc of docMap.values()) {
+            for (const rev of doc) {
+                const title = `${rev.path} (${rev.rev})`;
+                const id = rev.id;
+                const revStr = `${getNoFromRev(rev.rev)}`;
+                const revHash = rev.rev.split("-")[1].substring(0, 6);
+                const path = rev.path;
+                const uniqueChunkCount = rev.uniqueChunks.size;
+                const sharedChunkCount = rev.sharedChunks.size;
+                const uniqueChunkSize = getTotalSize(rev.uniqueChunks);
+                const sharedChunkSize = getTotalSize(rev.sharedChunks);
+                result.push({
+                    title,
+                    path,
+                    rev: revStr,
+                    revHash,
+                    id,
+                    uniqueChunkCount: uniqueChunkCount,
+                    sharedChunkCount,
+                    uniqueChunkSize: uniqueChunkSize,
+                    sharedChunkSize: sharedChunkSize,
+                });
+            }
+        }
+
+        const titleMap = {
+            title: "Title",
+            id: "Document ID",
+            path: "Path",
+            rev: "Revision No",
+            revHash: "Revision Hash",
+            uniqueChunkCount: "Unique Chunk Count",
+            sharedChunkCount: "Shared Chunk Count",
+            uniqueChunkSize: "Unique Chunk Size",
+            sharedChunkSize: "Shared Chunk Size",
+        } as const;
+        // Enumerate orphan chunks (not referenced by any document)
+        const orphanChunks = [...chunkMap.entries()].filter(([chunkId, infos]) => {
+            const totalRefCount = [...infos].reduce((acc, info) => acc + info.refCount, 0);
+            return totalRefCount === 0;
+        });
+        const orphanChunkSize = orphanChunks.reduce((acc, [chunkId, infos]) => {
+            for (const info of infos) {
+                acc += info.length;
+            }
+            return acc;
+        }, 0);
+        result.push({
+            title: "__orphan",
+            id: "__orphan",
+            path: "__orphan",
+            rev: "1",
+            revHash: "xxxxx",
+            uniqueChunkCount: orphanChunks.length,
+            sharedChunkCount: 0,
+            uniqueChunkSize: orphanChunkSize,
+            sharedChunkSize: 0,
+        } as any);
+
+        const csvSrc = result.map((e) => {
+            return [
+                `${e.title.replace(/"/g, '""')}"`,
+                `${e.id}`,
+                `${e.path}`,
+                `${e.rev}`,
+                `${e.revHash}`,
+                `${e.uniqueChunkCount}`,
+                `${e.sharedChunkCount}`,
+                `${e.uniqueChunkSize}`,
+                `${e.sharedChunkSize}`,
+            ].join("\t");
+        });
+        // Add title row
+        csvSrc.unshift(Object.values(titleMap).join("\t"));
+        const csv = csvSrc.join("\n");
+
+        // Prompt to copy to clipboard
+        await this.services.UI.promptCopyToClipboard("Database Analysis data (TSV):", csv);
+    }
 }

src/main.ts

@@ -26,7 +26,7 @@ import { ModuleFileAccessObsidian } from "./modules/coreObsidian/ModuleFileAcces
 import { ModuleInputUIObsidian } from "./modules/coreObsidian/ModuleInputUIObsidian.ts";
 import { ModuleMigration } from "./modules/essential/ModuleMigration.ts";
 
-import { ModuleCheckRemoteSize } from "./modules/coreFeatures/ModuleCheckRemoteSize.ts";
+import { ModuleCheckRemoteSize } from "./modules/essentialObsidian/ModuleCheckRemoteSize.ts";
 import { ModuleConflictResolver } from "./modules/coreFeatures/ModuleConflictResolver.ts";
 import { ModuleInteractiveConflictResolver } from "./modules/features/ModuleInteractiveConflictResolver.ts";
 import { ModuleLog } from "./modules/features/ModuleLog.ts";
@@ -170,9 +170,7 @@ export default class ObsidianLiveSyncPlugin
         new ModuleRedFlag(this),
         new ModuleInteractiveConflictResolver(this, this),
         new ModuleObsidianGlobalHistory(this, this),
-        // Common modules
-        // Note: Platform-dependent functions are not entirely dependent on the core only, as they are from platform-dependent modules. Stubbing is sometimes required.
-        new ModuleCheckRemoteSize(this),
+        new ModuleCheckRemoteSize(this, this),
         // Test and Dev Modules
         new ModuleDev(this, this),
         new ModuleReplicateTest(this, this),

src/modules/core/ModuleRebuilder.ts

@@ -1,5 +1,6 @@
 import { delay } from "octagonal-wheels/promises";
 import {
+    DEFAULT_SETTINGS,
     FLAGMD_REDFLAG2_HR,
     FLAGMD_REDFLAG3_HR,
     LOG_LEVEL_NOTICE,
@@ -58,7 +59,7 @@ Please enable them from the settings screen after setup is complete.`,
     async rebuildRemote() {
         await this.services.setting.suspendExtraSync();
         this.core.settings.isConfigured = true;
-
+        this.core.settings.notifyThresholdOfRemoteStorageSize = DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize;
         await this.services.setting.realiseSetting();
         await this.services.remote.markLocked();
         await this.services.remote.tryResetDatabase();
@@ -79,6 +80,7 @@ Please enable them from the settings screen after setup is complete.`,
         await this.services.setting.suspendExtraSync();
         // await this.askUseNewAdapter();
         this.core.settings.isConfigured = true;
+        this.core.settings.notifyThresholdOfRemoteStorageSize = DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize;
         await this.services.setting.realiseSetting();
         await this.resetLocalDatabase();
         await delay(1000);
@@ -191,6 +193,7 @@ Please enable them from the settings screen after setup is complete.`,
         await this.services.setting.suspendExtraSync();
         // await this.askUseNewAdapter();
         this.core.settings.isConfigured = true;
+        this.core.settings.notifyThresholdOfRemoteStorageSize = DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize;
         await this.suspendReflectingDatabase();
         await this.services.setting.realiseSetting();
         await this.resetLocalDatabase();

src/modules/essentialObsidian/ModuleCheckRemoteSize.ts

@@ -1,11 +1,17 @@
 import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
-import { AbstractModule } from "../AbstractModule.ts";
 import { sizeToHumanReadable } from "octagonal-wheels/number";
 import { $msg } from "src/lib/src/common/i18n.ts";
 import type { LiveSyncCore } from "../../main.ts";
+import { AbstractObsidianModule } from "../AbstractObsidianModule.ts";
+import { EVENT_REQUEST_CHECK_REMOTE_SIZE, eventHub } from "@/common/events.ts";
 
-export class ModuleCheckRemoteSize extends AbstractModule {
-    async _allScanStat(): Promise<boolean> {
+export class ModuleCheckRemoteSize extends AbstractObsidianModule {
+    checkRemoteSize(): Promise<boolean> {
+        this.settings.notifyThresholdOfRemoteStorageSize = 1;
+        return this._allScanStat();
+    }
+
+    private async _allScanStat(): Promise<boolean> {
         if (this.core.managers.networkManager.isOnline === false) {
             this._log("Network is offline, skipping remote size check.", LOG_LEVEL_INFO);
             return true;
@@ -109,7 +115,20 @@ export class ModuleCheckRemoteSize extends AbstractModule {
         }
         return true;
     }
+
+    private _everyOnloadStart(): Promise<boolean> {
+        this.addCommand({
+            id: "livesync-reset-remote-size-threshold-and-check",
+            name: "Reset notification threshold and check the remote database usage",
+            callback: async () => {
+                await this.checkRemoteSize();
+            },
+        });
+        eventHub.onEvent(EVENT_REQUEST_CHECK_REMOTE_SIZE, () => this.checkRemoteSize());
+        return Promise.resolve(true);
+    }
     onBindFunction(core: LiveSyncCore, services: typeof core.services): void {
         services.appLifecycle.handleOnScanningStartupIssues(this._allScanStat.bind(this));
+        services.appLifecycle.handleOnInitialise(this._everyOnloadStart.bind(this));
     }
 }

src/modules/features/SettingDialogue/PaneHatch.ts

@@ -26,7 +26,13 @@ import { addPrefix, shouldBeIgnored, stripAllPrefixes } from "../../../lib/src/s
 import { $msg } from "../../../lib/src/common/i18n.ts";
 import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
 import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
-import { EVENT_REQUEST_RUN_DOCTOR, EVENT_REQUEST_RUN_FIX_INCOMPLETE, eventHub } from "../../../common/events.ts";
+import {
+    EVENT_ANALYSE_DB_USAGE,
+    EVENT_REQUEST_CHECK_REMOTE_SIZE,
+    EVENT_REQUEST_RUN_DOCTOR,
+    EVENT_REQUEST_RUN_FIX_INCOMPLETE,
+    eventHub,
+} from "../../../common/events.ts";
 import { ICHeader, ICXHeader, PSCHeader } from "../../../common/types.ts";
 import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts";
 import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts";
@@ -182,6 +188,24 @@ ${stringifyYaml({
                     }
                 })
         );
+        new Setting(paneEl)
+            .setName("Analyse database usage")
+            .setDesc(
+                "Analyse database usage and generate a TSV report for diagnosis yourself. You can paste the generated report with any spreadsheet you like."
+            )
+            .addButton((button) =>
+                button.setButtonText("Analyse").onClick(() => {
+                    eventHub.emitEvent(EVENT_ANALYSE_DB_USAGE);
+                })
+            );
+        new Setting(paneEl)
+            .setName("Reset notification threshold and check the remote database usage")
+            .setDesc("Reset the remote storage size threshold and check the remote storage size again.")
+            .addButton((button) =>
+                button.setButtonText("Check").onClick(() => {
+                    eventHub.emitEvent(EVENT_REQUEST_CHECK_REMOTE_SIZE);
+                })
+            );
         new Setting(paneEl).autoWireToggle("writeLogToTheFile");
     });