From 77074cb92f41fbc9f1f2005168ebdcffdc3f5672 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 5 Dec 2025 11:07:59 +0000 Subject: [PATCH] ### New feature - We can analyse the local database with the `Analyse database usage` command. - We can reset the notification threshold and check the remote usage at once with the `Reset notification threshold and check the remote database usage` command. ### Fixed - Now the plug-in resets the remote size notification threshold after rebuild. --- src/common/events.ts | 4 + .../CmdLocalDatabaseMainte.ts | 223 ++++++++++++++++++ src/main.ts | 6 +- src/modules/core/ModuleRebuilder.ts | 5 +- .../ModuleCheckRemoteSize.ts | 25 +- .../features/SettingDialogue/PaneHatch.ts | 26 +- 6 files changed, 280 insertions(+), 9 deletions(-) rename src/modules/{coreFeatures => essentialObsidian}/ModuleCheckRemoteSize.ts (85%) diff --git a/src/common/events.ts b/src/common/events.ts index 6bd1172..86ff8c4 100644 --- a/src/common/events.ts +++ b/src/common/events.ts @@ -23,6 +23,8 @@ export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor"; export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete"; export const EVENT_ON_UNRESOLVED_ERROR = "on-unresolved-error"; +export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage"; +export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size"; // export const EVENT_FILE_CHANGED = "file-changed"; declare global { @@ -42,6 +44,8 @@ declare global { [EVENT_REQUEST_RUN_DOCTOR]: string; [EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined; [EVENT_ON_UNRESOLVED_ERROR]: undefined; + [EVENT_ANALYSE_DB_USAGE]: undefined; + [EVENT_REQUEST_CHECK_REMOTE_SIZE]: undefined; } } diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts index dddadaa..0ea34e0 100644 --- a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts @@ -7,12 +7,14 @@ import { type DocumentID, type EntryDoc, type EntryLeaf, + type FilePathWithPrefix, type MetaEntry, } from "../../lib/src/common/types"; import { getNoFromRev } from "../../lib/src/pouchdb/LiveSyncLocalDB"; import { LiveSyncCommands } from "../LiveSyncCommands"; import { serialized } from "octagonal-wheels/concurrency/lock_v2"; import { arrayToChunkedArray } from "octagonal-wheels/collection"; +import { EVENT_ANALYSE_DB_USAGE, eventHub } from "@/common/events"; const DB_KEY_SEQ = "gc-seq"; const DB_KEY_CHUNK_SET = "chunk-set"; const DB_KEY_DOC_USAGE_MAP = "doc-usage-map"; @@ -27,6 +29,15 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands { } onload(): void | Promise { // NO OP. + this.plugin.addCommand({ + id: "analyse-database", + name: "Analyse Database Usage (advanced)", + icon: "database-search", + callback: async () => { + await this.analyseDatabase(); + }, + }); + eventHub.onEvent(EVENT_ANALYSE_DB_USAGE, () => this.analyseDatabase()); } async allChunks(includeDeleted: boolean = false) { const p = this._progress("", LOG_LEVEL_NOTICE); @@ -485,4 +496,216 @@ Success: ${successCount}, Errored: ${errored}`; const kvDB = this.plugin.kvDB; await kvDB.set(DB_KEY_CHUNK_SET, chunkSet); } + + // Analyse the database and report chunk usage. + async analyseDatabase() { + if (!this.isAvailable()) return; + const db = this.localDatabase.localDatabase; + // Map of chunk ID to its info + type ChunkInfo = { + id: DocumentID; + refCount: number; + length: number; + }; + const chunkMap = new Map>(); + // Map of document ID to its info + type DocumentInfo = { + id: DocumentID; + rev: Rev; + chunks: Set; + uniqueChunks: Set; + sharedChunks: Set; + path: FilePathWithPrefix; + }; + const docMap = new Map>(); + const info = await db.info(); + // Total number of revisions to process (approximate) + const maxSeq = new Number(info.update_seq); + let processed = 0; + let read = 0; + let errored = 0; + // Fetch Tasks + const ft = [] as ReturnType[]; + // Fetch a specific revision of a document and make note of its chunks, or add chunk info. + const fetchRevision = async (id: DocumentID, rev: Rev, seq: string | number) => { + try { + processed++; + const doc = await db.get(id, { rev: rev }); + if (doc) { + if ("children" in doc) { + const id = doc._id; + const rev = doc._rev; + const children = (doc.children || []) as DocumentID[]; + const set = docMap.get(id) || new Set(); + set.add({ + id, + rev, + chunks: new Set(children), + uniqueChunks: new Set(), + sharedChunks: new Set(), + path: doc.path, + }); + docMap.set(id, set); + } else if (doc.type === EntryTypes.CHUNK) { + const id = doc._id as DocumentID; + if (chunkMap.has(id)) { + return; + } + if (doc._deleted) { + // Deleted chunk, skip (possibly resurrected later) + return; + } + const length = doc.data.length; + const set = chunkMap.get(id) || new Set(); + set.add({ id, length, refCount: 0 }); + chunkMap.set(id, set); + } + read++; + } else { + this._log(`Analysing Database: not found: ${id} / ${rev}`); + errored++; + } + } catch (error) { + this._log(`Error fetching document ${id} / ${rev}: $`, LOG_LEVEL_NOTICE); + this._log(error, LOG_LEVEL_VERBOSE); + errored++; + } + if (processed % 100 == 0) { + this._log(`Analysing database: ${read} (${errored}) / ${maxSeq} `, LOG_LEVEL_NOTICE, "db-analyse"); + } + }; + + // Enumerate all documents and their revisions. + const IDs = this.localDatabase.findEntryNames("", "", {}); + for await (const id of IDs) { + const revList = await this.localDatabase.getRaw(id as DocumentID, { + revs: true, + revs_info: true, + conflicts: true, + }); + const revInfos = revList._revs_info || []; + for (const revInfo of revInfos) { + // All available revisions should be processed. + // If the revision is not available, it means the revision is already tombstoned. + if (revInfo.status == "available") { + // Schedule fetch task + ft.push(fetchRevision(id as DocumentID, revInfo.rev, 0)); + } + } + } + // Wait for all fetch tasks to complete. + await Promise.all(ft); + // Reference count marking and unique/shared chunk classification. + for (const [, docRevs] of docMap) { + for (const docRev of docRevs) { + for (const chunkId of docRev.chunks) { + const chunkInfos = chunkMap.get(chunkId); + if (chunkInfos) { + for (const chunkInfo of chunkInfos) { + if (chunkInfo.refCount === 0) { + docRev.uniqueChunks.add(chunkId); + } else { + docRev.sharedChunks.add(chunkId); + } + chunkInfo.refCount++; + } + } + } + } + } + // Prepare results + const result = []; + // Calculate total size of chunks in the given set. + const getTotalSize = (ids: Set) => { + return [...ids].reduce((acc, chunkId) => { + const chunkInfos = chunkMap.get(chunkId); + if (chunkInfos) { + for (const chunkInfo of chunkInfos) { + acc += chunkInfo.length; + } + } + return acc; + }, 0); + }; + + // Compile results for each document revision + for (const doc of docMap.values()) { + for (const rev of doc) { + const title = `${rev.path} (${rev.rev})`; + const id = rev.id; + const revStr = `${getNoFromRev(rev.rev)}`; + const revHash = rev.rev.split("-")[1].substring(0, 6); + const path = rev.path; + const uniqueChunkCount = rev.uniqueChunks.size; + const sharedChunkCount = rev.sharedChunks.size; + const uniqueChunkSize = getTotalSize(rev.uniqueChunks); + const sharedChunkSize = getTotalSize(rev.sharedChunks); + result.push({ + title, + path, + rev: revStr, + revHash, + id, + uniqueChunkCount: uniqueChunkCount, + sharedChunkCount, + uniqueChunkSize: uniqueChunkSize, + sharedChunkSize: sharedChunkSize, + }); + } + } + + const titleMap = { + title: "Title", + id: "Document ID", + path: "Path", + rev: "Revision No", + revHash: "Revision Hash", + uniqueChunkCount: "Unique Chunk Count", + sharedChunkCount: "Shared Chunk Count", + uniqueChunkSize: "Unique Chunk Size", + sharedChunkSize: "Shared Chunk Size", + } as const; + // Enumerate orphan chunks (not referenced by any document) + const orphanChunks = [...chunkMap.entries()].filter(([chunkId, infos]) => { + const totalRefCount = [...infos].reduce((acc, info) => acc + info.refCount, 0); + return totalRefCount === 0; + }); + const orphanChunkSize = orphanChunks.reduce((acc, [chunkId, infos]) => { + for (const info of infos) { + acc += info.length; + } + return acc; + }, 0); + result.push({ + title: "__orphan", + id: "__orphan", + path: "__orphan", + rev: "1", + revHash: "xxxxx", + uniqueChunkCount: orphanChunks.length, + sharedChunkCount: 0, + uniqueChunkSize: orphanChunkSize, + sharedChunkSize: 0, + } as any); + + const csvSrc = result.map((e) => { + return [ + `${e.title.replace(/"/g, '""')}"`, + `${e.id}`, + `${e.path}`, + `${e.rev}`, + `${e.revHash}`, + `${e.uniqueChunkCount}`, + `${e.sharedChunkCount}`, + `${e.uniqueChunkSize}`, + `${e.sharedChunkSize}`, + ].join("\t"); + }); + // Add title row + csvSrc.unshift(Object.values(titleMap).join("\t")); + const csv = csvSrc.join("\n"); + + // Prompt to copy to clipboard + await this.services.UI.promptCopyToClipboard("Database Analysis data (TSV):", csv); + } } diff --git a/src/main.ts b/src/main.ts index bed288c..9159a79 100644 --- a/src/main.ts +++ b/src/main.ts @@ -26,7 +26,7 @@ import { ModuleFileAccessObsidian } from "./modules/coreObsidian/ModuleFileAcces import { ModuleInputUIObsidian } from "./modules/coreObsidian/ModuleInputUIObsidian.ts"; import { ModuleMigration } from "./modules/essential/ModuleMigration.ts"; -import { ModuleCheckRemoteSize } from "./modules/coreFeatures/ModuleCheckRemoteSize.ts"; +import { ModuleCheckRemoteSize } from "./modules/essentialObsidian/ModuleCheckRemoteSize.ts"; import { ModuleConflictResolver } from "./modules/coreFeatures/ModuleConflictResolver.ts"; import { ModuleInteractiveConflictResolver } from "./modules/features/ModuleInteractiveConflictResolver.ts"; import { ModuleLog } from "./modules/features/ModuleLog.ts"; @@ -170,9 +170,7 @@ export default class ObsidianLiveSyncPlugin new ModuleRedFlag(this), new ModuleInteractiveConflictResolver(this, this), new ModuleObsidianGlobalHistory(this, this), - // Common modules - // Note: Platform-dependent functions are not entirely dependent on the core only, as they are from platform-dependent modules. Stubbing is sometimes required. - new ModuleCheckRemoteSize(this), + new ModuleCheckRemoteSize(this, this), // Test and Dev Modules new ModuleDev(this, this), new ModuleReplicateTest(this, this), diff --git a/src/modules/core/ModuleRebuilder.ts b/src/modules/core/ModuleRebuilder.ts index ffc998c..7a4ea99 100644 --- a/src/modules/core/ModuleRebuilder.ts +++ b/src/modules/core/ModuleRebuilder.ts @@ -1,5 +1,6 @@ import { delay } from "octagonal-wheels/promises"; import { + DEFAULT_SETTINGS, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, LOG_LEVEL_NOTICE, @@ -58,7 +59,7 @@ Please enable them from the settings screen after setup is complete.`, async rebuildRemote() { await this.services.setting.suspendExtraSync(); this.core.settings.isConfigured = true; - + this.core.settings.notifyThresholdOfRemoteStorageSize = DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize; await this.services.setting.realiseSetting(); await this.services.remote.markLocked(); await this.services.remote.tryResetDatabase(); @@ -79,6 +80,7 @@ Please enable them from the settings screen after setup is complete.`, await this.services.setting.suspendExtraSync(); // await this.askUseNewAdapter(); this.core.settings.isConfigured = true; + this.core.settings.notifyThresholdOfRemoteStorageSize = DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize; await this.services.setting.realiseSetting(); await this.resetLocalDatabase(); await delay(1000); @@ -191,6 +193,7 @@ Please enable them from the settings screen after setup is complete.`, await this.services.setting.suspendExtraSync(); // await this.askUseNewAdapter(); this.core.settings.isConfigured = true; + this.core.settings.notifyThresholdOfRemoteStorageSize = DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize; await this.suspendReflectingDatabase(); await this.services.setting.realiseSetting(); await this.resetLocalDatabase(); diff --git a/src/modules/coreFeatures/ModuleCheckRemoteSize.ts b/src/modules/essentialObsidian/ModuleCheckRemoteSize.ts similarity index 85% rename from src/modules/coreFeatures/ModuleCheckRemoteSize.ts rename to src/modules/essentialObsidian/ModuleCheckRemoteSize.ts index 55363ed..bd9c8cf 100644 --- a/src/modules/coreFeatures/ModuleCheckRemoteSize.ts +++ b/src/modules/essentialObsidian/ModuleCheckRemoteSize.ts @@ -1,11 +1,17 @@ import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger"; -import { AbstractModule } from "../AbstractModule.ts"; import { sizeToHumanReadable } from "octagonal-wheels/number"; import { $msg } from "src/lib/src/common/i18n.ts"; import type { LiveSyncCore } from "../../main.ts"; +import { AbstractObsidianModule } from "../AbstractObsidianModule.ts"; +import { EVENT_REQUEST_CHECK_REMOTE_SIZE, eventHub } from "@/common/events.ts"; -export class ModuleCheckRemoteSize extends AbstractModule { - async _allScanStat(): Promise { +export class ModuleCheckRemoteSize extends AbstractObsidianModule { + checkRemoteSize(): Promise { + this.settings.notifyThresholdOfRemoteStorageSize = 1; + return this._allScanStat(); + } + + private async _allScanStat(): Promise { if (this.core.managers.networkManager.isOnline === false) { this._log("Network is offline, skipping remote size check.", LOG_LEVEL_INFO); return true; @@ -109,7 +115,20 @@ export class ModuleCheckRemoteSize extends AbstractModule { } return true; } + + private _everyOnloadStart(): Promise { + this.addCommand({ + id: "livesync-reset-remote-size-threshold-and-check", + name: "Reset notification threshold and check the remote database usage", + callback: async () => { + await this.checkRemoteSize(); + }, + }); + eventHub.onEvent(EVENT_REQUEST_CHECK_REMOTE_SIZE, () => this.checkRemoteSize()); + return Promise.resolve(true); + } onBindFunction(core: LiveSyncCore, services: typeof core.services): void { services.appLifecycle.handleOnScanningStartupIssues(this._allScanStat.bind(this)); + services.appLifecycle.handleOnInitialise(this._everyOnloadStart.bind(this)); } } diff --git a/src/modules/features/SettingDialogue/PaneHatch.ts b/src/modules/features/SettingDialogue/PaneHatch.ts index fa08b35..70ef112 100644 --- a/src/modules/features/SettingDialogue/PaneHatch.ts +++ b/src/modules/features/SettingDialogue/PaneHatch.ts @@ -26,7 +26,13 @@ import { addPrefix, shouldBeIgnored, stripAllPrefixes } from "../../../lib/src/s import { $msg } from "../../../lib/src/common/i18n.ts"; import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; -import { EVENT_REQUEST_RUN_DOCTOR, EVENT_REQUEST_RUN_FIX_INCOMPLETE, eventHub } from "../../../common/events.ts"; +import { + EVENT_ANALYSE_DB_USAGE, + EVENT_REQUEST_CHECK_REMOTE_SIZE, + EVENT_REQUEST_RUN_DOCTOR, + EVENT_REQUEST_RUN_FIX_INCOMPLETE, + eventHub, +} from "../../../common/events.ts"; import { ICHeader, ICXHeader, PSCHeader } from "../../../common/types.ts"; import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts"; import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts"; @@ -182,6 +188,24 @@ ${stringifyYaml({ } }) ); + new Setting(paneEl) + .setName("Analyse database usage") + .setDesc( + "Analyse database usage and generate a TSV report for diagnosis yourself. You can paste the generated report with any spreadsheet you like." + ) + .addButton((button) => + button.setButtonText("Analyse").onClick(() => { + eventHub.emitEvent(EVENT_ANALYSE_DB_USAGE); + }) + ); + new Setting(paneEl) + .setName("Reset notification threshold and check the remote database usage") + .setDesc("Reset the remote storage size threshold and check the remote storage size again.") + .addButton((button) => + button.setButtonText("Check").onClick(() => { + eventHub.emitEvent(EVENT_REQUEST_CHECK_REMOTE_SIZE); + }) + ); new Setting(paneEl).autoWireToggle("writeLogToTheFile"); });