## 0.24.8

### Fixed
-   Some parallel-processing tasks are now performed more safely.
-   Some error messages has been fixed.
### Improved
-   Synchronisation is now more efficient and faster.
-   Saving chunks is a bit more robust.
### New Feature
-   We can remove orphaned chunks again, now!

package-lock.json

@@ -18,7 +18,7 @@
                 "fflate": "^0.8.2",
                 "idb": "^8.0.0",
                 "minimatch": "^10.0.1",
-                "octagonal-wheels": "^0.1.21",
+                "octagonal-wheels": "^0.1.22",
                 "svelte-check": "^4.0.4",
                 "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
             },
@@ -5104,9 +5104,10 @@
             }
         },
         "node_modules/octagonal-wheels": {
-            "version": "0.1.21",
+            "version": "0.1.22",
-            "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.21.tgz",
+            "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.22.tgz",
-            "integrity": "sha512-yJYzli50IwXW1ARgVnHR8LpudhRay0pfkSYJyyxqDuk0WM8hbeWDWLtlB/77xga8WsqW1HnZZP/8LfyZYVg1ew==",
+            "integrity": "sha512-8wpQi1l1CYjEWtD8uMKjvaYR98r6XBQnpczDW/PdyB9E20HPFQYUyONfh5zd6g5/bfjtFchvd3KRE5TVauoucA==",
+            "license": "MIT",
             "dependencies": {
                 "idb": "^8.0.0"
             }
@@ -10308,9 +10309,9 @@
             }
         },
         "octagonal-wheels": {
-            "version": "0.1.21",
+            "version": "0.1.22",
-            "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.21.tgz",
+            "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.22.tgz",
-            "integrity": "sha512-yJYzli50IwXW1ARgVnHR8LpudhRay0pfkSYJyyxqDuk0WM8hbeWDWLtlB/77xga8WsqW1HnZZP/8LfyZYVg1ew==",
+            "integrity": "sha512-8wpQi1l1CYjEWtD8uMKjvaYR98r6XBQnpczDW/PdyB9E20HPFQYUyONfh5zd6g5/bfjtFchvd3KRE5TVauoucA==",
             "requires": {
                 "idb": "^8.0.0"
             }

package.json

@@ -72,7 +72,7 @@
         "fflate": "^0.8.2",
         "idb": "^8.0.0",
         "minimatch": "^10.0.1",
-        "octagonal-wheels": "^0.1.21",
+        "octagonal-wheels": "^0.1.22",
         "svelte-check": "^4.0.4",
         "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
     }

src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts

@@ -0,0 +1,265 @@
+import { sizeToHumanReadable } from "octagonal-wheels/number";
+import { LOG_LEVEL_NOTICE, type MetaEntry } from "../../lib/src/common/types";
+import { getNoFromRev } from "../../lib/src/pouchdb/LiveSyncLocalDB";
+import type { IObsidianModule } from "../../modules/AbstractObsidianModule";
+import { LiveSyncCommands } from "../LiveSyncCommands";
+
+export class LocalDatabaseMaintenance extends LiveSyncCommands implements IObsidianModule {
+    $everyOnload(): Promise<boolean> {
+        return Promise.resolve(true);
+    }
+    onunload(): void {
+        // NO OP.
+    }
+    onload(): void | Promise<void> {
+        // NO OP.
+    }
+    async allChunks(includeDeleted: boolean = false) {
+        const p = this._progress("", LOG_LEVEL_NOTICE);
+        p.log("Retrieving chunks informations..");
+        try {
+            const ret = await this.localDatabase.allChunks(includeDeleted);
+            return ret;
+        } finally {
+            p.done();
+        }
+    }
+    get database() {
+        return this.localDatabase.localDatabase;
+    }
+    clearHash() {
+        this.localDatabase.hashCaches.clear();
+    }
+
+    async confirm(title: string, message: string, affirmative = "Yes", negative = "No") {
+        return (
+            (await this.plugin.confirm.askSelectStringDialogue(message, [affirmative, negative], {
+                title,
+                defaultAction: affirmative,
+            })) === affirmative
+        );
+    }
+    isAvailable() {
+        if (!this.settings.doNotUseFixedRevisionForChunks) {
+            this._notice("Please enable 'Compute revisions for chunks' in settings to use Garbage Collection.");
+            return false;
+        }
+        if (this.settings.readChunksOnline) {
+            this._notice("Please disable 'Read chunks online' in settings to use Garbage Collection.");
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Resurrect deleted chunks that are still used in the database.
+     */
+    async resurrectChunks() {
+        if (!this.isAvailable()) return;
+        const { used, existing } = await this.allChunks(true);
+        const excessiveDeletions = [...existing]
+            .filter(([key, e]) => e._deleted)
+            .filter(([key, e]) => used.has(e._id))
+            .map(([key, e]) => e);
+        const completelyLostChunks = [] as string[];
+        // Data lost chunks : chunks that are deleted and data is purged.
+        const dataLostChunks = [...existing]
+            .filter(([key, e]) => e._deleted && e.data === "")
+            .map(([key, e]) => e)
+            .filter((e) => used.has(e._id));
+        for (const e of dataLostChunks) {
+            // Retrieve the data from the previous revision.
+            const doc = await this.database.get(e._id, { rev: e._rev, revs: true, revs_info: true, conflicts: true });
+            const history = doc._revs_info || [];
+            // Chunks are immutable. So, we can resurrect the chunk by copying the data from any of previous revisions.
+            let resurrected = null as null | string;
+            const availableRevs = history
+                .filter((e) => e.status == "available")
+                .map((e) => e.rev)
+                .sort((a, b) => getNoFromRev(a) - getNoFromRev(b));
+            for (const rev of availableRevs) {
+                const revDoc = await this.database.get(e._id, { rev: rev });
+                if (revDoc.type == "leaf" && revDoc.data !== "") {
+                    // Found the data.
+                    resurrected = revDoc.data;
+                    break;
+                }
+            }
+            // If the data is not found, we cannot resurrect the chunk, add it to the excessiveDeletions.
+            if (resurrected !== null) {
+                excessiveDeletions.push({ ...e, data: resurrected, _deleted: false });
+            } else {
+                completelyLostChunks.push(e._id);
+            }
+        }
+        // Chunks to be resurrected.
+        const resurrectChunks = excessiveDeletions.filter((e) => e.data !== "").map((e) => ({ ...e, _deleted: false }));
+
+        if (resurrectChunks.length == 0) {
+            this._notice("No chunks are found to be resurrected.");
+            return;
+        }
+        const message = `We have following chunks that are deleted but still used in the database.
+
+- Completely lost chunks: ${completelyLostChunks.length}
+- Resurrectable chunks: ${resurrectChunks.length}
+
+Do you want to resurrect these chunks?`;
+        if (await this.confirm("Resurrect Chunks", message, "Resurrect", "Cancel")) {
+            const result = await this.database.bulkDocs(resurrectChunks);
+            this.clearHash();
+            const resurrectedChunks = result.filter((e) => "ok" in e).map((e) => e.id);
+            this._notice(`Resurrected chunks: ${resurrectedChunks.length} / ${resurrectChunks.length}`);
+        } else {
+            this._notice("Resurrect operation is cancelled.");
+        }
+    }
+    /**
+     * Commit deletion of files that are marked as deleted.
+     * This method makes the deletion permanent, and the files will not be recovered.
+     * After this, chunks that are used in the deleted files become ready for compaction.
+     */
+    async commitFileDeletion() {
+        if (!this.isAvailable()) return;
+        const p = this._progress("", LOG_LEVEL_NOTICE);
+        p.log("Searching for deleted files..");
+        const docs = await this.database.allDocs<MetaEntry>({ include_docs: true });
+        const deletedDocs = docs.rows.filter(
+            (e) => (e.doc?.type == "newnote" || e.doc?.type == "plain") && e.doc?.deleted
+        );
+        if (deletedDocs.length == 0) {
+            p.done("No deleted files found.");
+            return;
+        }
+        p.log(`Found ${deletedDocs.length} deleted files.`);
+
+        const message = `We have following files that are marked as deleted.
+
+- Deleted files: ${deletedDocs.length}
+
+Are you sure to delete these files permanently?
+
+Note: **Make sure to synchronise all devices before deletion.**
+
+> [!Note]
+> This operation affects the database permanently. Deleted files will not be recovered after this operation.
+> And, the chunks that are used in the deleted files will be ready for compaction.`;
+
+        const deletingDocs = deletedDocs.map((e) => ({ ...e.doc, _deleted: true }) as MetaEntry);
+
+        if (await this.confirm("Delete Files", message, "Delete", "Cancel")) {
+            const result = await this.database.bulkDocs(deletingDocs);
+            this.clearHash();
+            p.done(`Deleted ${result.filter((e) => "ok" in e).length} / ${deletedDocs.length} files.`);
+        } else {
+            p.done("Deletion operation is cancelled.");
+        }
+    }
+    /**
+     * Commit deletion of chunks that are not used in the database.
+     * This method makes the deletion permanent, and the chunks will not be recovered if the database run compaction.
+     * After this, the database can shrink the database size by compaction.
+     * It is recommended to compact the database after this operation (History should be kept once before compaction).
+     */
+    async commitChunkDeletion() {
+        if (!this.isAvailable()) return;
+        const { existing } = await this.allChunks(true);
+        const deletedChunks = [...existing].filter(([key, e]) => e._deleted && e.data !== "").map(([key, e]) => e);
+        const deletedNotVacantChunks = deletedChunks.map((e) => ({ ...e, data: "", _deleted: true }));
+        const size = deletedChunks.reduce((acc, e) => acc + e.data.length, 0);
+        const humanSize = sizeToHumanReadable(size);
+        const message = `We have following chunks that are marked as deleted.
+
+- Deleted chunks: ${deletedNotVacantChunks.length} (${humanSize})
+
+Are you sure to delete these chunks permanently?
+
+Note: **Make sure to synchronise all devices before deletion.**
+
+> [!Note]
+> This operation finally reduces the capacity of the remote.`;
+
+        if (deletedNotVacantChunks.length == 0) {
+            this._notice("No deleted chunks found.");
+            return;
+        }
+        if (await this.confirm("Delete Chunks", message, "Delete", "Cancel")) {
+            const result = await this.database.bulkDocs(deletedNotVacantChunks);
+            this.clearHash();
+            this._notice(
+                `Deleted chunks: ${result.filter((e) => "ok" in e).length} / ${deletedNotVacantChunks.length}`
+            );
+        } else {
+            this._notice("Deletion operation is cancelled.");
+        }
+    }
+    /**
+     * Compact the database.
+     * This method removes all deleted chunks that are not used in the database.
+     * Make sure all devices are synchronized before running this method.
+     */
+    async markUnusedChunks() {
+        if (!this.isAvailable()) return;
+        const { used, existing } = await this.allChunks();
+        const existChunks = [...existing];
+        const unusedChunks = existChunks.filter(([key, e]) => !used.has(e._id)).map(([key, e]) => e);
+        const deleteChunks = unusedChunks.map((e) => ({
+            ...e,
+            _deleted: true,
+        }));
+        const size = deleteChunks.reduce((acc, e) => acc + e.data.length, 0);
+        const humanSize = sizeToHumanReadable(size);
+        if (deleteChunks.length == 0) {
+            this._notice("No unused chunks found.");
+            return;
+        }
+        const message = `We have following chunks that are not used from any files.
+
+- Chunks: ${deleteChunks.length} (${humanSize})
+
+Are you sure to mark these chunks to be deleted?
+
+Note: **Make sure to synchronise all devices before deletion.**
+
+> [!Note]
+> This operation will not reduces the capacity of the remote until permanent deletion.`;
+
+        if (await this.confirm("Mark unused chunks", message, "Mark", "Cancel")) {
+            const result = await this.database.bulkDocs(deleteChunks);
+            this.clearHash();
+            this._notice(`Marked chunks: ${result.filter((e) => "ok" in e).length} / ${deleteChunks.length}`);
+        }
+    }
+
+    async removeUnusedChunks() {
+        const { used, existing } = await this.allChunks();
+        const existChunks = [...existing];
+        const unusedChunks = existChunks.filter(([key, e]) => !used.has(e._id)).map(([key, e]) => e);
+        const deleteChunks = unusedChunks.map((e) => ({
+            ...e,
+            data: "",
+            _deleted: true,
+        }));
+        const size = unusedChunks.reduce((acc, e) => acc + e.data.length, 0);
+        const humanSize = sizeToHumanReadable(size);
+        if (deleteChunks.length == 0) {
+            this._notice("No unused chunks found.");
+            return;
+        }
+        const message = `We have following chunks that are not used from any files.
+
+- Chunks: ${deleteChunks.length} (${humanSize})
+
+Are you sure to delete these chunks?
+
+Note: **Make sure to synchronise all devices before deletion.**
+
+> [!Note]
+> Chunks referenced from deleted files are not deleted. Please run "Commit File Deletion" before this operation.`;
+
+        if (await this.confirm("Mark unused chunks", message, "Mark", "Cancel")) {
+            const result = await this.database.bulkDocs(deleteChunks);
+            this._notice(`Deleted chunks: ${result.filter((e) => "ok" in e).length} / ${deleteChunks.length}`);
+            this.clearHash();
+        }
+    }
+}

src/main.ts

@@ -81,6 +81,7 @@ import { ModuleRebuilder } from "./modules/core/ModuleRebuilder.ts";
 import { ModuleReplicateTest } from "./modules/extras/ModuleReplicateTest.ts";
 import { ModuleLiveSyncMain } from "./modules/main/ModuleLiveSyncMain.ts";
 import { ModuleExtraSyncObsidian } from "./modules/extraFeaturesObsidian/ModuleExtraSyncObsidian.ts";
+import { LocalDatabaseMaintenance } from "./features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts";
 
 function throwShouldBeOverridden(): never {
     throw new Error("This function should be overridden by the module.");
@@ -117,7 +118,7 @@ export default class ObsidianLiveSyncPlugin
     }
 
     // Keep order to display the dialogue in order.
-    addOns = [new ConfigSync(this), new HiddenFileSync(this)] as LiveSyncCommands[];
+    addOns = [new ConfigSync(this), new HiddenFileSync(this), new LocalDatabaseMaintenance(this)] as LiveSyncCommands[];
 
     modules = [
         new ModuleLiveSyncMain(this),

src/modules/core/ModuleFileHandler.ts

@@ -211,7 +211,7 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
                 ? await this.db.fetchEntryMeta(entryInfo, undefined, true)
                 : await this.db.fetchEntryMeta(entryInfo.path, undefined, true);
         if (!docEntry) {
-            this._log(`File ${entryInfo} is not exist on the database`, LOG_LEVEL_VERBOSE);
+            this._log(`File ${file?.path} is not exist on the database`, LOG_LEVEL_VERBOSE);
             return false;
         }
         const path = getPath(docEntry);

src/modules/essential/ModuleInitializerFile.ts

@@ -231,11 +231,6 @@ export class ModuleInitializerFile extends AbstractModule implements ICoreModule
                 const { file, doc } = e;
                 if (!this.core.$$isFileSizeExceeded(file.stat.size) && !this.core.$$isFileSizeExceeded(doc.size)) {
                     await this.syncFileBetweenDBandStorage(file, doc);
-                    // fireAndForget(() => this.checkAndApplySettingFromMarkdown(getPath(doc), true));
-                    // eventHub.emitEvent("event-file-changed", {
-                    //     file: getPath(doc),
-                    //     automated: true,
-                    // });
                 } else {
                     this._log(
                         `SYNC DATABASE AND STORAGE: ${getPath(doc)} has been skipped due to file size exceeding the limit`,

src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts

@@ -77,6 +77,7 @@ import { JournalSyncMinio } from "../../../lib/src/replication/journal/objectsto
 import { ICHeader, ICXHeader, PSCHeader } from "../../../common/types.ts";
 import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts";
 import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts";
+import { LocalDatabaseMaintenance } from "../../../features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts";
 
 export type OnUpdateResult = {
     visibility?: boolean;
@@ -2880,6 +2881,7 @@ ${stringifyYaml(pluginConfig)}`;
                             })
                     )
                     .addOnUpdate(onlyOnCouchDB);
+
                 new Setting(paneEl)
                     .setName("Reset journal received history")
                     .setDesc(
@@ -2923,7 +2925,53 @@ ${stringifyYaml(pluginConfig)}`;
                     )
                     .addOnUpdate(onlyOnMinIO);
             });
+            void addPanel(paneEl, "Garbage Collection (Beta)", (e) => e, onlyOnCouchDB).then((paneEl) => {
+                new Setting(paneEl)
+                    .setName("Remove all orphaned chunks")
+                    .setDesc("Remove all orphaned chunks from the local database.")
+                    .addButton((button) =>
+                        button
+                            .setButtonText("Remove")
+                            .setWarning()
+                            .setDisabled(false)
+                            .onClick(async () => {
+                                await this.plugin
+                                    .getAddOn<LocalDatabaseMaintenance>(LocalDatabaseMaintenance.name)
+                                    ?.removeUnusedChunks();
+                            })
+                    );
 
+                new Setting(paneEl)
+                    .setName("Resurrect deleted chunks")
+                    .setDesc(
+                        "If you have deleted chunks before fully synchronised and missed some chunks, you possibly can resurrect them."
+                    )
+                    .addButton((button) =>
+                        button
+                            .setButtonText("Try resurrect")
+                            .setWarning()
+                            .setDisabled(false)
+                            .onClick(async () => {
+                                await this.plugin
+                                    .getAddOn<LocalDatabaseMaintenance>(LocalDatabaseMaintenance.name)
+                                    ?.resurrectChunks();
+                            })
+                    );
+                new Setting(paneEl)
+                    .setName("Commit File Deletion")
+                    .setDesc("Completely delete all deleted documents from the local database.")
+                    .addButton((button) =>
+                        button
+                            .setButtonText("Delete")
+                            .setWarning()
+                            .setDisabled(false)
+                            .onClick(async () => {
+                                await this.plugin
+                                    .getAddOn<LocalDatabaseMaintenance>(LocalDatabaseMaintenance.name)
+                                    ?.commitFileDeletion();
+                            })
+                    );
+            });
             void addPanel(paneEl, "Rebuilding Operations (Local)").then((paneEl) => {
                 new Setting(paneEl)
                     .setName("Fetch from remote")
@@ -3076,33 +3124,6 @@ ${stringifyYaml(pluginConfig)}`;
                     .addOnUpdate(onlyOnMinIO);
             });
 
-            void addPanel(paneEl, "Deprecated").then((paneEl) => {
-                new Setting(paneEl)
-                    .setClass("sls-setting-obsolete")
-                    .setName("Run database cleanup")
-                    .setDesc(
-                        "Attempt to shrink the database by deleting unused chunks. This may not work consistently. Use the 'Rebuild everything' under Total Overhaul."
-                    )
-                    .addButton((button) =>
-                        button
-                            .setButtonText("DryRun")
-                            .setDisabled(false)
-                            .onClick(async () => {
-                                await this.dryRunGC();
-                            })
-                    )
-                    .addButton((button) =>
-                        button
-                            .setButtonText("Perform cleaning")
-                            .setDisabled(false)
-                            .setWarning()
-                            .onClick(async () => {
-                                this.closeSetting();
-                                await this.dbGC();
-                            })
-                    )
-                    .addOnUpdate(onlyOnCouchDB);
-            });
             void addPanel(paneEl, "Reset").then((paneEl) => {
                 new Setting(paneEl)
                     .setName("Delete local database to reset or uninstall Self-hosted LiveSync")