## 0.25.7

### Fixed

- Off-loaded chunking have been fixed to ensure proper functionality (#693).
- Chunk document ID assignment has been fixed.
- Replication prevention message during version up detection has been improved (#686).
- `Keep A` and `Keep B` on Conflict resolving dialogue has been renamed to `Use Base` and `Use Conflicted` (#691).

### Improved

- Metadata and content-size unmatched documents are now detected and reported, prevented to be applied to the storage.

### New Features

- `Scan for Broken files` has been implemented on `Hatch` -> `TroubleShooting`.

### Refactored

- Off-loaded processes have been refactored for the better maintainability.
- Removed unused code.

src/common/events.ts

@@ -20,6 +20,7 @@ export const EVENT_REQUEST_OPEN_P2P = "request-open-p2p";
 export const EVENT_REQUEST_CLOSE_P2P = "request-close-p2p";
 
 export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor";
+export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete";
 
 // export const EVENT_FILE_CHANGED = "file-changed";
 
@@ -38,6 +39,7 @@ declare global {
         [EVENT_REQUEST_COPY_SETUP_URI]: undefined;
         [EVENT_REQUEST_SHOW_SETUP_QR]: undefined;
         [EVENT_REQUEST_RUN_DOCTOR]: string;
+        [EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined;
     }
 }
 

src/modules/core/ModuleFileHandler.ts

@@ -18,7 +18,7 @@ import {
     getStoragePathFromUXFileInfo,
     markChangesAreSame,
 } from "../../common/utils";
-import { getDocDataAsArray, isDocContentSame, readContent } from "../../lib/src/common/utils";
+import { getDocDataAsArray, isDocContentSame, readAsBlob, readContent } from "../../lib/src/common/utils";
 import { shouldBeIgnored } from "../../lib/src/string_and_binary/path";
 import type { ICoreModule } from "../ModuleTypes";
 import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
@@ -259,6 +259,16 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule {
         const docData = readContent(docRead);
 
         if (existOnStorage && !force) {
+            // If we want to process size mismatched files -- in case of having files created by some integrations, enable the toggle.
+            if (!this.settings.processSizeMismatchedFiles) {
+                // Check the file is not corrupted
+                // (Zero is a special case, may be created by some APIs and it might be acceptable).
+                if (docRead.size != 0 && docRead.size !== readAsBlob(docRead).size) {
+                    this._log(`File ${path} seems to be corrupted! Writing prevented.`, LOG_LEVEL_NOTICE);
+                    return false;
+                }
+            }
+
             // The file is exist on the storage. Let's check the difference between the file and the entry.
             // But, if force is true, then it should be updated.
             // Ok, we have to compare.

src/modules/essential/ModuleMigration.ts

@@ -1,16 +1,20 @@
-import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
+import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "../../lib/src/common/logger.ts";
 import {
     EVENT_REQUEST_OPEN_P2P,
     EVENT_REQUEST_OPEN_SETTING_WIZARD,
     EVENT_REQUEST_OPEN_SETTINGS,
     EVENT_REQUEST_OPEN_SETUP_URI,
     EVENT_REQUEST_RUN_DOCTOR,
+    EVENT_REQUEST_RUN_FIX_INCOMPLETE,
     eventHub,
 } from "../../common/events.ts";
 import { AbstractModule } from "../AbstractModule.ts";
 import type { ICoreModule } from "../ModuleTypes.ts";
 import { $msg } from "src/lib/src/common/i18n.ts";
 import { performDoctorConsultation, RebuildOptions } from "../../lib/src/common/configForDoc.ts";
+import { getPath, isValidPath } from "../../common/utils.ts";
+import { isMetaEntry } from "../../lib/src/common/types.ts";
+import { isDeletedEntry, isDocContentSame, isLoadedEntry, readAsBlob } from "../../lib/src/common/utils.ts";
 
 export class ModuleMigration extends AbstractModule implements ICoreModule {
     async migrateUsingDoctor(skipRebuild: boolean = false, activateReason = "updated", forceRescan = false) {
@@ -96,12 +100,129 @@ export class ModuleMigration extends AbstractModule implements ICoreModule {
         return false;
     }
 
+    async checkIncompleteDocs(force: boolean = false): Promise<boolean> {
+        const incompleteDocsChecked = (await this.core.kvDB.get<boolean>("checkIncompleteDocs")) || false;
+        if (incompleteDocsChecked && !force) {
+            this._log("Incomplete docs check already done, skipping.", LOG_LEVEL_VERBOSE);
+            return Promise.resolve(true);
+        }
+
+        this._log("Checking for incomplete documents...", LOG_LEVEL_NOTICE, "check-incomplete");
+        const errorFiles = [];
+        for await (const metaDoc of this.localDatabase.findAllNormalDocs({ conflicts: true })) {
+            const path = getPath(metaDoc);
+
+            if (!isValidPath(path)) {
+                continue;
+            }
+            if (!(await this.core.$$isTargetFile(path, true))) {
+                continue;
+            }
+            if (!isMetaEntry(metaDoc)) {
+                continue;
+            }
+
+            const doc = await this.localDatabase.getDBEntryFromMeta(metaDoc);
+            if (!doc || !isLoadedEntry(doc)) {
+                continue;
+            }
+            if (isDeletedEntry(doc)) {
+                continue;
+            }
+            const storageFileContent = await this.core.storageAccess.readHiddenFileBinary(path);
+            // const storageFileBlob = createBlob(storageFileContent);
+            const sizeOnStorage = storageFileContent.byteLength;
+            const recordedSize = doc.size;
+            const docBlob = readAsBlob(doc);
+            const actualSize = docBlob.size;
+            if (recordedSize !== actualSize || sizeOnStorage !== actualSize || sizeOnStorage !== recordedSize) {
+                const contentMatched = await isDocContentSame(doc.data, storageFileContent);
+                errorFiles.push({ path, recordedSize, actualSize, storageSize: sizeOnStorage, contentMatched });
+                Logger(
+                    `Size mismatch for ${path}: ${recordedSize} (DB Recorded) , ${actualSize} (DB Stored) , ${sizeOnStorage} (Storage Stored), ${contentMatched ? "Content Matched" : "Content Mismatched"}`
+                );
+            }
+        }
+        if (errorFiles.length == 0) {
+            Logger("No size mismatches found", LOG_LEVEL_NOTICE);
+            await this.core.kvDB.set("checkIncompleteDocs", true);
+            return Promise.resolve(true);
+        }
+        Logger(`Found ${errorFiles.length} size mismatches`, LOG_LEVEL_NOTICE);
+        // We have to repair them following rules and situations:
+        // A. DB Recorded != DB Stored
+        //   A.1. DB Recorded == Storage Stored
+        //        Possibly recoverable from storage. Just overwrite the DB content with storage content.
+        //   A.2. Neither
+        //        Probably it cannot be resolved on this device. Even if the storage content is larger than DB Recorded, it possibly corrupted.
+        //        We do not fix it automatically. Leave it as is. Possibly other device can do this.
+        // B. DB Recorded == DB Stored ,  < Storage Stored
+        //   Very fragile, if DB Recorded size is less than Storage Stored size, we possibly repair the content (The issue was `unexpectedly shortened file`).
+        //   We do not fix it automatically, but it will be automatically overwritten in other process.
+        // C. DB Recorded == DB Stored ,  > Storage Stored
+        //   Probably restored by the user by resolving A or B on other device, We should overwrite the storage
+        //   Also do not fix it automatically. It should be overwritten by replication.
+        const recoverable = errorFiles.filter((e) => {
+            return e.recordedSize === e.storageSize;
+        });
+        const unrecoverable = errorFiles.filter((e) => {
+            return e.recordedSize !== e.storageSize;
+        });
+        const messageUnrecoverable =
+            unrecoverable.length > 0
+                ? $msg("moduleMigration.fix0256.messageUnrecoverable", {
+                      filesNotRecoverable: unrecoverable
+                          .map((e) => `- ${e.path} (M: ${e.recordedSize}, A: ${e.actualSize}, S: ${e.storageSize})`)
+                          .join("\n"),
+                  })
+                : "";
+
+        const message = $msg("moduleMigration.fix0256.message", {
+            files: recoverable
+                .map((e) => `- ${e.path} (M: ${e.recordedSize}, A: ${e.actualSize}, S: ${e.storageSize})`)
+                .join("\n"),
+            messageUnrecoverable,
+        });
+        const CHECK_IT_LATER = $msg("moduleMigration.fix0256.buttons.checkItLater");
+        const FIX = $msg("moduleMigration.fix0256.buttons.fix");
+        const DISMISS = $msg("moduleMigration.fix0256.buttons.DismissForever");
+        const ret = await this.core.confirm.askSelectStringDialogue(message, [CHECK_IT_LATER, FIX, DISMISS], {
+            title: $msg("moduleMigration.fix0256.title"),
+            defaultAction: CHECK_IT_LATER,
+        });
+        if (ret == FIX) {
+            for (const file of recoverable) {
+                // Overwrite the database with the files on the storage
+                const stubFile = this.core.storageAccess.getFileStub(file.path);
+                if (stubFile == null) {
+                    Logger(`Could not find stub file for ${file.path}`, LOG_LEVEL_NOTICE);
+                    continue;
+                }
+
+                stubFile.stat.mtime = Date.now();
+                const result = await this.core.fileHandler.storeFileToDB(stubFile, true, false);
+                if (result) {
+                    Logger(`Successfully restored ${file.path} from storage`);
+                } else {
+                    Logger(`Failed to restore ${file.path} from storage`, LOG_LEVEL_NOTICE);
+                }
+            }
+        } else if (ret === DISMISS) {
+            // User chose to dismiss the issue
+            await this.core.kvDB.set("checkIncompleteDocs", true);
+        }
+
+        return Promise.resolve(true);
+    }
+
     async $everyOnFirstInitialize(): Promise<boolean> {
         if (!this.localDatabase.isReady) {
             this._log($msg("moduleMigration.logLocalDatabaseNotReady"), LOG_LEVEL_NOTICE);
             return false;
         }
         if (this.settings.isConfigured) {
+            // TODO: Probably we have to check for insecure chunks
+            await this.checkIncompleteDocs();
             await this.migrateUsingDoctor(false);
             // await this.migrationCheck();
             await this.migrateDisableBulkSend();
@@ -120,6 +241,9 @@ export class ModuleMigration extends AbstractModule implements ICoreModule {
         eventHub.onEvent(EVENT_REQUEST_RUN_DOCTOR, async (reason) => {
             await this.migrateUsingDoctor(false, reason, true);
         });
+        eventHub.onEvent(EVENT_REQUEST_RUN_FIX_INCOMPLETE, async () => {
+            await this.checkIncompleteDocs(true);
+        });
         return Promise.resolve(true);
     }
 }

src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts

@@ -25,8 +25,8 @@ export class ConflictResolveModal extends Modal {
     title: string = "Conflicting changes";
 
     pluginPickMode: boolean = false;
-    localName: string = "Keep A";
-    remoteName: string = "Keep B";
+    localName: string = "Use Base";
+    remoteName: string = "Use Conflicted";
     offEvent?: ReturnType<typeof eventHub.onEvent>;
 
     constructor(app: App, filename: string, diff: diff_result, pluginPickMode?: boolean, remoteName?: string) {

src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts

@@ -17,8 +17,6 @@ import { delay, isObjectDifferent, sizeToHumanReadable } from "../../../lib/src/
 import { versionNumberString2Number } from "../../../lib/src/string_and_binary/convert.ts";
 import { Logger } from "../../../lib/src/common/logger.ts";
 import { checkSyncInfo } from "@/lib/src/pouchdb/negotiation.ts";
-import { balanceChunkPurgedDBs } from "@/lib/src/pouchdb/chunks.ts";
-import { purgeUnreferencedChunks } from "@/lib/src/pouchdb/chunks.ts";
 import { testCrypt } from "../../../lib/src/encryption/e2ee_v2.ts";
 import ObsidianLiveSyncPlugin from "../../../main.ts";
 import { scheduleTask } from "../../../common/utils.ts";
@@ -38,7 +36,6 @@ import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
 import { fireAndForget, yieldNextAnimationFrame } from "octagonal-wheels/promises";
 import { confirmWithMessage } from "../../coreObsidian/UILib/dialogs.ts";
 import { EVENT_REQUEST_RELOAD_SETTING_TAB, eventHub } from "../../../common/events.ts";
-import { skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
 import { JournalSyncMinio } from "../../../lib/src/replication/journal/objectstore/JournalSyncMinio.ts";
 import { paneChangeLog } from "./PaneChangeLog.ts";
 import {
@@ -861,49 +858,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
         });
     }
 
-    async dryRunGC() {
-        await skipIfDuplicated("cleanup", async () => {
-            const replicator = this.plugin.$$getReplicator();
-            if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
-            const remoteDBConn = await replicator.connectRemoteCouchDBWithSetting(
-                this.plugin.settings,
-                this.plugin.$$isMobile()
-            );
-            if (typeof remoteDBConn == "string") {
-                Logger(remoteDBConn);
-                return;
-            }
-            await purgeUnreferencedChunks(remoteDBConn.db, true, this.plugin.settings, false);
-            await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, true);
-            this.plugin.localDatabase.clearCaches();
-        });
-    }
-
-    async dbGC() {
-        // Lock the remote completely once.
-        await skipIfDuplicated("cleanup", async () => {
-            const replicator = this.plugin.$$getReplicator();
-            if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
-            await this.plugin.$$getReplicator().markRemoteLocked(this.plugin.settings, true, true);
-            const remoteDBConnection = await replicator.connectRemoteCouchDBWithSetting(
-                this.plugin.settings,
-                this.plugin.$$isMobile()
-            );
-            if (typeof remoteDBConnection == "string") {
-                Logger(remoteDBConnection);
-                return;
-            }
-            await purgeUnreferencedChunks(remoteDBConnection.db, false, this.plugin.settings, true);
-            await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, false);
-            this.plugin.localDatabase.clearCaches();
-            await balanceChunkPurgedDBs(this.plugin.localDatabase.localDatabase, remoteDBConnection.db);
-            this.plugin.localDatabase.refreshSettings();
-            Logger(
-                "The remote database has been cleaned up! Other devices will be cleaned up on the next synchronisation."
-            );
-        });
-    }
-
     getMinioJournalSyncClient() {
         const id = this.plugin.settings.accessKey;
         const key = this.plugin.settings.secretKey;

src/modules/features/SettingDialogue/PaneChangeLog.ts

@@ -3,6 +3,7 @@ import { versionNumberString2Number } from "../../../lib/src/string_and_binary/c
 import { $msg } from "../../../lib/src/common/i18n.ts";
 import { fireAndForget } from "octagonal-wheels/promises";
 import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts";
+import { visibleOnly } from "./SettingPane.ts";
 //@ts-ignore
 const manifestVersion: string = MANIFEST_VERSION || "-";
 //@ts-ignore
@@ -10,8 +11,34 @@ const updateInformation: string = UPDATE_INFO || "";
 
 const lastVersion = ~~(versionNumberString2Number(manifestVersion) / 1000);
 export function paneChangeLog(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement): void {
-    const informationDivEl = this.createEl(paneEl, "div", { text: "" });
+    const cx = this.createEl(
+        paneEl,
+        "div",
+        {
+            cls: "op-warn-info",
+        },
+        undefined,
+        visibleOnly(() => !this.isConfiguredAs("versionUpFlash", ""))
+    );
 
+    this.createEl(
+        cx,
+        "div",
+        {
+            text: this.editingSettings.versionUpFlash,
+        },
+        undefined
+    );
+    this.createEl(cx, "button", { text: $msg("obsidianLiveSyncSettingTab.btnGotItAndUpdated") }, (e) => {
+        e.addClass("mod-cta");
+        e.addEventListener("click", () => {
+            fireAndForget(async () => {
+                this.editingSettings.versionUpFlash = "";
+                await this.saveAllDirtySettings();
+            });
+        });
+    });
+    const informationDivEl = this.createEl(paneEl, "div", { text: "" });
     const tmpDiv = createDiv();
     // tmpDiv.addClass("sls-header-button");
     tmpDiv.addClass("op-warn-info");

src/modules/features/SettingDialogue/PaneHatch.ts

@@ -26,7 +26,7 @@ import { addPrefix, shouldBeIgnored, stripAllPrefixes } from "../../../lib/src/s
 import { $msg } from "../../../lib/src/common/i18n.ts";
 import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
 import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
-import { EVENT_REQUEST_RUN_DOCTOR, eventHub } from "../../../common/events.ts";
+import { EVENT_REQUEST_RUN_DOCTOR, EVENT_REQUEST_RUN_FIX_INCOMPLETE, eventHub } from "../../../common/events.ts";
 import { ICHeader, ICXHeader, PSCHeader } from "../../../common/types.ts";
 import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts";
 import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts";
@@ -50,6 +50,19 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
                         eventHub.emitEvent(EVENT_REQUEST_RUN_DOCTOR, "you wanted(Thank you)!");
                     })
             );
+        new Setting(paneEl)
+            .setName($msg("Setting.TroubleShooting.ScanBrokenFiles"))
+            .setDesc($msg("Setting.TroubleShooting.ScanBrokenFiles.Desc"))
+            .addButton((button) =>
+                button
+                    .setButtonText("Scan for Broken files")
+                    .setCta()
+                    .setDisabled(false)
+                    .onClick(() => {
+                        this.closeSetting();
+                        eventHub.emitEvent(EVENT_REQUEST_RUN_FIX_INCOMPLETE);
+                    })
+            );
         new Setting(paneEl).setName("Prepare the 'report' to create an issue").addButton((button) =>
             button
                 .setButtonText("Copy Report to clipboard")
@@ -190,7 +203,7 @@ ${stringifyYaml({
                             );
                             infoGroupEl.appendChild(
                                 this.createEl(infoGroupEl, "div", {
-                                    text: `Database: Modified: ${!fileOnDB ? `Missing:` : `${new Date(fileOnDB.mtime).toLocaleString()}, Size:${fileOnDB.size}`}`,
+                                    text: `Database: Modified: ${!fileOnDB ? `Missing:` : `${new Date(fileOnDB.mtime).toLocaleString()}, Size:${fileOnDB.size} (actual size:${readAsBlob(fileOnDB).size})`}`,
                                 })
                             );
                         })

src/modules/features/SettingDialogue/PanePatches.ts

@@ -82,6 +82,7 @@ export function panePatches(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElemen
     void addPanel(paneEl, "Edge case addressing (Behaviour)").then((paneEl) => {
         new Setting(paneEl).autoWireToggle("doNotSuspendOnFetching");
         new Setting(paneEl).setClass("wizardHidden").autoWireToggle("doNotDeleteFolder");
+        new Setting(paneEl).autoWireToggle("processSizeMismatchedFiles");
     });
 
     void addPanel(paneEl, "Edge case addressing (Processing)").then((paneEl) => {

src/modules/features/SettingDialogue/PaneSyncSettings.ts

@@ -7,7 +7,6 @@ import {
 import { Logger } from "../../../lib/src/common/logger.ts";
 import { $msg } from "../../../lib/src/common/i18n.ts";
 import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
-import { fireAndForget } from "octagonal-wheels/promises";
 import { EVENT_REQUEST_COPY_SETUP_URI, eventHub } from "../../../common/events.ts";
 import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts";
 import type { PageFunctions } from "./SettingPane.ts";
@@ -17,30 +16,6 @@ export function paneSyncSettings(
     paneEl: HTMLElement,
     { addPanel, addPane }: PageFunctions
 ): void {
-    if (this.editingSettings.versionUpFlash != "") {
-        const c = this.createEl(
-            paneEl,
-            "div",
-            {
-                text: this.editingSettings.versionUpFlash,
-                cls: "op-warn sls-setting-hidden",
-            },
-            (el) => {
-                this.createEl(el, "button", { text: $msg("obsidianLiveSyncSettingTab.btnGotItAndUpdated") }, (e) => {
-                    e.addClass("mod-cta");
-                    e.addEventListener("click", () => {
-                        fireAndForget(async () => {
-                            this.editingSettings.versionUpFlash = "";
-                            await this.saveAllDirtySettings();
-                            c.remove();
-                        });
-                    });
-                });
-            },
-            visibleOnly(() => !this.isConfiguredAs("versionUpFlash", ""))
-        );
-    }
-
     this.createEl(paneEl, "div", {
         text: $msg("obsidianLiveSyncSettingTab.msgSelectAndApplyPreset"),
         cls: "wizardOnly",