From 5abba74f3b0318bd2075fbb705570bc22ca649a1 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 15 Aug 2025 10:51:39 +0100 Subject: [PATCH] ## 0.25.7 ### Fixed - Off-loaded chunking have been fixed to ensure proper functionality (#693). - Chunk document ID assignment has been fixed. - Replication prevention message during version up detection has been improved (#686). - `Keep A` and `Keep B` on Conflict resolving dialogue has been renamed to `Use Base` and `Use Conflicted` (#691). ### Improved - Metadata and content-size unmatched documents are now detected and reported, prevented to be applied to the storage. ### New Features - `Scan for Broken files` has been implemented on `Hatch` -> `TroubleShooting`. ### Refactored - Off-loaded processes have been refactored for the better maintainability. - Removed unused code. --- src/common/events.ts | 2 + src/lib | 2 +- src/modules/core/ModuleFileHandler.ts | 12 +- src/modules/essential/ModuleMigration.ts | 126 +++++++++++++++++- .../ConflictResolveModal.ts | 4 +- .../ObsidianLiveSyncSettingTab.ts | 46 ------- .../features/SettingDialogue/PaneChangeLog.ts | 29 +++- .../features/SettingDialogue/PaneHatch.ts | 17 ++- .../features/SettingDialogue/PanePatches.ts | 1 + .../SettingDialogue/PaneSyncSettings.ts | 25 ---- 10 files changed, 185 insertions(+), 79 deletions(-) diff --git a/src/common/events.ts b/src/common/events.ts index 94d4bf3..3f4ed9c 100644 --- a/src/common/events.ts +++ b/src/common/events.ts @@ -20,6 +20,7 @@ export const EVENT_REQUEST_OPEN_P2P = "request-open-p2p"; export const EVENT_REQUEST_CLOSE_P2P = "request-close-p2p"; export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor"; +export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete"; // export const EVENT_FILE_CHANGED = "file-changed"; @@ -38,6 +39,7 @@ declare global { [EVENT_REQUEST_COPY_SETUP_URI]: undefined; [EVENT_REQUEST_SHOW_SETUP_QR]: undefined; [EVENT_REQUEST_RUN_DOCTOR]: string; + [EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined; } } diff --git a/src/lib b/src/lib index 7a0d8e4..1f51336 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 7a0d8e449a1d9810ef8156a7352ed7098925eade +Subproject commit 1f51336162157983e63759808906366dad533fdb diff --git a/src/modules/core/ModuleFileHandler.ts b/src/modules/core/ModuleFileHandler.ts index 3a32d91..c72521f 100644 --- a/src/modules/core/ModuleFileHandler.ts +++ b/src/modules/core/ModuleFileHandler.ts @@ -18,7 +18,7 @@ import { getStoragePathFromUXFileInfo, markChangesAreSame, } from "../../common/utils"; -import { getDocDataAsArray, isDocContentSame, readContent } from "../../lib/src/common/utils"; +import { getDocDataAsArray, isDocContentSame, readAsBlob, readContent } from "../../lib/src/common/utils"; import { shouldBeIgnored } from "../../lib/src/string_and_binary/path"; import type { ICoreModule } from "../ModuleTypes"; import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; @@ -259,6 +259,16 @@ export class ModuleFileHandler extends AbstractModule implements ICoreModule { const docData = readContent(docRead); if (existOnStorage && !force) { + // If we want to process size mismatched files -- in case of having files created by some integrations, enable the toggle. + if (!this.settings.processSizeMismatchedFiles) { + // Check the file is not corrupted + // (Zero is a special case, may be created by some APIs and it might be acceptable). + if (docRead.size != 0 && docRead.size !== readAsBlob(docRead).size) { + this._log(`File ${path} seems to be corrupted! Writing prevented.`, LOG_LEVEL_NOTICE); + return false; + } + } + // The file is exist on the storage. Let's check the difference between the file and the entry. // But, if force is true, then it should be updated. // Ok, we have to compare. diff --git a/src/modules/essential/ModuleMigration.ts b/src/modules/essential/ModuleMigration.ts index 05e38fc..8327d52 100644 --- a/src/modules/essential/ModuleMigration.ts +++ b/src/modules/essential/ModuleMigration.ts @@ -1,16 +1,20 @@ -import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; +import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "../../lib/src/common/logger.ts"; import { EVENT_REQUEST_OPEN_P2P, EVENT_REQUEST_OPEN_SETTING_WIZARD, EVENT_REQUEST_OPEN_SETTINGS, EVENT_REQUEST_OPEN_SETUP_URI, EVENT_REQUEST_RUN_DOCTOR, + EVENT_REQUEST_RUN_FIX_INCOMPLETE, eventHub, } from "../../common/events.ts"; import { AbstractModule } from "../AbstractModule.ts"; import type { ICoreModule } from "../ModuleTypes.ts"; import { $msg } from "src/lib/src/common/i18n.ts"; import { performDoctorConsultation, RebuildOptions } from "../../lib/src/common/configForDoc.ts"; +import { getPath, isValidPath } from "../../common/utils.ts"; +import { isMetaEntry } from "../../lib/src/common/types.ts"; +import { isDeletedEntry, isDocContentSame, isLoadedEntry, readAsBlob } from "../../lib/src/common/utils.ts"; export class ModuleMigration extends AbstractModule implements ICoreModule { async migrateUsingDoctor(skipRebuild: boolean = false, activateReason = "updated", forceRescan = false) { @@ -96,12 +100,129 @@ export class ModuleMigration extends AbstractModule implements ICoreModule { return false; } + async checkIncompleteDocs(force: boolean = false): Promise { + const incompleteDocsChecked = (await this.core.kvDB.get("checkIncompleteDocs")) || false; + if (incompleteDocsChecked && !force) { + this._log("Incomplete docs check already done, skipping.", LOG_LEVEL_VERBOSE); + return Promise.resolve(true); + } + + this._log("Checking for incomplete documents...", LOG_LEVEL_NOTICE, "check-incomplete"); + const errorFiles = []; + for await (const metaDoc of this.localDatabase.findAllNormalDocs({ conflicts: true })) { + const path = getPath(metaDoc); + + if (!isValidPath(path)) { + continue; + } + if (!(await this.core.$$isTargetFile(path, true))) { + continue; + } + if (!isMetaEntry(metaDoc)) { + continue; + } + + const doc = await this.localDatabase.getDBEntryFromMeta(metaDoc); + if (!doc || !isLoadedEntry(doc)) { + continue; + } + if (isDeletedEntry(doc)) { + continue; + } + const storageFileContent = await this.core.storageAccess.readHiddenFileBinary(path); + // const storageFileBlob = createBlob(storageFileContent); + const sizeOnStorage = storageFileContent.byteLength; + const recordedSize = doc.size; + const docBlob = readAsBlob(doc); + const actualSize = docBlob.size; + if (recordedSize !== actualSize || sizeOnStorage !== actualSize || sizeOnStorage !== recordedSize) { + const contentMatched = await isDocContentSame(doc.data, storageFileContent); + errorFiles.push({ path, recordedSize, actualSize, storageSize: sizeOnStorage, contentMatched }); + Logger( + `Size mismatch for ${path}: ${recordedSize} (DB Recorded) , ${actualSize} (DB Stored) , ${sizeOnStorage} (Storage Stored), ${contentMatched ? "Content Matched" : "Content Mismatched"}` + ); + } + } + if (errorFiles.length == 0) { + Logger("No size mismatches found", LOG_LEVEL_NOTICE); + await this.core.kvDB.set("checkIncompleteDocs", true); + return Promise.resolve(true); + } + Logger(`Found ${errorFiles.length} size mismatches`, LOG_LEVEL_NOTICE); + // We have to repair them following rules and situations: + // A. DB Recorded != DB Stored + // A.1. DB Recorded == Storage Stored + // Possibly recoverable from storage. Just overwrite the DB content with storage content. + // A.2. Neither + // Probably it cannot be resolved on this device. Even if the storage content is larger than DB Recorded, it possibly corrupted. + // We do not fix it automatically. Leave it as is. Possibly other device can do this. + // B. DB Recorded == DB Stored , < Storage Stored + // Very fragile, if DB Recorded size is less than Storage Stored size, we possibly repair the content (The issue was `unexpectedly shortened file`). + // We do not fix it automatically, but it will be automatically overwritten in other process. + // C. DB Recorded == DB Stored , > Storage Stored + // Probably restored by the user by resolving A or B on other device, We should overwrite the storage + // Also do not fix it automatically. It should be overwritten by replication. + const recoverable = errorFiles.filter((e) => { + return e.recordedSize === e.storageSize; + }); + const unrecoverable = errorFiles.filter((e) => { + return e.recordedSize !== e.storageSize; + }); + const messageUnrecoverable = + unrecoverable.length > 0 + ? $msg("moduleMigration.fix0256.messageUnrecoverable", { + filesNotRecoverable: unrecoverable + .map((e) => `- ${e.path} (M: ${e.recordedSize}, A: ${e.actualSize}, S: ${e.storageSize})`) + .join("\n"), + }) + : ""; + + const message = $msg("moduleMigration.fix0256.message", { + files: recoverable + .map((e) => `- ${e.path} (M: ${e.recordedSize}, A: ${e.actualSize}, S: ${e.storageSize})`) + .join("\n"), + messageUnrecoverable, + }); + const CHECK_IT_LATER = $msg("moduleMigration.fix0256.buttons.checkItLater"); + const FIX = $msg("moduleMigration.fix0256.buttons.fix"); + const DISMISS = $msg("moduleMigration.fix0256.buttons.DismissForever"); + const ret = await this.core.confirm.askSelectStringDialogue(message, [CHECK_IT_LATER, FIX, DISMISS], { + title: $msg("moduleMigration.fix0256.title"), + defaultAction: CHECK_IT_LATER, + }); + if (ret == FIX) { + for (const file of recoverable) { + // Overwrite the database with the files on the storage + const stubFile = this.core.storageAccess.getFileStub(file.path); + if (stubFile == null) { + Logger(`Could not find stub file for ${file.path}`, LOG_LEVEL_NOTICE); + continue; + } + + stubFile.stat.mtime = Date.now(); + const result = await this.core.fileHandler.storeFileToDB(stubFile, true, false); + if (result) { + Logger(`Successfully restored ${file.path} from storage`); + } else { + Logger(`Failed to restore ${file.path} from storage`, LOG_LEVEL_NOTICE); + } + } + } else if (ret === DISMISS) { + // User chose to dismiss the issue + await this.core.kvDB.set("checkIncompleteDocs", true); + } + + return Promise.resolve(true); + } + async $everyOnFirstInitialize(): Promise { if (!this.localDatabase.isReady) { this._log($msg("moduleMigration.logLocalDatabaseNotReady"), LOG_LEVEL_NOTICE); return false; } if (this.settings.isConfigured) { + // TODO: Probably we have to check for insecure chunks + await this.checkIncompleteDocs(); await this.migrateUsingDoctor(false); // await this.migrationCheck(); await this.migrateDisableBulkSend(); @@ -120,6 +241,9 @@ export class ModuleMigration extends AbstractModule implements ICoreModule { eventHub.onEvent(EVENT_REQUEST_RUN_DOCTOR, async (reason) => { await this.migrateUsingDoctor(false, reason, true); }); + eventHub.onEvent(EVENT_REQUEST_RUN_FIX_INCOMPLETE, async () => { + await this.checkIncompleteDocs(true); + }); return Promise.resolve(true); } } diff --git a/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts b/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts index 3ef3654..c5af855 100644 --- a/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts +++ b/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts @@ -25,8 +25,8 @@ export class ConflictResolveModal extends Modal { title: string = "Conflicting changes"; pluginPickMode: boolean = false; - localName: string = "Keep A"; - remoteName: string = "Keep B"; + localName: string = "Use Base"; + remoteName: string = "Use Conflicted"; offEvent?: ReturnType; constructor(app: App, filename: string, diff: diff_result, pluginPickMode?: boolean, remoteName?: string) { diff --git a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts index 21d82ca..aaaa76b 100644 --- a/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts +++ b/src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts @@ -17,8 +17,6 @@ import { delay, isObjectDifferent, sizeToHumanReadable } from "../../../lib/src/ import { versionNumberString2Number } from "../../../lib/src/string_and_binary/convert.ts"; import { Logger } from "../../../lib/src/common/logger.ts"; import { checkSyncInfo } from "@/lib/src/pouchdb/negotiation.ts"; -import { balanceChunkPurgedDBs } from "@/lib/src/pouchdb/chunks.ts"; -import { purgeUnreferencedChunks } from "@/lib/src/pouchdb/chunks.ts"; import { testCrypt } from "../../../lib/src/encryption/e2ee_v2.ts"; import ObsidianLiveSyncPlugin from "../../../main.ts"; import { scheduleTask } from "../../../common/utils.ts"; @@ -38,7 +36,6 @@ import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; import { fireAndForget, yieldNextAnimationFrame } from "octagonal-wheels/promises"; import { confirmWithMessage } from "../../coreObsidian/UILib/dialogs.ts"; import { EVENT_REQUEST_RELOAD_SETTING_TAB, eventHub } from "../../../common/events.ts"; -import { skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; import { JournalSyncMinio } from "../../../lib/src/replication/journal/objectstore/JournalSyncMinio.ts"; import { paneChangeLog } from "./PaneChangeLog.ts"; import { @@ -861,49 +858,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { }); } - async dryRunGC() { - await skipIfDuplicated("cleanup", async () => { - const replicator = this.plugin.$$getReplicator(); - if (!(replicator instanceof LiveSyncCouchDBReplicator)) return; - const remoteDBConn = await replicator.connectRemoteCouchDBWithSetting( - this.plugin.settings, - this.plugin.$$isMobile() - ); - if (typeof remoteDBConn == "string") { - Logger(remoteDBConn); - return; - } - await purgeUnreferencedChunks(remoteDBConn.db, true, this.plugin.settings, false); - await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, true); - this.plugin.localDatabase.clearCaches(); - }); - } - - async dbGC() { - // Lock the remote completely once. - await skipIfDuplicated("cleanup", async () => { - const replicator = this.plugin.$$getReplicator(); - if (!(replicator instanceof LiveSyncCouchDBReplicator)) return; - await this.plugin.$$getReplicator().markRemoteLocked(this.plugin.settings, true, true); - const remoteDBConnection = await replicator.connectRemoteCouchDBWithSetting( - this.plugin.settings, - this.plugin.$$isMobile() - ); - if (typeof remoteDBConnection == "string") { - Logger(remoteDBConnection); - return; - } - await purgeUnreferencedChunks(remoteDBConnection.db, false, this.plugin.settings, true); - await purgeUnreferencedChunks(this.plugin.localDatabase.localDatabase, false); - this.plugin.localDatabase.clearCaches(); - await balanceChunkPurgedDBs(this.plugin.localDatabase.localDatabase, remoteDBConnection.db); - this.plugin.localDatabase.refreshSettings(); - Logger( - "The remote database has been cleaned up! Other devices will be cleaned up on the next synchronisation." - ); - }); - } - getMinioJournalSyncClient() { const id = this.plugin.settings.accessKey; const key = this.plugin.settings.secretKey; diff --git a/src/modules/features/SettingDialogue/PaneChangeLog.ts b/src/modules/features/SettingDialogue/PaneChangeLog.ts index f77d2e1..be0e0c3 100644 --- a/src/modules/features/SettingDialogue/PaneChangeLog.ts +++ b/src/modules/features/SettingDialogue/PaneChangeLog.ts @@ -3,6 +3,7 @@ import { versionNumberString2Number } from "../../../lib/src/string_and_binary/c import { $msg } from "../../../lib/src/common/i18n.ts"; import { fireAndForget } from "octagonal-wheels/promises"; import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts"; +import { visibleOnly } from "./SettingPane.ts"; //@ts-ignore const manifestVersion: string = MANIFEST_VERSION || "-"; //@ts-ignore @@ -10,8 +11,34 @@ const updateInformation: string = UPDATE_INFO || ""; const lastVersion = ~~(versionNumberString2Number(manifestVersion) / 1000); export function paneChangeLog(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement): void { - const informationDivEl = this.createEl(paneEl, "div", { text: "" }); + const cx = this.createEl( + paneEl, + "div", + { + cls: "op-warn-info", + }, + undefined, + visibleOnly(() => !this.isConfiguredAs("versionUpFlash", "")) + ); + this.createEl( + cx, + "div", + { + text: this.editingSettings.versionUpFlash, + }, + undefined + ); + this.createEl(cx, "button", { text: $msg("obsidianLiveSyncSettingTab.btnGotItAndUpdated") }, (e) => { + e.addClass("mod-cta"); + e.addEventListener("click", () => { + fireAndForget(async () => { + this.editingSettings.versionUpFlash = ""; + await this.saveAllDirtySettings(); + }); + }); + }); + const informationDivEl = this.createEl(paneEl, "div", { text: "" }); const tmpDiv = createDiv(); // tmpDiv.addClass("sls-header-button"); tmpDiv.addClass("op-warn-info"); diff --git a/src/modules/features/SettingDialogue/PaneHatch.ts b/src/modules/features/SettingDialogue/PaneHatch.ts index 648b2c5..c42f297 100644 --- a/src/modules/features/SettingDialogue/PaneHatch.ts +++ b/src/modules/features/SettingDialogue/PaneHatch.ts @@ -26,7 +26,7 @@ import { addPrefix, shouldBeIgnored, stripAllPrefixes } from "../../../lib/src/s import { $msg } from "../../../lib/src/common/i18n.ts"; import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; -import { EVENT_REQUEST_RUN_DOCTOR, eventHub } from "../../../common/events.ts"; +import { EVENT_REQUEST_RUN_DOCTOR, EVENT_REQUEST_RUN_FIX_INCOMPLETE, eventHub } from "../../../common/events.ts"; import { ICHeader, ICXHeader, PSCHeader } from "../../../common/types.ts"; import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts"; import { EVENT_REQUEST_SHOW_HISTORY } from "../../../common/obsidianEvents.ts"; @@ -50,6 +50,19 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, eventHub.emitEvent(EVENT_REQUEST_RUN_DOCTOR, "you wanted(Thank you)!"); }) ); + new Setting(paneEl) + .setName($msg("Setting.TroubleShooting.ScanBrokenFiles")) + .setDesc($msg("Setting.TroubleShooting.ScanBrokenFiles.Desc")) + .addButton((button) => + button + .setButtonText("Scan for Broken files") + .setCta() + .setDisabled(false) + .onClick(() => { + this.closeSetting(); + eventHub.emitEvent(EVENT_REQUEST_RUN_FIX_INCOMPLETE); + }) + ); new Setting(paneEl).setName("Prepare the 'report' to create an issue").addButton((button) => button .setButtonText("Copy Report to clipboard") @@ -190,7 +203,7 @@ ${stringifyYaml({ ); infoGroupEl.appendChild( this.createEl(infoGroupEl, "div", { - text: `Database: Modified: ${!fileOnDB ? `Missing:` : `${new Date(fileOnDB.mtime).toLocaleString()}, Size:${fileOnDB.size}`}`, + text: `Database: Modified: ${!fileOnDB ? `Missing:` : `${new Date(fileOnDB.mtime).toLocaleString()}, Size:${fileOnDB.size} (actual size:${readAsBlob(fileOnDB).size})`}`, }) ); }) diff --git a/src/modules/features/SettingDialogue/PanePatches.ts b/src/modules/features/SettingDialogue/PanePatches.ts index b4256de..9abde61 100644 --- a/src/modules/features/SettingDialogue/PanePatches.ts +++ b/src/modules/features/SettingDialogue/PanePatches.ts @@ -82,6 +82,7 @@ export function panePatches(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElemen void addPanel(paneEl, "Edge case addressing (Behaviour)").then((paneEl) => { new Setting(paneEl).autoWireToggle("doNotSuspendOnFetching"); new Setting(paneEl).setClass("wizardHidden").autoWireToggle("doNotDeleteFolder"); + new Setting(paneEl).autoWireToggle("processSizeMismatchedFiles"); }); void addPanel(paneEl, "Edge case addressing (Processing)").then((paneEl) => { diff --git a/src/modules/features/SettingDialogue/PaneSyncSettings.ts b/src/modules/features/SettingDialogue/PaneSyncSettings.ts index 42982c9..2cdbc74 100644 --- a/src/modules/features/SettingDialogue/PaneSyncSettings.ts +++ b/src/modules/features/SettingDialogue/PaneSyncSettings.ts @@ -7,7 +7,6 @@ import { import { Logger } from "../../../lib/src/common/logger.ts"; import { $msg } from "../../../lib/src/common/i18n.ts"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; -import { fireAndForget } from "octagonal-wheels/promises"; import { EVENT_REQUEST_COPY_SETUP_URI, eventHub } from "../../../common/events.ts"; import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts"; import type { PageFunctions } from "./SettingPane.ts"; @@ -17,30 +16,6 @@ export function paneSyncSettings( paneEl: HTMLElement, { addPanel, addPane }: PageFunctions ): void { - if (this.editingSettings.versionUpFlash != "") { - const c = this.createEl( - paneEl, - "div", - { - text: this.editingSettings.versionUpFlash, - cls: "op-warn sls-setting-hidden", - }, - (el) => { - this.createEl(el, "button", { text: $msg("obsidianLiveSyncSettingTab.btnGotItAndUpdated") }, (e) => { - e.addClass("mod-cta"); - e.addEventListener("click", () => { - fireAndForget(async () => { - this.editingSettings.versionUpFlash = ""; - await this.saveAllDirtySettings(); - c.remove(); - }); - }); - }); - }, - visibleOnly(() => !this.isConfiguredAs("versionUpFlash", "")) - ); - } - this.createEl(paneEl, "div", { text: $msg("obsidianLiveSyncSettingTab.msgSelectAndApplyPreset"), cls: "wizardOnly",