Merge pull request #1326 from vrtmrz/fix/body-size-and-verify

Prevent and repair incorrect file size records

src/common/messages/LiveSyncProvisionalMessages.ts

@@ -286,6 +286,7 @@ export const liveSyncProvisionalEnglishMessages = {
     "Apply database revision to Vault": "Apply database revision to Vault",
     "Mark this revision as the Vault version": "Mark this revision as the Vault version",
     "Store Vault file as a child of this revision": "Store Vault file as a child of this revision",
+    "Correct recorded file size": "Correct recorded file size",
     "Apply logical deletion to Vault": "Apply logical deletion to Vault",
     "Apply logical deletion ${REVISION} to ${FILE}? The current Vault file will be removed.":
         "Apply logical deletion ${REVISION} to ${FILE}? The current Vault file will be removed.",

src/modules/features/SettingDialogue/PaneHatch.ts

@@ -32,12 +32,14 @@ import {
     chooseAndCopyFileDatabaseInfo,
     collectFileDatabaseInfoPaths,
     copyFileDatabaseInfo,
+    inspectFileDatabaseInfo,
     retryReadFileDatabaseRevision,
 } from "@/serviceFeatures/fileDatabaseInfo.ts";
 import {
     discardLiveBranch,
     discardUnreadableLiveRevision,
     inspectFileRepair,
+    repairRevisionSize,
     type FileRepairInspection,
     type FileRepairRevision,
 } from "@/serviceFeatures/fileRepair.ts";
@@ -786,6 +788,31 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
                         },
                     });
                 }
+                if (metadata.revision && policy.repairSize) {
+                    revisionActions.push({
+                        title: $msg("Correct recorded file size"),
+                        run: async () => {
+                            await runMutation(
+                                `Correct the recorded size on database revision ${metadata.revision}`,
+                                async () => {
+                                    const result = await repairRevisionSize(
+                                        this.core,
+                                        path,
+                                        metadata.revision!
+                                    );
+                                    if (result !== "repaired" && result !== "unchanged") {
+                                        return false;
+                                    }
+                                    const current = await inspectFileDatabaseInfo(this.core, path);
+                                    const currentRevision = current.database.currentRevision;
+                                    return currentRevision !== null
+                                        ? storeStorageOnRevision(path, currentRevision, false)
+                                        : false;
+                                }
+                            );
+                        },
+                    });
+                }
                 if (metadata.revision && policy.storeVaultOnBranch) {
                     revisionActions.push({
                         title: $msg("Store Vault file as a child of this revision"),

src/serviceFeatures/fileRepair.ts

@@ -1,3 +1,4 @@
+import { isNotFoundError } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc";
 import type { LoadedEntry } from "@vrtmrz/livesync-commonlib/compat/common/types";
 import { createBlob, isDocContentSame, readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
 import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
@@ -12,6 +13,8 @@ import {
 
 export type FileRepairCore = FileDatabaseInfoCore & {
     fileHandler: Pick<IFileHandler, "deleteRevisionFromDB">;
+    localDatabase: FileDatabaseInfoCore["localDatabase"] &
+        Pick<import("@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB").LiveSyncLocalDB, "putRaw">;
     storageAccess: FileDatabaseInfoCore["storageAccess"] & Pick<StorageAccess, "readHiddenFileBinary">;
 };
 
@@ -29,11 +32,7 @@ export type FileRepairInspection = {
     requiresAttention: boolean;
 };
 
-export type DiscardUnreadableRevisionResult =
-    | "discarded"
-    | "failed"
-    | "no-longer-live"
-    | "revision-is-readable";
+export type DiscardUnreadableRevisionResult = "discarded" | "failed" | "no-longer-live" | "revision-is-readable";
 
 export type DiscardLiveBranchResult = "discarded" | "failed" | "no-longer-live" | "only-live-revision";
 
@@ -67,9 +66,7 @@ export async function inspectFileRepair(core: FileRepairCore, path: string): Pro
     const winnerRepresentsStoredFile = winner !== undefined && !winner.metadata.deleted;
     const databaseAndStorageDiffer =
         information.storage.exists !== winnerRepresentsStoredFile ||
-        (information.storage.exists &&
-            winnerRepresentsStoredFile &&
-            winner.contentMatchesStorage === false);
+        (information.storage.exists && winnerRepresentsStoredFile && winner.contentMatchesStorage === false);
     const unreadableLiveRevision =
         information.database.unavailableConflictRevisions.length > 0 ||
         revisions.some(({ contentReadable }) => !contentReadable);
@@ -77,6 +74,10 @@ export async function inspectFileRepair(core: FileRepairCore, path: string): Pro
         databaseAndStorageDiffer ||
         information.database.conflictCount > 0 ||
         unreadableLiveRevision ||
+        revisions.some(
+            ({ loadedEntry, metadata }) =>
+                loadedEntry !== false && readAsBlob(loadedEntry).size !== metadata.recordedSize
+        ) ||
         (information.database.exists && winner === undefined);
 
     return {
@@ -92,10 +93,9 @@ export async function discardUnreadableLiveRevision(
     revision: string
 ): Promise<DiscardUnreadableRevisionResult> {
     const latest = await inspectFileDatabaseInfo(core, path);
-    const liveRevisions = [
-        latest.database.currentRevision,
-        ...latest.database.conflictRevisions,
-    ].filter((candidate): candidate is string => candidate !== null);
+    const liveRevisions = [latest.database.currentRevision, ...latest.database.conflictRevisions].filter(
+        (candidate): candidate is string => candidate !== null
+    );
     if (!liveRevisions.includes(revision)) {
         return "no-longer-live";
     }
@@ -116,10 +116,9 @@ export async function discardLiveBranch(
     revision: string
 ): Promise<DiscardLiveBranchResult> {
     const latest = await inspectFileDatabaseInfo(core, path);
-    const liveRevisions = [
-        latest.database.currentRevision,
-        ...latest.database.conflictRevisions,
-    ].filter((candidate): candidate is string => candidate !== null);
+    const liveRevisions = [latest.database.currentRevision, ...latest.database.conflictRevisions].filter(
+        (candidate): candidate is string => candidate !== null
+    );
     if (!liveRevisions.includes(revision)) {
         return "no-longer-live";
     }
@@ -130,3 +129,74 @@ export async function discardLiveBranch(
     const deleted = await core.fileHandler.deleteRevisionFromDB(latest.databasePath, revision);
     return deleted ? "discarded" : "failed";
 }
+
+export function canRepairRevisionSize(inspection: FileRepairInspection, revision: FileRepairRevision): boolean {
+    return (
+        revision.role === "winner" &&
+        revision.metadata.current &&
+        revision.metadata.revision !== null &&
+        revision.metadata.revision === inspection.information.database.currentRevision &&
+        revision.contentReadable &&
+        !revision.metadata.deleted &&
+        revision.metadata.storageLayout === "chunked" &&
+        inspection.information.database.conflictCount === 0 &&
+        inspection.information.database.unavailableConflictRevisions.length === 0 &&
+        inspection.information.storage.exists &&
+        revision.contentMatchesStorage === true &&
+        revision.loadedEntry !== false &&
+        readAsBlob(revision.loadedEntry).size !== revision.metadata.recordedSize
+    );
+}
+
+export async function repairRevisionSize(
+    core: FileRepairCore,
+    path: string,
+    revision: string
+): Promise<"repaired" | "unchanged" | "stale" | "unsafe" | "failed"> {
+    const inspection = await inspectFileRepair(core, path);
+    if (inspection.information.database.currentRevision !== revision) return "stale";
+    const winner = inspection.revisions.find((candidate) => candidate.role === "winner");
+    if (!winner || winner.loadedEntry === false) return "unsafe";
+    if (!canRepairRevisionSize(inspection, winner)) {
+        return !winner.metadata.deleted &&
+            winner.contentMatchesStorage === true &&
+            inspection.information.database.conflictCount === 0 &&
+            readAsBlob(winner.loadedEntry).size === winner.metadata.recordedSize
+            ? "unchanged"
+            : "unsafe";
+    }
+    const content = readAsBlob(winner.loadedEntry);
+    const raw = await core.localDatabase.localDatabase
+        .get(winner.metadata.documentId, { conflicts: true })
+        .catch((error: unknown) => {
+            if (isNotFoundError(error)) return false as const;
+            throw error;
+        });
+    if (raw === false) return "stale";
+    if (
+        (raw.type !== "plain" && raw.type !== "newnote") ||
+        raw._rev !== revision ||
+        raw._deleted ||
+        ("deleted" in raw && raw.deleted) ||
+        raw._conflicts?.length
+    )
+        return "stale";
+    if (
+        !(await core.storageAccess.isExistsIncludeHidden(path)) ||
+        !(await isDocContentSame(content, createBlob(await core.storageAccess.readHiddenFileBinary(path))))
+    ) {
+        return "unsafe";
+    }
+    // Preserve the exact stored document; loaded content must never be embedded in Metadata.
+    const corrected = { ...raw, size: content.size };
+    delete corrected._conflicts;
+    delete corrected._revs_info;
+    delete corrected._revisions;
+    try {
+        const result = await core.localDatabase.putRaw(corrected);
+        return result.ok ? "repaired" : "failed";
+    } catch (error) {
+        if ((error as { status?: number }).status === 409) return "stale";
+        throw error;
+    }
+}

src/serviceFeatures/fileRepair.unit.spec.ts

@@ -1,9 +1,8 @@
+import PouchDB from "pouchdb-core";
+import MemoryAdapter from "pouchdb-adapter-memory";
+PouchDB.plugin(MemoryAdapter);
 import { describe, expect, it, vi } from "vitest";
-import {
-    discardLiveBranch,
-    discardUnreadableLiveRevision,
-    inspectFileRepair,
-} from "./fileRepair";
+import { discardLiveBranch, discardUnreadableLiveRevision, inspectFileRepair, repairRevisionSize } from "./fileRepair";
 
 function createCore() {
     const current = {
@@ -48,6 +47,7 @@ function createCore() {
             getFilesIncludeHidden: vi.fn(async () => ["note.md"]),
         },
         localDatabase: {
+            putRaw: vi.fn(async (_doc: unknown) => ({ ok: true, id: "f:note", rev: "4-repaired" })),
             localDatabase: {
                 get: vi.fn(async (_id: string, options?: { rev?: string }) =>
                     options?.rev === "2-conflict" ? conflict : current
@@ -94,6 +94,20 @@ function createCore() {
 }
 
 describe("file repair inspection", () => {
+    it("reports a size-only mismatch even when Vault bytes match", async () => {
+        const { core, current } = createCore();
+        current._conflicts = [];
+        current.size = 45;
+        expect((await inspectFileRepair(core as never, "note.md")).requiresAttention).toBe(true);
+    });
+
+    it("omits matching bytes with correct size despite differing timestamps", async () => {
+        const { core, current } = createCore();
+        current._conflicts = [];
+        current.mtime = 9000;
+        expect((await inspectFileRepair(core as never, "note.md")).requiresAttention).toBe(false);
+    });
+
     it("shows the winner and every conflict revision independently", async () => {
         const { core } = createCore();
 
@@ -146,12 +160,10 @@ describe("file repair inspection", () => {
     it("rechecks liveness and readability before discarding an exact revision", async () => {
         const { core, deleteRevisionFromDB } = createCore();
 
-        await expect(
-            discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")
-        ).resolves.toBe("discarded");
-        await expect(
-            discardUnreadableLiveRevision(core as never, "note.md", "3-current")
-        ).resolves.toBe("revision-is-readable");
+        await expect(discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")).resolves.toBe("discarded");
+        await expect(discardUnreadableLiveRevision(core as never, "note.md", "3-current")).resolves.toBe(
+            "revision-is-readable"
+        );
 
         expect(deleteRevisionFromDB).toHaveBeenCalledOnce();
         expect(deleteRevisionFromDB).toHaveBeenCalledWith("note.md", "2-conflict");
@@ -175,9 +187,7 @@ describe("file repair inspection", () => {
                 }),
             }),
         ]);
-        await expect(
-            discardUnreadableLiveRevision(core as never, "note.md", "1-root")
-        ).resolves.toBe("discarded");
+        await expect(discardUnreadableLiveRevision(core as never, "note.md", "1-root")).resolves.toBe("discarded");
         expect(deleteRevisionFromDB).toHaveBeenCalledWith("note.md", "1-root");
     });
 
@@ -188,9 +198,9 @@ describe("file repair inspection", () => {
             _conflicts: [],
         });
 
-        await expect(
-            discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")
-        ).resolves.toBe("no-longer-live");
+        await expect(discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")).resolves.toBe(
+            "no-longer-live"
+        );
 
         expect(deleteRevisionFromDB).not.toHaveBeenCalled();
     });
@@ -198,9 +208,7 @@ describe("file repair inspection", () => {
     it("discards an exact readable winner while another live branch remains", async () => {
         const { core, deleteRevisionFromDB } = createCore();
 
-        await expect(
-            discardLiveBranch(core as never, "note.md", "3-current")
-        ).resolves.toBe("discarded");
+        await expect(discardLiveBranch(core as never, "note.md", "3-current")).resolves.toBe("discarded");
 
         expect(deleteRevisionFromDB).toHaveBeenCalledWith("note.md", "3-current");
     });
@@ -209,9 +217,7 @@ describe("file repair inspection", () => {
         const { core, current, deleteRevisionFromDB } = createCore();
         current._conflicts = [];
 
-        await expect(
-            discardLiveBranch(core as never, "note.md", "3-current")
-        ).resolves.toBe("only-live-revision");
+        await expect(discardLiveBranch(core as never, "note.md", "3-current")).resolves.toBe("only-live-revision");
 
         expect(deleteRevisionFromDB).not.toHaveBeenCalled();
     });
@@ -219,10 +225,112 @@ describe("file repair inspection", () => {
     it("refuses to discard a branch which is no longer live", async () => {
         const { core, deleteRevisionFromDB } = createCore();
 
-        await expect(
-            discardLiveBranch(core as never, "note.md", "1-stale")
-        ).resolves.toBe("no-longer-live");
+        await expect(discardLiveBranch(core as never, "note.md", "1-stale")).resolves.toBe("no-longer-live");
 
         expect(deleteRevisionFromDB).not.toHaveBeenCalled();
     });
 });
+
+describe("recorded size repair", () => {
+    function repairFixture() {
+        const fixture = createCore();
+        fixture.current._conflicts = [];
+        fixture.current.size = 45;
+        return fixture;
+    }
+    it("writes a child changing only size and omitting response metadata", async () => {
+        const { core, current } = repairFixture();
+        expect(await repairRevisionSize(core as never, "note.md", current._rev)).toBe("repaired");
+        const corrected = core.localDatabase.putRaw.mock.calls[0][0];
+        const expected = { ...current, size: 7 } as Partial<typeof current>;
+        delete expected._conflicts;
+        delete expected._revs_info;
+        expect(corrected).toEqual(expected);
+    });
+    it.each(["conflict", "different-body", "missing-chunk", "deleted", "absent-vault"])("refuses %s", async (state) => {
+        const { core, current } = repairFixture();
+        if (state === "conflict") current._conflicts = ["2-conflict"];
+        if (state === "different-body")
+            core.storageAccess.readHiddenFileBinary.mockResolvedValue(new TextEncoder().encode("edit").buffer);
+        if (state === "missing-chunk") core.localDatabase.allDocsRaw.mockResolvedValue({ rows: [] });
+        if (state === "deleted") current.deleted = true;
+        if (state === "absent-vault") core.storageAccess.isExistsIncludeHidden.mockResolvedValue(false);
+        expect(await repairRevisionSize(core as never, "note.md", current._rev)).toBe("unsafe");
+        expect(core.localDatabase.putRaw).not.toHaveBeenCalled();
+    });
+    it("refuses a selected revision replaced before inspection", async () => {
+        const { core } = repairFixture();
+        expect(await repairRevisionSize(core as never, "note.md", "2-old")).toBe("stale");
+        expect(core.localDatabase.putRaw).not.toHaveBeenCalled();
+    });
+    it("refuses a revision advancing between inspection and final metadata read", async () => {
+        const { core, current } = repairFixture();
+        const read = core.localDatabase.localDatabase.get;
+        read.mockResolvedValueOnce(current)
+            .mockResolvedValueOnce(current)
+            .mockResolvedValueOnce({ ...current, _rev: "4-advanced" });
+        expect(await repairRevisionSize(core as never, "note.md", current._rev)).toBe("stale");
+        expect(core.localDatabase.putRaw).not.toHaveBeenCalled();
+    });
+    it("refuses a Vault edit after inspection", async () => {
+        const { core, current } = repairFixture();
+        core.storageAccess.readHiddenFileBinary
+            .mockResolvedValueOnce(new TextEncoder().encode("current").buffer)
+            .mockResolvedValueOnce(new TextEncoder().encode("new edit").buffer);
+        expect(await repairRevisionSize(core as never, "note.md", current._rev)).toBe("unsafe");
+        expect(core.localDatabase.putRaw).not.toHaveBeenCalled();
+    });
+    it("treats a document removed after inspection as stale", async () => {
+        const { core, current } = repairFixture();
+        core.localDatabase.localDatabase.get
+            .mockResolvedValueOnce(current)
+            .mockResolvedValueOnce(current)
+            .mockRejectedValueOnce({ status: 404 });
+        expect(await repairRevisionSize(core as never, "note.md", current._rev)).toBe("stale");
+        expect(core.localDatabase.putRaw).not.toHaveBeenCalled();
+    });
+    it("refuses advancement during the last Vault read through CAS", async () => {
+        const { core, current } = repairFixture();
+        core.localDatabase.putRaw.mockRejectedValue({ status: 409 });
+        expect(await repairRevisionSize(core as never, "note.md", current._rev)).toBe("stale");
+    });
+    it("does not repeat a correction or change mtime for matching content", async () => {
+        const { core, current } = repairFixture();
+        current.size = 7;
+        expect(await repairRevisionSize(core as never, "note.md", current._rev)).toBe("unchanged");
+        expect(core.localDatabase.putRaw).not.toHaveBeenCalled();
+    });
+});
+
+describe("size repair revision ancestry", () => {
+    it("preserves metadata and chunks in a real PouchDB child and is idempotent", async () => {
+        const { core, current } = createCore();
+        const database = new PouchDB(`size-repair-${crypto.randomUUID()}`, { adapter: "memory" });
+        try {
+            const metadata = { ...current } as Partial<typeof current> & { _id: string };
+            delete metadata._rev;
+            delete metadata._conflicts;
+            delete metadata._revs_info;
+            const stored = await database.put({ ...metadata, size: 45 });
+            core.localDatabase.localDatabase = database as never;
+            core.localDatabase.putRaw.mockImplementation((doc) => database.put(doc as never));
+            core.localDatabase.getDBEntryFromMeta.mockImplementation(async (meta) => ({ ...meta, data: ["current"] }));
+            expect(await repairRevisionSize(core as never, "note.md", stored.rev)).toBe("repaired");
+            const corrected = (await database.get(current._id, { revs: true })) as typeof current & {
+                _revisions: { ids: string[] };
+            };
+            expect(corrected.size).toBe(7);
+            expect(corrected.children).toEqual(current.children);
+            expect(corrected.ctime).toBe(current.ctime);
+            expect(corrected.mtime).toBe(current.mtime);
+            expect(corrected.path).toBe(current.path);
+            expect(corrected).not.toHaveProperty("data");
+            expect(corrected._revisions.ids[1]).toBe(stored.rev.split("-")[1]);
+            expect(await repairRevisionSize(core as never, "note.md", corrected._rev)).toBe("unchanged");
+            expect((await inspectFileRepair(core as never, "note.md")).requiresAttention).toBe(false);
+            expect(core.localDatabase.putRaw).toHaveBeenCalledOnce();
+        } finally {
+            await database.destroy();
+        }
+    });
+});

src/serviceFeatures/fileRepairPresentation.ts

@@ -8,10 +8,7 @@ import {
     readAsBlob,
 } from "@vrtmrz/livesync-commonlib/compat/common/utils";
 import { isPlainText } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
-import type {
-    FileRepairInspection,
-    FileRepairRevision,
-} from "./fileRepair";
+import { canRepairRevisionSize, type FileRepairInspection, type FileRepairRevision } from "./fileRepair";
 
 export type FileRepairRevisionActions = {
     compareWithVault: boolean;
@@ -22,6 +19,7 @@ export type FileRepairRevisionActions = {
     retryRevision: boolean;
     discardBranch: boolean;
     discardRevision: boolean;
+    repairSize: boolean;
 };
 
 export type FileRepairTimestampRelation =
@@ -87,6 +85,7 @@ export function getFileRepairRevisionActions(
             !hasConflictBranches &&
             !revision.metadata.deleted &&
             !revision.contentReadable,
+        repairSize: canRepairRevisionSize(inspection, revision),
     };
 }
 

src/serviceFeatures/fileRepairPresentation.unit.spec.ts

@@ -87,6 +87,7 @@ describe("file repair presentation", () => {
             retryRevision: false,
             discardRevision: false,
             discardBranch: true,
+            repairSize: false,
         });
     });
 
@@ -104,6 +105,58 @@ describe("file repair presentation", () => {
         });
     });
 
+    it("offers size correction for a readable, matching, conflict-free winner", () => {
+        const { inspection, revision } = createInspection({
+            role: "winner",
+            metadata: {
+                ...createInspection().revision.metadata,
+                current: true,
+                recordedSize: 9,
+            },
+            contentMatchesStorage: true,
+        });
+        inspection.information.database.conflictCount = 0;
+        inspection.information.database.conflictRevisions = [];
+        inspection.information.database.currentRevision = revision.metadata.revision;
+        inspection.information.storage.size = 7;
+
+        expect(getFileRepairRevisionActions(inspection, revision)).toMatchObject({
+            repairSize: true,
+        });
+    });
+
+    it("does not offer size correction for unsafe or already-correct revisions", () => {
+        const conflict = createInspection({ contentMatchesStorage: true });
+        expect(getFileRepairRevisionActions(conflict.inspection, conflict.revision)).toMatchObject({
+            repairSize: false,
+        });
+
+        const unreadable = createInspection({
+            role: "winner",
+            contentReadable: false,
+            contentMatchesStorage: true,
+        });
+        unreadable.inspection.information.database.conflictCount = 0;
+        unreadable.inspection.information.database.conflictRevisions = [];
+        unreadable.inspection.information.storage.size = 7;
+        unreadable.revision.metadata.current = true;
+        unreadable.inspection.information.database.currentRevision = unreadable.revision.metadata.revision;
+        expect(getFileRepairRevisionActions(unreadable.inspection, unreadable.revision)).toMatchObject({
+            repairSize: false,
+        });
+
+        const unchanged = createInspection({
+            role: "winner",
+            contentMatchesStorage: true,
+        });
+        unchanged.inspection.information.database.conflictCount = 0;
+        unchanged.inspection.information.database.conflictRevisions = [];
+        unchanged.inspection.information.database.currentRevision = unchanged.revision.metadata.revision;
+        expect(getFileRepairRevisionActions(unchanged.inspection, unchanged.revision)).toMatchObject({
+            repairSize: false,
+        });
+    });
+
     it("does not offer a text comparison for a binary file", () => {
         const { inspection, revision } = createInspection();
         inspection.information.path = "image.png";
@@ -191,6 +244,16 @@ describe("file repair presentation", () => {
         });
     });
 
+    it("does not offer size correction for legacy inline content", () => {
+        const { inspection, revision } = createInspection({ role: "winner", contentMatchesStorage: true });
+        revision.metadata.current = true;
+        revision.metadata.storageLayout = "legacy-inline";
+        inspection.information.database.currentRevision = revision.metadata.revision;
+        inspection.information.database.conflictCount = 0;
+        inspection.information.database.conflictRevisions = [];
+        expect(getFileRepairRevisionActions(inspection, revision).repairSize).toBe(false);
+    });
+
     it("reports recorded, decoded, Vault-size, and timestamp differences", () => {
         const { inspection, revision } = createInspection();