Merge pull request #1283 from s-qin/perf/startup-storage-stat

Avoid whole-file storage reads during healthy startup integrity checks

src/serviceFeatures/startupLifecycle/incompleteDocuments.ts

@@ -31,7 +31,7 @@ export interface IncompleteDocumentsDependencies {
     readonly localDatabase: Pick<LiveSyncLocalDB, "findAllNormalDocs" | "getDBEntryFromMeta">;
     readonly getPath: StartupPathReader;
     readonly isTargetFile: (path: string) => Promise<boolean>;
-    readonly storageAccess: Pick<StorageAccess, "readHiddenFileBinary" | "getFileStub">;
+    readonly storageAccess: Pick<StorageAccess, "readHiddenFileBinary" | "statHidden" | "getFileStub">;
     readonly fileHandler: Pick<IFileHandler, "storeFileToDB">;
     readonly keyValueDB: Pick<KeyValueDatabase, "get" | "set">;
     readonly noticeGroups: NoticeGroups;
@@ -94,16 +94,11 @@ export async function checkIncompleteDocuments(
             }
             const isConflicted = metaDoc?._conflicts && metaDoc._conflicts.length > 0;
 
-            let storageFileContent;
-            try {
-                storageFileContent = await dependencies.storageAccess.readHiddenFileBinary(path);
-            } catch (e) {
-                Logger(`Failed to read file ${path}: Possibly unprocessed or missing`);
-                Logger(e, LOG_LEVEL_VERBOSE);
+            const storageStat = await dependencies.storageAccess.statHidden(path);
+            if (!storageStat || storageStat.type !== "file") {
                 continue;
             }
-            // const storageFileBlob = createBlob(storageFileContent);
-            const sizeOnStorage = storageFileContent.byteLength;
+            let sizeOnStorage = storageStat.size;
             const recordedSize = doc.size;
             const docBlob = readAsBlob(doc);
             const actualSize = docBlob.size;
@@ -113,6 +108,19 @@ export async function checkIncompleteDocuments(
                 sizeOnStorage !== recordedSize ||
                 isConflicted
             ) {
+                let storageFileContent;
+                try {
+                    storageFileContent = await dependencies.storageAccess.readHiddenFileBinary(path);
+                } catch (e) {
+                    Logger(`Failed to read file ${path}: Possibly unprocessed or missing`);
+                    Logger(e, LOG_LEVEL_VERBOSE);
+                    continue;
+                }
+                sizeOnStorage = storageFileContent.byteLength;
+                // The file may have changed since stat, so recheck using the bytes read.
+                if (recordedSize === actualSize && sizeOnStorage === actualSize && !isConflicted) {
+                    continue;
+                }
                 const contentMatched = await isDocContentSame(doc.data, storageFileContent);
                 errorFiles.push({
                     path,
@@ -125,6 +133,17 @@ export async function checkIncompleteDocuments(
                 Logger(
                     `Size mismatch for ${path}: ${recordedSize} (DB Recorded) , ${actualSize} (DB Stored) , ${sizeOnStorage} (Storage Stored), ${contentMatched ? "Content Matched" : "Content Mismatched"} ${isConflicted ? "Conflicted" : "Not Conflicted"}`
                 );
+            } else {
+                const currentStat = await dependencies.storageAccess.statHidden(path);
+                if (
+                    !currentStat ||
+                    currentStat.type !== "file" ||
+                    currentStat.size !== storageStat.size ||
+                    currentStat.mtime !== storageStat.mtime ||
+                    currentStat.ctime !== storageStat.ctime
+                ) {
+                    throw new Error("Storage changed during the incomplete document check");
+                }
             }
         }
         if (errorFiles.length == 0) {

src/serviceFeatures/startupLifecycle/incompleteDocuments.unit.spec.ts

@@ -99,6 +99,7 @@ function createDependencies(findAllNormalDocs: FindAllNormalDocs = noDocuments,
         isTargetFile: vi.fn(async () => true),
         storageAccess: {
             readHiddenFileBinary,
+            statHidden: vi.fn(async (path: string) => getFixture(path)?.stub.stat ?? null),
             getFileStub,
         },
         fileHandler: { storeFileToDB },
@@ -157,6 +158,74 @@ describe("checkIncompleteDocuments", () => {
         expect(noticeGroups.finish).toHaveBeenCalledWith("startup-integrity-check");
     });
 
+    it("does not read healthy storage contents only to determine their size", async () => {
+        const healthy = documentFixture("healthy.md", { storageContent: "abc" });
+        const fixture = createDependencies(documentsFrom([healthy]), [healthy]);
+
+        await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true);
+
+        expect(fixture.readHiddenFileBinary).not.toHaveBeenCalled();
+        expect(fixture.askSelectStringDialogue).not.toHaveBeenCalled();
+        expect(fixture.dependencies.keyValueDB.set).toHaveBeenCalledWith("checkIncompleteDocs", true);
+    });
+
+    it("does not mark a scan complete when storage changes during the size check", async () => {
+        const healthy = documentFixture("changing.md", { storageContent: "abc" });
+        const fixture = createDependencies(documentsFrom([healthy]), [healthy]);
+        vi.mocked(fixture.dependencies.storageAccess.statHidden)
+            .mockResolvedValueOnce(healthy.stub.stat)
+            .mockResolvedValueOnce({ ...healthy.stub.stat, mtime: 3 });
+
+        await expect(checkIncompleteDocuments(fixture.dependencies)).rejects.toThrow("Storage changed");
+
+        expect(fixture.readHiddenFileBinary).not.toHaveBeenCalled();
+        expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled();
+    });
+
+    it("does not offer repair when all sizes agree after reading storage", async () => {
+        const healthy = documentFixture("settled.md", { storageContent: "abc" });
+        const fixture = createDependencies(documentsFrom([healthy]), [healthy]);
+        vi.mocked(fixture.dependencies.storageAccess.statHidden).mockResolvedValueOnce({ ...healthy.stub.stat, size: 4 });
+
+        await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true);
+
+        expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("settled.md");
+        expect(fixture.askSelectStringDialogue).not.toHaveBeenCalled();
+        expect(fixture.storeFileToDB).not.toHaveBeenCalled();
+        expect(fixture.dependencies.keyValueDB.set).toHaveBeenCalledWith("checkIncompleteDocs", true);
+    });
+
+    it("uses the bytes read from storage when a size mismatch remains", async () => {
+        const mismatched = documentFixture("changing.md", { recordedSize: 3, storageContent: "abcde" });
+        const fixture = createDependencies(documentsFrom([mismatched]), [mismatched]);
+        vi.mocked(fixture.dependencies.storageAccess.statHidden).mockResolvedValueOnce({
+            ...mismatched.stub.stat,
+            size: 4,
+        });
+        fixture.askSelectStringDialogue.mockImplementation(selectButton(0));
+
+        await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true);
+
+        expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("changing.md");
+        expect(fixture.askSelectStringDialogue).toHaveBeenCalledOnce();
+        expect(fixture.askSelectStringDialogue.mock.calls[0]?.[0]).toEqual(
+            expect.stringContaining("changing.md (M: 3, A: 3, S: 5)")
+        );
+        expect(fixture.storeFileToDB).not.toHaveBeenCalled();
+        expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled();
+    });
+
+    it("still reads conflicted storage for the existing repair dialogue", async () => {
+        const conflicted = documentFixture("conflicted.md", { storageContent: "abc", conflicts: ["2-other"] });
+        const fixture = createDependencies(documentsFrom([conflicted]), [conflicted]);
+        fixture.askSelectStringDialogue.mockImplementation(selectButton(0));
+
+        await expect(checkIncompleteDocuments(fixture.dependencies)).resolves.toBe(true);
+
+        expect(fixture.readHiddenFileBinary).toHaveBeenCalledWith("conflicted.md");
+        expect(fixture.dependencies.keyValueDB.set).not.toHaveBeenCalled();
+    });
+
     it("repairs a recoverable document when FIX is selected", async () => {
         const recoverable = documentFixture("recoverable.md");
         const fixture = createDependencies(documentsFrom([recoverable]), [recoverable]);

updates.md

@@ -12,6 +12,10 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
 
 ## Unreleased
 
+### Synchronisation
+
+- We can run startup integrity checks with lower memory use for healthy Vault files. Files that change between the size check and content reading no longer trigger an unnecessary repair warning when their actual sizes agree and there are no conflicts. (#1283)
+
 ## 1.0.40
 
 24th October, 2026