fix: await optional file database saves

This commit is contained in:
vorotamoroz committed 2026-10-25 13:26:02 +00:00
1 parent b3a43abc2e
commit 62b2edba9d
11 files changed
+1113 -20

No files matched your search

@@ -0,0 +1,260 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
vi.mock("@/deps.ts", () => ({}));
vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} }));
vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" }));
vi.mock("./configureHiddenFileSyncMode.ts", () => ({ configureHiddenFileSyncMode: vi.fn() }));
import type { FilePath, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { readContent } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { HiddenFileSync } from "./CmdHiddenFileSync";
import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase";
const path = ".obsidian/plugins/example/main.js" as FilePath;
const content = "module.exports = class ExamplePlugin {};";
const fixtureOwners: Awaited<ReturnType<typeof createOptionalFileSyncDatabase>>[] = [];
async function getText(entry: Parameters<typeof readContent>[0]) {
const value = await readContent(entry);
return typeof value === "string" ? value : new TextDecoder().decode(value);
}
async function createHiddenFileSync() {
const fixture = await createOptionalFileSyncDatabase();
fixtureOwners.push(fixture);
fixture.settings.usePluginSync = false;
fixture.settings.syncInternalFilesTargetPatterns = "" as typeof fixture.settings.syncInternalFilesTargetPatterns;
fixture.settings.syncInternalFilesIgnorePatterns = "" as typeof fixture.settings.syncInternalFilesIgnorePatterns;
let fileContent = content;
let stat: UXStat | null = {
ctime: 1_000,
mtime: 2_000,
size: fileContent.length,
type: "file",
};
const values = new Map<string, unknown>();
const kvDB = {
get: vi.fn(async <T>(key: string) => values.get(key) as T | undefined),
set: vi.fn(async <T>(key: string, value: T) => {
values.set(key, value);
}),
};
const storageAccess = {
statHidden: vi.fn(async () => stat),
readHiddenFileAuto: vi.fn(async () => fileContent),
writeHiddenFileAuto: vi.fn(async () => undefined),
removeHidden: vi.fn(async () => true),
isExistsIncludeHidden: vi.fn(async () => true),
ensureDir: vi.fn(async () => undefined),
triggerHiddenFile: vi.fn(async () => undefined),
setFile(nextContent: string | null, mtime = 2_000) {
fileContent = nextContent ?? "";
stat = nextContent === null ? null : { ctime: 1_000, mtime, size: fileContent.length, type: "file" };
},
};
const databaseService = { isDatabaseReady: vi.fn(() => true) };
const appLifecycle = { isReady: vi.fn(() => false), isSuspended: vi.fn(() => true) };
const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync;
Object.assign(hiddenFileSync, {
core: {
settings: fixture.settings,
localDatabase: fixture.localDatabase,
kvDB,
storageAccess,
services: {
...fixture.services,
database: databaseService,
appLifecycle,
path: {
...fixture.services.path,
markChangesAreSame: vi.fn(),
unmarkChanges: vi.fn(),
},
vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) },
},
},
cacheFileRegExps: new Map(),
cacheCustomisationSyncIgnoredFiles: new Map(),
_fileInfoLastProcessed: new Map(),
_databaseInfoLastProcessed: new Map(),
_fileInfoLastKnown: new Map(),
semaphore: Semaphore(10),
_log: vi.fn(),
});
return { fixture, hiddenFileSync, storageAccess, kvDB, databaseService, appLifecycle };
}
afterEach(async () => {
vi.restoreAllMocks();
for (const fixture of fixtureOwners.splice(0)) await fixture.dispose();
});
describe("Hidden File Sync optional save outcomes", () => {
it("retries a failed database write when the storage fingerprint is unchanged", async () => {
const { fixture, hiddenFileSync } = await createHiddenFileSync();
await expect(hiddenFileSync.isTargetFile(path)).resolves.toBe(true);
vi.spyOn(fixture.localDatabase, "putDBEntry").mockRejectedValueOnce(new Error("temporary database failure"));
await expect(hiddenFileSync.trackStorageFileModification(path)).resolves.toBe(false);
await expect(hiddenFileSync.trackStorageFileModification(path)).resolves.toBe(true);
expect(fixture.localDatabase.putDBEntry).toHaveBeenCalledTimes(2);
const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath);
expect(saved).not.toBe(false);
});
it("rejects a failed strict write and retries the same fingerprint without delaying readiness", async () => {
const { fixture, hiddenFileSync, appLifecycle } = await createHiddenFileSync();
const putDBEntry = vi
.spyOn(fixture.localDatabase, "putDBEntry")
.mockRejectedValueOnce(new Error("temporary database failure"));
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow(
"temporary database failure"
);
expect(hiddenFileSync.getLastProcessedFileKey(path)).toBeUndefined();
await expect(
hiddenFileSync._processOptionalFileEventWithResult(path, {
whenFlushRequested: new Promise<void>(() => undefined),
})
).resolves.toMatchObject({
kind: "completed",
});
expect(putDBEntry).toHaveBeenCalledTimes(2);
const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath);
expect(saved).not.toBe(false);
if (saved) expect(await getText(saved)).toBe(content);
expect(appLifecycle.isReady()).toBe(false);
expect(appLifecycle.isSuspended()).toBe(true);
});
it("rejects a false database-write result and can store a later edit after a same-content event", async () => {
const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync();
const putDBEntry = vi
.spyOn(fixture.localDatabase, "putDBEntry")
.mockImplementationOnce(async () => false as never);
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow(
`Failed to store hidden file in the database: ${path}`
);
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "completed",
});
const first = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath);
expect(first).not.toBe(false);
if (first) {
const firstRevision = first._rev;
hiddenFileSync._fileInfoLastProcessed.delete(path);
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "completed",
});
const unchanged = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath);
expect(unchanged).not.toBe(false);
if (unchanged) expect(unchanged._rev).toBe(firstRevision);
}
storageAccess.setFile("module.exports = class UpdatedExamplePlugin {};", 3_000);
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "completed",
});
const updated = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath);
expect(updated).not.toBe(false);
if (updated) expect(await getText(updated)).toBe("module.exports = class UpdatedExamplePlugin {};");
expect(putDBEntry).toHaveBeenCalledTimes(3);
});
it("vetoes when the physical database is unavailable and ignores disabled or filtered events", async () => {
const { fixture, hiddenFileSync, databaseService } = await createHiddenFileSync();
databaseService.isDatabaseReady.mockReturnValue(false);
await expect(
hiddenFileSync._processOptionalFileEventWithResult(path, {
whenFlushRequested: new Promise<void>(() => undefined),
})
).resolves.toMatchObject({ kind: "veto", value: expect.any(String) });
databaseService.isDatabaseReady.mockReturnValue(true);
fixture.settings.syncInternalFiles = false;
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "not-applicable",
});
fixture.settings.syncInternalFiles = true;
fixture.settings.syncInternalFilesTargetPatterns =
"^\\.obsidian/snippets/" as typeof fixture.settings.syncInternalFilesTargetPatterns;
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "not-applicable",
});
});
it("treats an absent replayed deletion as complete and leaves cache initialisation intact", async () => {
const { hiddenFileSync, storageAccess, kvDB } = await createHiddenFileSync();
storageAccess.setFile(null);
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "completed",
});
expect(hiddenFileSync.getLastProcessedFileKey(path)).toBe("0-0");
const lifecycle = hiddenFileSync as unknown as {
_everyOnDatabaseInitialized(showNotice: boolean): Promise<boolean>;
};
hiddenFileSync.core.settings.syncInternalFiles = false;
await lifecycle._everyOnDatabaseInitialized(false);
expect(kvDB.get).toHaveBeenCalledTimes(3);
expect(hiddenFileSync.getLastProcessedFileKey(path)).toBe("0-0");
});
it("rejects actual deletion failures and completes duplicate deletions without another write", async () => {
const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync();
await fixture.localDatabase.putDBEntry(optionalFileEntry(`i:${path}`, content));
storageAccess.setFile(null);
const putRaw = vi
.spyOn(fixture.localDatabase, "putRaw")
.mockRejectedValueOnce(new Error("temporary deletion failure"));
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).rejects.toThrow(
"temporary deletion failure"
);
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "completed",
});
hiddenFileSync.resetLastProcessedFile([path]);
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "completed",
});
expect(putRaw).toHaveBeenCalledTimes(2);
});
it("recreates an empty hidden file when its database entry is logically deleted", async () => {
const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync();
await fixture.localDatabase.putDBEntry(optionalFileEntry(`i:${path}`, ""));
await expect(hiddenFileSync.deleteInternalFileOnDatabase(path)).resolves.toBe(true);
const deletedBase = await fixture.localDatabase.getDBEntry(
`i:${path}` as FilePath,
undefined,
false,
true,
true
);
expect(deletedBase).not.toBe(false);
if (deletedBase) expect(deletedBase.deleted).toBe(true);
if (!deletedBase) throw new Error("The logically deleted test entry could not be loaded");
vi.spyOn(fixture.localDatabase, "getDBEntry").mockResolvedValueOnce(deletedBase);
storageAccess.setFile("", 4_000);
const putDBEntry = vi.spyOn(fixture.localDatabase, "putDBEntry");
await expect(hiddenFileSync._processOptionalFileEventWithResult(path)).resolves.toMatchObject({
kind: "completed",
});
expect(putDBEntry).toHaveBeenCalledTimes(1);
const recreated = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath);
expect(recreated).not.toBe(false);
if (recreated) {
expect(recreated.deleted).not.toBe(true);
expect(await getText(recreated)).toBe("");
}
});
});
@@ -54,6 +54,8 @@ import { EVENT_SETTING_SAVED, eventHub } from "@/common/events.ts";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
import type { LiveSyncCore } from "@/main.ts";
import { tryGetFilePath } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc";
import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils";
import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import { configureHiddenFileSyncMode, type ConfigureHiddenFileSyncResult } from "./configureHiddenFileSyncMode.ts";
import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts";
import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts";
@@ -161,9 +163,7 @@ export class HiddenFileSync extends LiveSyncCommands {
// return true;
// }
private async _everyOnDatabaseInitialized(showNotice: boolean) {
this._fileInfoLastProcessed = await autosaveCache(this.kvDB, "hidden-file-lastProcessed");
this._databaseInfoLastProcessed = await autosaveCache(this.kvDB, "hidden-file-lastProcessed-database");
this._fileInfoLastKnown = await autosaveCache(this.kvDB, "hidden-file-lastKnown");
await this.ensureProcessedCaches();
if (this.isThisModuleEnabled()) {
if (this._fileInfoLastProcessed.size == 0 && this._fileInfoLastProcessed.size == 0) {
this._log(`No cache found. Performing startup scan.`, LOG_LEVEL_VERBOSE);
@@ -249,6 +249,26 @@ export class HiddenFileSync extends LiveSyncCommands {
return false;
}
async _processOptionalFileEventWithResult(
path: FilePath,
_options?: OptionalFileEventOptions
): Promise<HandlerOutcome<void, string>> {
if (!this.isThisModuleEnabled()) return { kind: "not-applicable" };
if (!(await this.isTargetFile(path))) return { kind: "not-applicable" };
if (!this._isDatabaseReady()) {
return { kind: "veto", value: "The local database is not ready" };
}
await this.ensureProcessedCaches();
if (!this.isThisModuleEnabled() || !(await this.isTargetFile(path))) {
return { kind: "not-applicable" };
}
if (!(await this.trackStorageFileModificationStrict(path))) {
return { kind: "not-applicable" };
}
return { kind: "completed" };
}
_anyGetOptionalConflictCheckMethod(path: FilePathWithPrefix): Promise<boolean | "newer"> {
if (isInternalMetadata(path)) {
this.queueConflictCheck(path);
@@ -317,6 +337,51 @@ export class HiddenFileSync extends LiveSyncCommands {
_fileInfoLastKnown!: MapLike<string, number>;
_databaseInfoLastProcessed!: MapLike<string, string>;
private processedCacheStore: unknown;
private processedCacheStoreReady = false;
private processedCacheLoadStore: unknown;
private processedCacheLoad?: Promise<void>;
private async ensureProcessedCaches() {
const store = this.kvDB;
if (!store) throw new Error("The Hidden File Sync cache store is unavailable");
if (this.processedCacheStoreReady && this.processedCacheStore === store) return;
if (this.processedCacheLoad && this.processedCacheLoadStore === store) {
await this.processedCacheLoad;
if (!this.processedCacheStoreReady || this.processedCacheStore !== this.kvDB) {
throw new Error("The Hidden File Sync cache store changed while loading its processing state");
}
return;
}
let load!: Promise<void>;
load = Promise.all([
autosaveCache<string, string>(store, "hidden-file-lastProcessed"),
autosaveCache<string, string>(store, "hidden-file-lastProcessed-database"),
autosaveCache<string, number>(store, "hidden-file-lastKnown"),
])
.then(([fileInfoLastProcessed, databaseInfoLastProcessed, fileInfoLastKnown]) => {
if (this.processedCacheLoad !== load || this.kvDB !== store) return;
this._fileInfoLastProcessed = fileInfoLastProcessed;
this._databaseInfoLastProcessed = databaseInfoLastProcessed;
this._fileInfoLastKnown = fileInfoLastKnown;
this.processedCacheStore = store;
this.processedCacheStoreReady = true;
})
.finally(() => {
if (this.processedCacheLoad === load) {
this.processedCacheLoad = undefined;
this.processedCacheLoadStore = undefined;
}
});
this.processedCacheLoadStore = store;
this.processedCacheLoad = load;
await load;
if (!this.processedCacheStoreReady || this.processedCacheStore !== this.kvDB) {
throw new Error("The Hidden File Sync cache store changed while loading its processing state");
}
}
statToKey(stat: UXStat | null) {
return `${stat?.mtime ?? 0}-${stat?.size ?? 0}`;
}
@@ -600,7 +665,6 @@ Offline Changed files: ${processFiles.length}`;
this._log(`Hidden file:${path} is changed.`, LOG_LEVEL_VERBOSE);
stat = cache.stat;
}
this.updateLastProcessedFile(path, stat!);
const lastIsNotFound = !lastKey || lastKey.endsWith("-0-0");
const nowIsNotFound = cache.deleted;
const type = lastIsNotFound && nowIsNotFound ? "invalid" : nowIsNotFound ? "delete" : "modified";
@@ -651,6 +715,27 @@ Offline Changed files: ${processFiles.length}`;
return true;
}
private async trackStorageFileModificationStrict(path: FilePath): Promise<boolean> {
return await this.serializedForEvent(path, async () => {
if (!this.isThisModuleEnabled() || !(await this.isTargetFile(path))) return false;
const stat = await this.core.storageAccess.statHidden(path);
if (stat != null && stat.type != "file") return true;
const key = await this.fileToStatKey(path, stat);
if (this.getLastProcessedFileKey(path) === key) {
this._log(`${path} Already processed.`, LOG_LEVEL_DEBUG);
return true;
}
const fileInfo = await this.loadFileWithInfo(path);
if (fileInfo.deleted) {
await this.deleteInternalFileOnDatabaseStrict(path);
return true;
}
await this.storeInternalFileToDatabaseStrict(fileInfo);
return true;
});
}
// --> Conflict processing
// Keep one in-flight conflict check per path so repeated sync events do not close the active merge dialogue.
@@ -1500,6 +1585,18 @@ Offline Changed files: ${files.length}`;
// --> Storage To Database Functions
async __loadBaseSaveData(file: FilePath, includeContent = true): Promise<LoadedEntry | false> {
return await this.loadBaseSaveData(file, includeContent, false);
}
private async loadBaseSaveDataStrict(file: FilePath, includeContent = true): Promise<LoadedEntry | false> {
return await this.loadBaseSaveData(file, includeContent, true);
}
private async loadBaseSaveData(
file: FilePath,
includeContent: boolean,
strict: boolean
): Promise<LoadedEntry | false> {
const prefixedFileName = addPrefix(file, ICHeader);
const id = await this.path2id(prefixedFileName, ICHeader);
try {
@@ -1525,6 +1622,7 @@ Offline Changed files: ${files.length}`;
return old;
}
} catch (ex) {
if (strict) throw ex;
this._log(`Getting base save data failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
@@ -1552,6 +1650,18 @@ Offline Changed files: ${files.length}`;
}
async storeInternalFileToDatabase(file: InternalFileInfo | UXFileInfo, forceWrite = false) {
return await this.storeInternalFileToDatabaseCore(file, forceWrite, false);
}
private async storeInternalFileToDatabaseStrict(file: InternalFileInfo | UXFileInfo, forceWrite = false) {
return await this.storeInternalFileToDatabaseCore(file, forceWrite, true);
}
private async storeInternalFileToDatabaseCore(
file: InternalFileInfo | UXFileInfo,
forceWrite: boolean,
strict: boolean
) {
const storeFilePath = stripAllPrefixes(file.path);
const storageFilePath = file.path;
if (await this.services.vault.isIgnoredByIgnoreFile(storageFilePath)) {
@@ -1565,9 +1675,11 @@ Offline Changed files: ${files.length}`;
if (fileInfo.deleted) {
throw new Error(`Hidden file:${storeFilePath} is deleted. This should not be occurred.`);
}
const baseData = await this.__loadBaseSaveData(storeFilePath, true);
const baseData = strict
? await this.loadBaseSaveDataStrict(storeFilePath, true)
: await this.__loadBaseSaveData(storeFilePath, true);
if (baseData === false) throw new Error("Failed to load base data");
if (baseData._rev && !forceWrite) {
if (baseData._rev && !baseData.deleted && !baseData._deleted && !forceWrite) {
// Not newly created, we should check the content is actually modified.
const isSame = await isDocContentSame(readAsBlob(baseData), fileInfo.body);
if (isSame) {
@@ -1592,11 +1704,15 @@ Offline Changed files: ${files.length}`;
this.updateLastProcessed(storeFilePath, saveData, fileInfo.stat);
}
const success = ret && ret.ok;
if (!success && strict) {
throw new Error(`Failed to store hidden file in the database: ${storeFilePath}`);
}
this._log(`STORAGE --> DB:${storageFilePath}: (hidden) ${success ? "Done" : "Failed"}`);
return success;
} catch (ex) {
this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
if (strict) throw ex;
return false;
}
});
@@ -1676,6 +1792,14 @@ Offline Changed files: ${files.length}`;
}
async deleteInternalFileOnDatabase(filenameSrc: FilePath, forceWrite = false) {
return await this.deleteInternalFileOnDatabaseCore(filenameSrc, forceWrite, false);
}
private async deleteInternalFileOnDatabaseStrict(filenameSrc: FilePath, forceWrite = false) {
return await this.deleteInternalFileOnDatabaseCore(filenameSrc, forceWrite, true);
}
private async deleteInternalFileOnDatabaseCore(filenameSrc: FilePath, _forceWrite: boolean, strict: boolean) {
const storeFilePath = filenameSrc;
const storageFilePath = filenameSrc;
const displayFileName = filenameSrc;
@@ -1686,18 +1810,26 @@ Offline Changed files: ${files.length}`;
}
return await serialized("file-" + prefixedFileName, async () => {
try {
const baseData = await this.__loadBaseSaveData(storeFilePath, false);
const baseData = strict
? await this.loadBaseSaveDataStrict(storeFilePath, false)
: await this.__loadBaseSaveData(storeFilePath, false);
if (baseData === false) throw new Error("Failed to load base data during deleting");
if (baseData._conflicts !== undefined) {
for (const conflictRev of baseData._conflicts) {
await this.localDatabase.removeRevision(baseData._id, conflictRev);
const removed = await this.localDatabase.removeRevision(baseData._id, conflictRev);
if (strict && removed === false) {
throw new Error(`Failed to remove conflicted hidden-file revision ${conflictRev}`);
}
this._log(
`STORAGE -x> DB: ${displayFileName}: (hidden) conflict removed ${baseData._rev} => ${conflictRev}`,
LOG_LEVEL_VERBOSE
);
}
}
if (baseData.deleted) {
const alreadyDeleted = strict
? !baseData._rev || baseData.deleted || baseData._deleted
: baseData.deleted;
if (alreadyDeleted) {
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) already deleted`, LOG_LEVEL_VERBOSE);
this.updateLastProcessedDeletion(storeFilePath, baseData);
return true;
@@ -1718,11 +1850,13 @@ Offline Changed files: ${files.length}`;
return true;
} else {
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`);
if (strict) throw new Error(`Failed to delete hidden file from the database: ${storeFilePath}`);
return false;
}
} catch (ex) {
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
if (strict) throw ex;
return false;
}
});
@@ -2084,6 +2218,9 @@ Offline Changed files: ${files.length}`;
// services.databaseEvents.handleOnDatabaseInitialisation(this._everyOnInitializeDatabase.bind(this));
services.appLifecycle.onSettingLoaded.addHandler(this._everyOnloadAfterLoadSettings.bind(this));
services.fileProcessing.processOptionalFileEvent.addHandler(this._anyProcessOptionalFileEvent.bind(this));
services.fileProcessing.processOptionalFileEventWithResult?.addHandler(
this._processOptionalFileEventWithResult.bind(this)
);
services.conflict.getOptionalConflictCheckMethod.addHandler(this._anyGetOptionalConflictCheckMethod.bind(this));
services.replication.processOptionalSynchroniseResult.addHandler(this._anyProcessOptionalSyncFiles.bind(this));
services.setting.onRealiseSetting.addHandler(this._everyRealizeSettingSyncMode.bind(this));