Use document views in file processing and history

Replace broad entry inputs with the information used by file processing,
history, and JSON comparison. Separate customisation content and save-base
results from database-document models, and keep content fields out of
logical-deletion Metadata while preserving the parent revision.

Record the accepted design, compatibility boundaries, and verification.
Requires Commonlib commit 751b29a74340da57122b8ab11b80d00c40272120.
The registry dependency remains at 0.2.3 pending publication and adoption.
This commit is contained in:
vorotamoroz committed 2026-11-03 08:22:21 +00:00
1 parent 0fe7d82f46
commit 80ce481575
17 files changed
+1036 -159

No files matched your search

+4 -3
View File
@@ -1,12 +1,13 @@
import type { TFile } from "@/deps";
import type { FilePathWithPrefix, LoadedEntry } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { DocumentIdentity } from "@vrtmrz/livesync-commonlib/documents";
export const EVENT_REQUEST_SHOW_HISTORY = "show-history";
declare global {
interface LSEvents {
[EVENT_REQUEST_SHOW_HISTORY]:
| { file: TFile; fileOnDB: LoadedEntry }
| { file: FilePathWithPrefix; fileOnDB: LoadedEntry };
| { file: TFile; fileOnDB: DocumentIdentity }
| { file: FilePathWithPrefix; fileOnDB: DocumentIdentity };
}
}
+23 -13
View File
@@ -12,7 +12,7 @@ import {
import type {
EntryDoc,
LoadedEntry,
EntryTypeNotes,
InternalFileEntry,
FilePathWithPrefix,
FilePath,
@@ -20,6 +20,7 @@ import type {
SavingEntry,
diff_result,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { DocumentPathRef, FileContentView, FileProperties } from "@vrtmrz/livesync-commonlib/documents";
import {
CANCELLED,
LEAVE_TO_SUBSEQUENT,
@@ -272,7 +273,7 @@ export interface IPluginDataExDisplay {
name: string;
term: string;
displayName?: string;
files: (LoadedEntryPluginDataExFile | PluginDataExFile)[];
files: (CustomisationFileContent | PluginDataExFile)[];
version?: string;
mtime: number;
}
@@ -286,7 +287,16 @@ export type PluginDataExDisplay = {
version?: string;
mtime: number;
};
type LoadedEntryPluginDataExFile = LoadedEntry & PluginDataExFile;
interface CustomisationDocumentContent extends FileContentView, DocumentPathRef, FileProperties {
readonly _rev?: string;
readonly datatype: EntryTypeNotes;
}
interface CustomisationFileContent extends PluginDataExFile, DocumentPathRef {
readonly _rev?: string;
readonly ctime: number;
readonly datatype: EntryTypeNotes;
}
type ConfigFileApplicationTarget = {
readonly path: FilePath;
readonly stat: { readonly mtime: number; readonly ctime: number };
@@ -329,7 +339,7 @@ export class PluginDataExDisplayV2 {
term: string;
files = [] as LoadedEntryPluginDataExFile[];
files = [] as CustomisationFileContent[];
name: string;
confKey: string;
@@ -338,11 +348,11 @@ export class PluginDataExDisplayV2 {
this.category = `${data.category}`;
this.name = `${data.name}`;
this.term = `${data.term}`;
this.files = [...(data.files as LoadedEntryPluginDataExFile[])];
this.files = [...(data.files as CustomisationFileContent[])];
this.confKey = `${categoryToFolder(this.category, this.term)}${this.name}`;
this.applyLoadedManifest();
}
async setFile(file: LoadedEntryPluginDataExFile) {
async setFile(file: CustomisationFileContent) {
const old = this.files.find((e) => e.filename == file.filename);
if (old) {
if (old.mtime == file.mtime && (await isDocContentSame(old.data, file.data))) return;
@@ -668,8 +678,8 @@ export class ConfigSync extends LiveSyncCommands {
async createPluginDataExFileV2(
unifiedPathV2: FilePathWithPrefix,
loaded?: LoadedEntry
): Promise<false | LoadedEntryPluginDataExFile> {
loaded?: CustomisationDocumentContent
): Promise<false | CustomisationFileContent> {
const { category, key, filename, device } = this.parseUnifiedPath(unifiedPathV2);
if (!loaded) {
const d = await this.localDatabase.getDBEntry(unifiedPathV2);
@@ -691,7 +701,7 @@ export class ConfigSync extends LiveSyncCommands {
const dataSrc = getDocData(loaded.data);
const dataStart = dataSrc.indexOf(DUMMY_END);
const data = dataSrc.substring(dataStart + DUMMY_END.length);
const file: LoadedEntryPluginDataExFile = {
const file: CustomisationFileContent = {
...loaded,
hash: "",
data: [base64ToString(data)],
@@ -900,14 +910,14 @@ export class ConfigSync extends LiveSyncCommands {
async compareUsingDisplayData(dataA: IPluginDataExDisplay, dataB: IPluginDataExDisplay, compareEach = false) {
const loadFile = async (data: IPluginDataExDisplay) => {
if (data instanceof PluginDataExDisplayV2 || compareEach) {
return data.files[0] as LoadedEntryPluginDataExFile;
return data.files[0] as CustomisationFileContent;
}
const loadDoc = await this.localDatabase.getDBEntry(data.documentPath);
if (!loadDoc) return false;
const pluginData = deserialize(getDocDataAsArray(loadDoc.data), {}) as PluginDataEx;
pluginData.documentPath = data.documentPath;
const file = pluginData.files[0];
const doc = { ...loadDoc, ...file, datatype: "newnote" } as LoadedEntryPluginDataExFile;
const doc = { ...loadDoc, ...file, datatype: "newnote" } as CustomisationFileContent;
return doc;
};
const fileA = await loadFile(dataA);
@@ -1004,7 +1014,7 @@ export class ConfigSync extends LiveSyncCommands {
const files = data.files;
const applyBinaryContent = createModificationHandler<
ConfigFileApplicationTarget,
LoadedEntryPluginDataExFile,
CustomisationFileContent,
ArrayBuffer,
ArrayBuffer
>({
@@ -1021,7 +1031,7 @@ export class ConfigSync extends LiveSyncCommands {
});
const applyTextContent = createModificationHandler<
ConfigFileApplicationTarget,
LoadedEntryPluginDataExFile,
CustomisationFileContent,
string,
string
>({
@@ -1,14 +1,15 @@
import { App, Modal } from "@/deps.ts";
import { type FilePath, type LoadedEntry } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { type FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
import JsonResolvePane from "./JsonResolvePane.svelte";
import { waitForSignal } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { mount, unmount } from "svelte";
import type { JsonComparisonDocument } from "./jsonComparison.ts";
export class JsonResolveModal extends Modal {
// result: Array<[number, string]>;
filename: FilePath;
callback?: (keepRev?: string, mergedStr?: string) => Promise<void>;
docs: LoadedEntry[];
docs: JsonComparisonDocument[];
component?: ReturnType<typeof mount>;
nameA: string;
nameB: string;
@@ -20,7 +21,7 @@ export class JsonResolveModal extends Modal {
constructor(
app: App,
filename: FilePath,
docs: LoadedEntry[],
docs: JsonComparisonDocument[],
callback: (keepRev?: string, mergedStr?: string) => Promise<void>,
nameA?: string,
nameB?: string,
@@ -1,12 +1,13 @@
<script lang="ts">
import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "@/deps.ts";
import type { FilePath, LoadedEntry } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { decodeBinary, readString } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/convert";
import { getDocData, isObjectDifferent, mergeObject } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { $msg as translateMessage } from "@/common/translation";
import type { JsonComparisonDocument } from "./jsonComparison.ts";
interface Props {
docs?: LoadedEntry[];
docs?: JsonComparisonDocument[];
callback?: (keepRev?: string, mergedStr?: string) => Promise<void>;
filename?: FilePath;
nameA?: string;
@@ -63,7 +64,7 @@
type SelectModes = "" | "A" | "B" | "AB" | "BA";
let mode: SelectModes = $state(defaultSelect as SelectModes);
function docToString(doc: LoadedEntry) {
function docToString(doc: JsonComparisonDocument) {
return doc.datatype == "plain" ? getDocData(doc.data) : readString(new Uint8Array(decodeBinary(doc.data)));
}
function revStringToRevNumber(rev?: string) {
@@ -0,0 +1,9 @@
import type { DocumentID, EntryTypeNotes } from "@vrtmrz/livesync-commonlib/compat/common/types";
export interface JsonComparisonDocument {
readonly _id: DocumentID;
readonly _rev?: string;
readonly mtime: number;
readonly datatype: EntryTypeNotes;
readonly data: string | string[];
}
@@ -1,6 +1,16 @@
import { type ListedFiles } from "@/deps.ts";
import type {
DeletionState,
CompatibleFileToSave,
CurrentFileDocument,
FileChangeRef,
FileContentView,
FileMetadata,
FileProperties,
} from "@vrtmrz/livesync-commonlib/documents";
import type { JsonComparisonDocument } from "@/features/HiddenFileCommon/jsonComparison";
import {
type LoadedEntry,
type AnyEntry,
type FilePathWithPrefix,
type FilePath,
LOG_LEVEL_INFO,
@@ -8,7 +18,7 @@ import {
LOG_LEVEL_VERBOSE,
MODE_SELECTIVE,
MODE_PAUSED,
type SavingEntry,
type EntryTypeNotes,
type DocumentID,
type UXFileInfo,
type UXStat,
@@ -63,6 +73,30 @@ import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlu
import { $msg } from "@/common/translation";
type SyncDirection = "push" | "pull" | "safe" | "pullForce" | "pushForce";
interface FileProcessingStamp extends DeletionState {
readonly mtime: number;
readonly size: number;
readonly _rev?: string;
}
interface ModificationTime extends DeletionState {
readonly mtime?: number;
}
interface FileModificationTime extends DeletionState {
readonly stat?: ModificationTime;
}
interface FileContent extends FileContentView, Pick<FileProperties, "ctime" | "mtime"> {}
interface ComparisonFile extends JsonComparisonDocument {
readonly path: FilePathWithPrefix;
}
interface SaveBase extends FileContentView, FileProperties, FileChangeRef {
readonly datatype: EntryTypeNotes;
}
type HiddenFileInitialisationProgress = {
log(message: string): void;
once(message: string): void;
@@ -76,7 +110,7 @@ export const HIDDEN_FILE_SYNC_DISABLED_NOTICE =
"Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files.";
function getComparingMTime(
doc: (MetaEntry | LoadedEntry | false) | UXFileInfo | UXStat | null | undefined,
doc: ModificationTime | FileModificationTime | false | null | undefined,
includeDeleted = false
) {
if (doc === null) return 0;
@@ -87,7 +121,7 @@ function getComparingMTime(
if ("_deleted" in doc && doc._deleted) return 0;
}
if ("stat" in doc) return doc.stat?.mtime ?? 0;
return doc.mtime ?? 0;
return "mtime" in doc ? (doc.mtime ?? 0) : 0;
}
export class HiddenFileSync extends LiveSyncCommands {
@@ -283,7 +317,7 @@ export class HiddenFileSync extends LiveSyncCommands {
return Promise.resolve(false);
}
async _anyProcessOptionalSyncFiles(doc: LoadedEntry): Promise<boolean> {
async _anyProcessOptionalSyncFiles(doc: FileChangeRef): Promise<boolean> {
if (isInternalMetadata(doc._id)) {
if (this.isThisModuleEnabled()) {
//system file
@@ -388,10 +422,10 @@ export class HiddenFileSync extends LiveSyncCommands {
}
}
statToKey(stat: UXStat | null) {
statToKey(stat: Pick<UXStat, "mtime" | "size"> | null) {
return `${stat?.mtime ?? 0}-${stat?.size ?? 0}`;
}
docToKey(doc: LoadedEntry | MetaEntry) {
docToKey(doc: FileProcessingStamp) {
return `${doc.mtime}-${doc.size}-${doc._rev}-${doc._deleted || doc.deleted || false ? "-0" : "-1"}`;
}
async fileToStatKey(file: FilePath, stat: UXStat | null = null) {
@@ -399,7 +433,7 @@ export class HiddenFileSync extends LiveSyncCommands {
return this.statToKey(stat);
}
updateLastProcessedFile(file: FilePath, keySrc: string | UXStat) {
updateLastProcessedFile(file: FilePath, keySrc: string | Pick<UXStat, "mtime" | "size">) {
const key = typeof keySrc == "string" ? keySrc : this.statToKey(keySrc);
const splitted = key.split("-");
if (splitted[0] != "0") {
@@ -441,11 +475,11 @@ export class HiddenFileSync extends LiveSyncCommands {
getLastProcessedDatabaseKey(file: FilePath) {
return this._databaseInfoLastProcessed.get(file);
}
updateLastProcessedDatabase(file: FilePath, keySrc: string | MetaEntry | LoadedEntry) {
updateLastProcessedDatabase(file: FilePath, keySrc: string | FileProcessingStamp) {
const key = typeof keySrc == "string" ? keySrc : this.docToKey(keySrc);
this._databaseInfoLastProcessed.set(file, key);
}
updateLastProcessed(path: FilePath, db: MetaEntry | LoadedEntry, stat: UXStat) {
updateLastProcessed(path: FilePath, db: FileProcessingStamp, stat: Pick<UXStat, "mtime" | "size">) {
this.updateLastProcessedDatabase(path, db);
this.updateLastProcessedFile(path, this.statToKey(stat));
const dbMTime = getComparingMTime(db);
@@ -456,7 +490,7 @@ export class HiddenFileSync extends LiveSyncCommands {
this.services.path.markChangesAreSame(path, getComparingMTime(db), getComparingMTime(stat));
}
}
updateLastProcessedDeletion(path: FilePath, db: MetaEntry | LoadedEntry | false) {
updateLastProcessedDeletion(path: FilePath, db: FileProcessingStamp | false) {
this.services.path.unmarkChanges(path);
if (db) this.updateLastProcessedDatabase(path, db);
this.updateLastProcessedFile(path, this.statToKey(null));
@@ -502,7 +536,7 @@ export class HiddenFileSync extends LiveSyncCommands {
}
}
async updateLastProcessedAsActualDatabase(file: FilePath, doc?: MetaEntry | LoadedEntry | null | false) {
async updateLastProcessedAsActualDatabase(file: FilePath, doc?: FileProcessingStamp | null | false) {
const dbPath = addPrefix(file, ICHeader);
if (!doc) doc = await this.localDatabase.getDBEntryMeta(dbPath);
if (!doc) return;
@@ -932,7 +966,7 @@ Offline Changed files: ${processFiles.length}`;
}
);
showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
showJSONMergeDialogAndMerge(docA: ComparisonFile, docB: ComparisonFile): Promise<boolean> {
return new Promise((res) => {
this._log("Opening data-merging dialog", LOG_LEVEL_VERBOSE);
const docs = [docA, docB];
@@ -1001,7 +1035,7 @@ Offline Changed files: ${processFiles.length}`;
// <-- Conflict processing
// --> Event Source Handler (Database)
getDocProps(doc: LoadedEntry) {
getDocProps(doc: FileChangeRef) {
/*
type DocumentProps = {
id: DocumentID;
@@ -1025,7 +1059,7 @@ Offline Changed files: ${processFiles.length}`;
const isDeleted = doc._deleted || doc.deleted || false;
return { id, rev, revDisplay, prefixedPath, path, isDeleted, shortenedId, shortenedPath };
}
async processReplicationResult(doc: LoadedEntry): Promise<boolean> {
async processReplicationResult(doc: FileChangeRef): Promise<boolean> {
const info = this.getDocProps(doc);
const path = info.path;
if (this.pendingConflictChecks.has(addPrefix(path, ICHeader))) {
@@ -1618,11 +1652,20 @@ Offline Changed files: ${files.length}`;
// --> Storage To Database Functions
async __loadBaseSaveData(file: FilePath, includeContent = true): Promise<LoadedEntry | false> {
__loadBaseSaveData(file: FilePath, includeContent?: true): Promise<SaveBase | false>;
__loadBaseSaveData(file: FilePath, includeContent: false): Promise<FileMetadata | false>;
__loadBaseSaveData(file: FilePath, includeContent: boolean): Promise<SaveBase | FileMetadata | false>;
async __loadBaseSaveData(file: FilePath, includeContent = true): Promise<SaveBase | FileMetadata | false> {
return await this.loadBaseSaveData(file, includeContent, false);
}
private async loadBaseSaveDataStrict(file: FilePath, includeContent = true): Promise<LoadedEntry | false> {
private loadBaseSaveDataStrict(file: FilePath, includeContent?: true): Promise<SaveBase | false>;
private loadBaseSaveDataStrict(file: FilePath, includeContent: false): Promise<FileMetadata | false>;
private loadBaseSaveDataStrict(file: FilePath, includeContent: boolean): Promise<SaveBase | FileMetadata | false>;
private async loadBaseSaveDataStrict(
file: FilePath,
includeContent = true
): Promise<SaveBase | FileMetadata | false> {
return await this.loadBaseSaveData(file, includeContent, true);
}
@@ -1630,15 +1673,15 @@ Offline Changed files: ${files.length}`;
file: FilePath,
includeContent: boolean,
strict: boolean
): Promise<LoadedEntry | false> {
): Promise<SaveBase | FileMetadata | false> {
const prefixedFileName = addPrefix(file, ICHeader);
const id = await this.path2id(prefixedFileName, ICHeader);
try {
const old = includeContent
let old = includeContent
? await this.localDatabase.getDBEntry(prefixedFileName, undefined, false, true)
: await this.localDatabase.getDBEntryMeta(prefixedFileName, { conflicts: true }, true);
if (old === false) {
const baseSaveData: LoadedEntry = {
old = {
_id: id,
data: [],
path: prefixedFileName,
@@ -1651,10 +1694,17 @@ Offline Changed files: ${files.length}`;
type: "newnote",
eden: {},
};
return baseSaveData;
} else {
return old;
}
if (includeContent) return old;
if (old.type === "notes") {
const { datatype: _datatype, ...metadata } = old;
void _datatype;
return { ...metadata, children: [] };
}
const { data: _data, datatype: _datatype, ...metadata } = old;
void _data;
void _datatype;
return metadata;
} catch (ex) {
if (strict) throw ex;
this._log(`Getting base save data failed`);
@@ -1723,12 +1773,11 @@ Offline Changed files: ${files.length}`;
return undefined;
}
}
const saveData: SavingEntry = {
const saveData: CompatibleFileToSave = {
...baseData,
data: fileInfo.body,
mtime: fileInfo.stat.mtime,
size: fileInfo.stat.size,
children: [],
deleted: false,
type: baseData.datatype,
};
@@ -1802,19 +1851,16 @@ Offline Changed files: ${files.length}`;
if (storedRevision === false) {
return false;
}
this.updateLastProcessed(
storeFilePath,
{
...baseData,
_rev: storedRevision,
path: prefixedFileName,
ctime: fileInfo.stat.ctime,
mtime: fileInfo.stat.mtime,
size: fileInfo.stat.size,
deleted: false,
},
fileInfo.stat
);
const savedMetadata = {
...baseData,
_rev: storedRevision,
path: prefixedFileName,
ctime: fileInfo.stat.ctime,
mtime: fileInfo.stat.mtime,
size: fileInfo.stat.size,
deleted: false,
};
this.updateLastProcessed(storeFilePath, savedMetadata, fileInfo.stat);
this._log(`STORAGE --> DB:${storageFilePath}: (hidden, selected branch) Done`);
return true;
} catch (ex) {
@@ -1868,13 +1914,20 @@ Offline Changed files: ${files.length}`;
this.updateLastProcessedDeletion(storeFilePath, baseData);
return true;
}
const saveData: LoadedEntry = {
...baseData,
const {
data: _data,
datatype: _datatype,
...metadata
}: FileMetadata & { data?: string | string[]; datatype?: EntryTypeNotes } = baseData;
void _data;
void _datatype;
const saveData: CurrentFileDocument = {
...metadata,
mtime,
size: 0,
children: [],
deleted: true,
type: baseData.datatype,
type: baseData.type === "plain" ? "plain" : "newnote",
};
const ret = await this.localDatabase.putRaw(saveData);
if (ret && ret.ok) {
@@ -1903,7 +1956,7 @@ Offline Changed files: ${files.length}`;
async extractInternalFileFromDatabase(
storageFilePath: FilePath,
force = false,
metaEntry?: MetaEntry | LoadedEntry,
metaEntry?: AnyEntry,
preventDoubleProcess = true,
onlyNew = false,
includeDeletion = true,
@@ -2016,7 +2069,7 @@ Offline Changed files: ${files.length}`;
}
}
async __writeFile(storageFilePath: FilePath, fileOnDB: LoadedEntry, force: boolean): Promise<false | UXStat> {
async __writeFile(storageFilePath: FilePath, fileOnDB: FileContent, force: boolean): Promise<false | UXStat> {
try {
const statBefore = await this.core.storageAccess.statHidden(storageFilePath);
const isExist = statBefore != null;
@@ -1,4 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import PouchDB from "pouchdb-core";
import MemoryAdapter from "pouchdb-adapter-memory";
import { readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import {
type DocumentID,
LOG_LEVEL_NOTICE,
@@ -33,6 +36,155 @@ vi.mock("./configureHiddenFileSyncMode.ts", () => ({
import { HiddenFileSync } from "./CmdHiddenFileSync.ts";
import { configureHiddenFileSyncMode } from "./configureHiddenFileSyncMode.ts";
import type { FileContentView, RevisionRef } from "@vrtmrz/livesync-commonlib/documents";
import type { JsonComparisonDocument } from "@/features/HiddenFileCommon/jsonComparison";
async function checkInputContracts(sync: HiddenFileSync, path: FilePath, id: DocumentID) {
sync.docToKey({ mtime: 1, size: 2, _rev: "2-current", deleted: false });
sync.updateLastProcessed(path, { mtime: 1, size: 2 }, { mtime: 1, size: 2 });
await sync.__writeFile(path, { path, type: "plain", data: ["body"], ctime: 1, mtime: 2 }, false);
const metadata = await sync.__loadBaseSaveData(path, false);
if (metadata !== false && metadata.type !== "notes") {
// @ts-expect-error Current Metadata does not supply reconstructed content.
metadata.data;
}
const base = await sync.__loadBaseSaveData(path, true);
if (base !== false) {
const content: FileContentView = base;
// @ts-expect-error A new save base does not promise a database revision.
const revision: RevisionRef = base;
void [content, revision];
}
const comparison: JsonComparisonDocument = { _id: id, mtime: 1, datatype: "plain", data: ["{}"] };
// @ts-expect-error JSON comparison requires its existing decoding discriminator.
const missingDatatype: JsonComparisonDocument = { _id: id, mtime: 1, data: ["{}"] };
void [comparison, missingDatatype];
}
void checkInputContracts;
describe("HiddenFileSync save-base contracts", () => {
function createSaveBase(type: "plain" | "notes" = "plain") {
const path = ".obsidian/example.bin" as FilePath;
const stored = {
_id: "i:example" as DocumentID,
_rev: "2-existing",
path: `i:${path}` as FilePathWithPrefix,
ctime: 7,
mtime: 11,
size: 3,
type,
datatype: type === "plain" ? "plain" : "newnote",
data: type === "plain" ? "" : "AP8B",
children: [],
eden: {},
deleted: false,
};
const written: unknown[] = [];
const localDatabase = {
getDBEntryMeta: vi.fn(async () => stored as unknown),
getDBEntry: vi.fn(async () => ({ ...stored, data: ["AP8B"] })),
putRaw: vi.fn(async (document: unknown) => {
written.push(structuredClone(document));
return { ok: true, rev: "3-deleted" };
}),
removeRevision: vi.fn(),
};
const sync = Object.create(HiddenFileSync.prototype) as HiddenFileSync;
Object.assign(sync, {
core: { services: { vault: { isIgnoredByIgnoreFile: vi.fn(async () => false) } } },
localDatabase,
path2id: vi.fn(async () => stored._id),
updateLastProcessedDeletion: vi.fn(),
_log: vi.fn(),
});
return { sync, localDatabase, path, stored, written };
}
it("returns current Metadata without placeholder content in metadata mode", async () => {
const { sync, localDatabase, path, stored } = createSaveBase();
const metadata = await sync.__loadBaseSaveData(path, false);
expect(metadata).toMatchObject({ _id: stored._id, _rev: stored._rev, type: "plain" });
expect(metadata).not.toHaveProperty("data");
expect(metadata).not.toHaveProperty("datatype");
expect(localDatabase.getDBEntry).not.toHaveBeenCalled();
});
it("keeps the inline body in legacy Metadata for the existing decoder", async () => {
const { sync, path } = createSaveBase("notes");
expect(await sync.__loadBaseSaveData(path, false)).toMatchObject({ type: "notes", data: "AP8B" });
});
it("retains content and revision in content mode without a Metadata read", async () => {
const { sync, localDatabase, path, stored } = createSaveBase("notes");
expect(await sync.__loadBaseSaveData(path)).toMatchObject({
_rev: stored._rev,
data: ["AP8B"],
datatype: "newnote",
});
expect(localDatabase.getDBEntryMeta).not.toHaveBeenCalled();
});
it("represents a new save base without claiming database content in metadata mode", async () => {
const { sync, localDatabase, path } = createSaveBase();
localDatabase.getDBEntryMeta.mockResolvedValue(false);
const metadata = await sync.__loadBaseSaveData(path, false);
expect(metadata).toMatchObject({ mtime: 0, size: 0, type: "newnote" });
expect(metadata).not.toHaveProperty("_rev");
expect(metadata).not.toHaveProperty("data");
expect(metadata).not.toHaveProperty("datatype");
});
it.each(["plain", "notes"] as const)("stores a %s logical deletion without content fields", async (type) => {
const { sync, localDatabase, path, stored, written } = createSaveBase(type);
await expect(sync.deleteInternalFileOnDatabase(path)).resolves.toBe(true);
const deletion = written[0];
expect(deletion).toMatchObject({
_id: stored._id,
_rev: stored._rev,
path: stored.path,
ctime: stored.ctime,
type: type === "plain" ? "plain" : "newnote",
deleted: true,
size: 0,
children: [],
});
expect(deletion).not.toHaveProperty("data");
expect(deletion).not.toHaveProperty("datatype");
expect(localDatabase.getDBEntry).not.toHaveBeenCalled();
expect(localDatabase.removeRevision).not.toHaveBeenCalled();
});
it("retains legacy bytes in the parent revision after a Metadata-only logical deletion", async () => {
const { sync, localDatabase, path, stored } = createSaveBase("notes");
const MemoryPouchDB = PouchDB.plugin(MemoryAdapter);
const database = new MemoryPouchDB<Record<string, unknown>>(`hidden-contract-${crypto.randomUUID()}`, {
adapter: "memory",
});
try {
const { _rev: _revision, ...initial } = stored;
const original = await database.put(initial);
localDatabase.getDBEntryMeta.mockResolvedValue({ ...stored, _rev: original.rev });
localDatabase.putRaw.mockImplementation(async (document: unknown) => {
const result = await database.put(document as PouchDB.Core.PutDocument<Record<string, unknown>>);
return { ok: true, rev: result.rev };
});
await expect(sync.deleteInternalFileOnDatabase(path)).resolves.toBe(true);
const deleted = await database.get(stored._id, { revs: true });
expect(deleted).toMatchObject({ type: "newnote", deleted: true, children: [] });
expect(deleted).not.toHaveProperty("data");
expect(deleted).not.toHaveProperty("datatype");
expect(deleted._revisions?.ids[1]).toBe(original.rev.split("-")[1]);
const previous = await database.get(stored._id, { rev: original.rev });
expect(previous.data).toBe("AP8B");
const restored = readAsBlob({ path: stored.path, datatype: "newnote", data: previous.data as string });
expect(new Uint8Array(await restored.arrayBuffer())).toEqual(new Uint8Array([0, 255, 1]));
} finally {
await database.destroy();
}
});
});
function createHiddenRevisionOperation() {
const path = ".obsidian/plugins/example/data.json" as FilePath;
@@ -40,7 +192,7 @@ function createHiddenRevisionOperation() {
path,
name: "data.json",
isInternal: true,
body: new Blob(["{\"value\":\"vault\"}"]),
body: new Blob(['{"value":"vault"}']),
stat: {
ctime: 1,
mtime: 2,
@@ -66,12 +218,11 @@ function createHiddenRevisionOperation() {
_rev: "3-winner",
} as MetaEntry;
const databaseFileAccess = {
fetchEntryMeta: vi.fn(
async (_path: unknown, revision?: string) =>
revision === selected._rev ? selected : winner
fetchEntryMeta: vi.fn(async (_path: unknown, revision?: string) =>
revision === selected._rev ? selected : winner
),
getConflictedRevs: vi.fn(async () => [selected._rev]),
fetchEntryFromMeta: vi.fn(async () => ({ ...selected, data: "{\"value\":\"database\"}" })),
fetchEntryFromMeta: vi.fn(async () => ({ ...selected, data: '{"value":"database"}' })),
storeWithBaseRevision: vi.fn(async () => "3-vault-child"),
};
const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync;
@@ -609,16 +760,11 @@ describe("HiddenFileSync configuration-change notices", () => {
describe("HiddenFileSync exact revision repair operations", () => {
it("stores the current hidden Vault file as a child of the selected live revision", async () => {
const {
hiddenFileSync,
file,
selected,
databaseFileAccess,
} = createHiddenRevisionOperation();
const { hiddenFileSync, file, selected, databaseFileAccess } = createHiddenRevisionOperation();
await expect(
hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(file, selected._rev!)
).resolves.toBe(true);
await expect(hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(file, selected._rev!)).resolves.toBe(
true
);
expect(databaseFileAccess.storeWithBaseRevision).toHaveBeenCalledWith(
expect.objectContaining({
@@ -637,36 +783,22 @@ describe("HiddenFileSync exact revision repair operations", () => {
});
it("refuses to extend a hidden-file revision which is no longer live", async () => {
const {
hiddenFileSync,
file,
selected,
databaseFileAccess,
} = createHiddenRevisionOperation();
const { hiddenFileSync, file, selected, databaseFileAccess } = createHiddenRevisionOperation();
databaseFileAccess.getConflictedRevs.mockResolvedValue([]);
await expect(
hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(file, selected._rev!)
).resolves.toBe(false);
await expect(hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(file, selected._rev!)).resolves.toBe(
false
);
expect(databaseFileAccess.storeWithBaseRevision).not.toHaveBeenCalled();
expect(hiddenFileSync.updateLastProcessed).not.toHaveBeenCalled();
});
it("does not create a hidden-file child when asked only to mark a revision which differs from the Vault", async () => {
const {
hiddenFileSync,
file,
selected,
databaseFileAccess,
} = createHiddenRevisionOperation();
const { hiddenFileSync, file, selected, databaseFileAccess } = createHiddenRevisionOperation();
await expect(
hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(
file,
selected._rev!,
false
)
hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(file, selected._rev!, false)
).resolves.toBe(false);
expect(databaseFileAccess.storeWithBaseRevision).not.toHaveBeenCalled();
@@ -674,61 +806,39 @@ describe("HiddenFileSync exact revision repair operations", () => {
});
it("marks a matching hidden-file revision without creating a child", async () => {
const {
hiddenFileSync,
file,
selected,
databaseFileAccess,
} = createHiddenRevisionOperation();
const { hiddenFileSync, file, selected, databaseFileAccess } = createHiddenRevisionOperation();
databaseFileAccess.fetchEntryFromMeta.mockResolvedValue({
...selected,
data: "{\"value\":\"vault\"}",
data: '{"value":"vault"}',
});
await expect(
hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(
file,
selected._rev!,
false
)
hiddenFileSync.storeInternalFileToDatabaseWithBaseRevision(file, selected._rev!, false)
).resolves.toBe(true);
expect(databaseFileAccess.storeWithBaseRevision).not.toHaveBeenCalled();
expect(hiddenFileSync.updateLastProcessed).toHaveBeenCalledWith(
file.path,
selected,
file.stat
);
expect(hiddenFileSync.updateLastProcessed).toHaveBeenCalledWith(file.path, selected, file.stat);
});
it("applies the selected live hidden-file revision through the existing extraction path", async () => {
const {
hiddenFileSync,
path,
selected,
} = createHiddenRevisionOperation();
const { hiddenFileSync, path, selected } = createHiddenRevisionOperation();
const extract = vi.fn(async () => true);
hiddenFileSync.extractInternalFileFromDatabase = extract;
await expect(
hiddenFileSync.extractInternalFileRevisionFromDatabase(path, selected._rev!, true)
).resolves.toBe(true);
await expect(hiddenFileSync.extractInternalFileRevisionFromDatabase(path, selected._rev!, true)).resolves.toBe(
true
);
expect(extract).toHaveBeenCalledWith(path, true, undefined, true, false, true, selected._rev);
});
it("does not apply a hidden-file revision which ceased to be live", async () => {
const {
hiddenFileSync,
path,
selected,
databaseFileAccess,
} = createHiddenRevisionOperation();
const { hiddenFileSync, path, selected, databaseFileAccess } = createHiddenRevisionOperation();
databaseFileAccess.getConflictedRevs.mockResolvedValue([]);
await expect(
hiddenFileSync.extractInternalFileRevisionFromDatabase(path, selected._rev!, true)
).resolves.toBe(false);
await expect(hiddenFileSync.extractInternalFileRevisionFromDatabase(path, selected._rev!, true)).resolves.toBe(
false
);
expect(databaseFileAccess.fetchEntryFromMeta).not.toHaveBeenCalled();
});
+2 -2
View File
@@ -2,12 +2,12 @@ import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import {
LOG_LEVEL_INFO,
LOG_LEVEL_NOTICE,
type AnyEntry,
type DocumentID,
type FilePath,
type FilePathWithPrefix,
type LOG_LEVEL,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { DocumentPathRef } from "@vrtmrz/livesync-commonlib/documents";
import { MARK_DONE } from "@/modules/features/ModuleLog.ts";
import type { LiveSyncCore } from "@/main.ts";
// import { __$checkInstanceBinding } from "@vrtmrz/livesync-commonlib/compat/dev/checks";
@@ -33,7 +33,7 @@ export abstract class LiveSyncCommands {
return await this.services.path.path2id(filename, prefix);
}
getPath(entry: AnyEntry): FilePathWithPrefix {
getPath(entry: DocumentPathRef): FilePathWithPrefix {
return this.services.path.getPath(entry);
}
+4 -3
View File
@@ -1,5 +1,6 @@
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
import type { AnyEntry, FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { DocumentPathRef } from "@vrtmrz/livesync-commonlib/documents";
import type { IMinimumLiveSyncCommands, LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils";
@@ -34,11 +35,11 @@ export abstract class AbstractModule<
this.core.settings = value;
}
getPath(entry: AnyEntry): FilePathWithPrefix {
getPath(entry: DocumentPathRef): FilePathWithPrefix {
return this.services.path.getPath(entry);
}
getPathWithoutPrefix(entry: AnyEntry): FilePathWithPrefix {
getPathWithoutPrefix(entry: DocumentPathRef): FilePathWithPrefix {
return stripAllPrefixes(this.services.path.getPath(entry));
}
@@ -10,6 +10,7 @@ import {
LOG_LEVEL_NOTICE,
LOG_LEVEL_VERBOSE,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { FileContentView } from "@vrtmrz/livesync-commonlib/documents";
import { Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger";
import { isErrorOfMissingDoc } from "@vrtmrz/livesync-commonlib/compat/pouchdb/utils_couchdb";
import { fireAndForget, getDocData } from "@vrtmrz/livesync-commonlib/compat/common/utils";
@@ -42,7 +43,7 @@ function isComparableTextDecode(path: string) {
const ext = path.split(".").splice(-1)[0].toLowerCase();
return ["json"].includes(ext);
}
function readDocument(w: LoadedEntry) {
function readDocument(w: FileContentView) {
if (w.data.length == 0) return "";
if (isImage(w.path)) {
return new Uint8Array(decodeBinary(w.data));
@@ -1,7 +1,8 @@
import { type TFile } from "@/deps.ts";
import { eventHub } from "@/common/events.ts";
import { EVENT_REQUEST_SHOW_HISTORY } from "@/common/obsidianEvents.ts";
import type { FilePathWithPrefix, LoadedEntry, DocumentID } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { FilePathWithPrefix, DocumentID } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { DocumentIdentity } from "@vrtmrz/livesync-commonlib/documents";
import { AbstractObsidianModule } from "@/modules/AbstractObsidianModule.ts";
import { DocumentHistoryModal } from "./DocumentHistory/DocumentHistoryModal.ts";
import { fireAndForget } from "octagonal-wheels/promises";
@@ -26,7 +27,7 @@ export class ModuleObsidianDocumentHistory extends AbstractObsidianModule {
});
eventHub.onEvent(
EVENT_REQUEST_SHOW_HISTORY,
({ file, fileOnDB }: { file: TFile | FilePathWithPrefix; fileOnDB: LoadedEntry }) => {
({ file, fileOnDB }: { file: TFile | FilePathWithPrefix; fileOnDB: DocumentIdentity }) => {
this.showHistory(file, fileOnDB._id);
}
);
+3 -3
View File
@@ -4,6 +4,7 @@ import type {
FilePath,
FilePathWithPrefix,
LoadedEntry,
MetaEntry,
ObsidianLiveSyncSettings,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { getFileRegExp } from "@vrtmrz/livesync-commonlib/compat/common/utils";
@@ -14,7 +15,7 @@ import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/
import type { IPathService, IUIService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import { addPrefix, stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
type DatabaseMeta = LoadedEntry & {
type DatabaseMeta = MetaEntry & {
_rawStorageType: string | null;
_legacyBodyPresent: boolean;
_revs_info?: Array<{
@@ -182,12 +183,11 @@ async function getLocalDatabaseMeta(
_conflicts: raw._conflicts,
_revs_info: raw._revs_info,
path: target ? (raw.path ?? path) : path,
data: legacyBodyPresent ? raw.data : "",
...(legacy ? { data: legacyBodyPresent ? raw.data : "" } : {}),
ctime: raw.ctime ?? 0,
mtime: raw.mtime ?? 0,
size: raw.size ?? 0,
children: type === "newnote" || type === "plain" ? (raw.children ?? []) : [],
datatype: type === "newnote" ? "newnote" : "plain",
deleted: raw.deleted ?? raw._deleted,
type,
eden: raw.eden ?? {},
@@ -10,10 +10,9 @@ import {
type AnyEntry,
type EntryDoc,
type EntryLeaf,
type LoadedEntry,
type MetaEntry,
type ObsidianLiveSyncSettings,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { DocumentPathRef, FileChangeRef, FileMetadataNotification } from "@vrtmrz/livesync-commonlib/documents";
import { isChunk } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils";
import {
LOG_LEVEL_DEBUG,
@@ -88,7 +87,7 @@ export class ReplicateResultProcessor {
private logError(e: unknown) {
Logger(e, LOG_LEVEL_VERBOSE);
}
private reportVaultReflectionFailure(entry: MetaEntry, cause?: unknown) {
private reportVaultReflectionFailure(entry: DocumentPathRef, cause?: unknown) {
this.context.onVaultReflectionFailure?.();
this.log(
`Live replication could not reflect ${this.getPath(entry)} from the local database to the Vault; this path remains eligible for a later Vault scan.`,
@@ -106,7 +105,7 @@ export class ReplicateResultProcessor {
return this.context.services;
}
getPath(entry: AnyEntry): string {
getPath(entry: DocumentPathRef): string {
return this.services.path.getPath(entry);
}
@@ -531,8 +530,7 @@ export class ReplicateResultProcessor {
}
// Phase 2.1: process the document and apply to storage
// This function is serialized per document to avoid race-condition for the same document.
private _applyToDatabase(doc_: PouchDB.Core.ExistingDocument<AnyEntry>) {
const dbDoc = doc_ as LoadedEntry; // It has no `data`
private _applyToDatabase(dbDoc: PouchDB.Core.ExistingDocument<AnyEntry>) {
const path = this.getPath(dbDoc);
return withLock(`replication-process:${dbDoc._id}`, async () => {
const docNote = `${path} (${shortenId(dbDoc._id)}, ${shortenRev(dbDoc._rev)})`;
@@ -547,9 +545,7 @@ export class ReplicateResultProcessor {
const isDeleted = dbDoc._deleted === true || ("deleted" in dbDoc && dbDoc.deleted === true);
// Gather full document if not deleted
const doc = isDeleted
? { ...dbDoc, data: "" }
: await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, false, true);
const doc = isDeleted ? dbDoc : await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, false, true);
if (!doc) {
// Failed to gather content
this.context.onVaultReflectionFailure?.();
@@ -563,14 +559,14 @@ export class ReplicateResultProcessor {
} else if (this.services.vault.isValidPath(this.getPath(doc))) {
// Apply to storage if the path is valid
try {
const reflected = await this.applyToStorage(doc as MetaEntry);
const reflected = await this.applyToStorage(doc);
if (!reflected) {
this.reportVaultReflectionFailure(doc as MetaEntry);
this.reportVaultReflectionFailure(doc);
return;
}
this.log(`Processed: ${docNote}`, LOG_LEVEL_DEBUG);
} catch (error) {
this.reportVaultReflectionFailure(doc as MetaEntry, error);
this.reportVaultReflectionFailure(doc, error);
}
} else {
// Should process, but have an invalid path
@@ -584,7 +580,7 @@ export class ReplicateResultProcessor {
* @param entry
* @returns
*/
protected applyToStorage(entry: MetaEntry) {
protected applyToStorage(entry: FileMetadataNotification) {
return this.withCounting(
() => this.services.replication.processSynchroniseResult(entry),
this.services.replication.storageApplyingCount
@@ -596,10 +592,10 @@ export class ReplicateResultProcessor {
* @param dbDoc Document to check
* @returns True if processing is required; false otherwise
*/
protected async checkIsChangeRequiredForDatabaseProcessing(dbDoc: LoadedEntry): Promise<boolean> {
protected async checkIsChangeRequiredForDatabaseProcessing(dbDoc: FileChangeRef): Promise<boolean> {
const path = this.getPath(dbDoc);
try {
const savedDoc = await this.localDatabase.getRaw<LoadedEntry>(dbDoc._id, {
const savedDoc = await this.localDatabase.getRaw<AnyEntry>(dbDoc._id, {
conflicts: true,
revs_info: true,
});