Keep unreadable revisions for explicit repair

This commit is contained in:
vorotamoroz
2026-07-24 16:15:46 +00:00
parent 658ad6dfe4
commit 07cba4ce83
21 changed files with 2036 additions and 180 deletions
@@ -9,8 +9,7 @@
export const liveSyncProvisionalEnglishMessages = {
"This first setup has several short steps because it confirms encryption, the connection method, and which device provides the initial data. Once it is complete, additional devices can reuse a Setup URI.":
"This first setup has several short steps because it confirms encryption, the connection method, and which device provides the initial data. Once it is complete, additional devices can reuse a Setup URI.",
"Setup Complete: Preparing to Fetch from Another Device":
"Setup Complete: Preparing to Fetch from Another Device",
"Setup Complete: Preparing to Fetch from Another Device": "Setup Complete: Preparing to Fetch from Another Device",
"The P2P connection has been configured successfully. The initial synchronisation data must now be fetched from an online source device.":
"The P2P connection has been configured successfully. The initial synchronisation data must now be fetched from an online source device.",
"After restarting, select an online source device for the initial Fetch. The local LiveSync database on this device will be rebuilt from that source. Unsynchronised files in this Vault may conflict with the fetched data.":
@@ -66,6 +65,53 @@ export const liveSyncProvisionalEnglishMessages = {
"This file has ${COUNT} unresolved versions. They will be reviewed one pair at a time.",
"Sync now": "Sync now",
"Apply pending changes now": "Apply pending changes now",
"Copy database information for the active file": "Copy database information for the active file",
"Copy database information for a file": "Copy database information for a file",
"Copy revision, conflict, and local chunk availability information, including document and chunk identifiers but not file contents.":
"Copy revision, conflict, and local chunk availability information, including document and chunk identifiers but not file contents.",
"Choose file": "Choose file",
"Choose a file to inspect": "Choose a file to inspect",
"Database information for ${FILE}": "Database information for ${FILE}",
"All revisions and chunk availability below are a snapshot of this device's local database; the remote is not queried. Review the Vault-relative path, document identifier, content-derived chunk identifiers, and metadata before sharing this report. File contents are omitted.":
"All revisions and chunk availability below are a snapshot of this device's local database; the remote is not queried. Review the Vault-relative path, document identifier, content-derived chunk identifiers, and metadata before sharing this report. File contents are omitted.",
"Vault file: modified ${TIME}, size ${SIZE}": "Vault file: modified ${TIME}, size ${SIZE}",
"Vault file: missing": "Vault file: missing",
"Local database document: missing": "Local database document: missing",
"${ROLE}: ${REVISION}": "${ROLE}: ${REVISION}",
"Winner revision": "Winner revision",
"Conflict revision": "Conflict revision",
"Unknown revision": "Unknown revision",
"Logical deletion": "Logical deletion",
"Readable on this device; recorded size ${RECORDED}, decoded size ${ACTUAL}":
"Readable on this device; recorded size ${RECORDED}, decoded size ${ACTUAL}",
"Unreadable on this device; ${COUNT} referenced chunks are missing or deleted":
"Unreadable on this device; ${COUNT} referenced chunks are missing or deleted",
"Matches the current Vault file": "Matches the current Vault file",
"Differs from the current Vault file": "Differs from the current Vault file",
"Retry reading revision": "Retry reading revision",
"Discard unreadable revision": "Discard unreadable revision",
"Discard database revision ${REVISION} of ${FILE}? This creates a logical deletion for that exact live revision. Missing content cannot be recovered by this action.":
"Discard database revision ${REVISION} of ${FILE}? This creates a logical deletion for that exact live revision. Missing content cannot be recovered by this action.",
"Revision metadata is unavailable on this device": "Revision metadata is unavailable on this device",
"Shared ancestor ${REVISION} is not readable on this device. Automatic three-way merging may be unavailable, but the live revisions remain available for explicit review.":
"Shared ancestor ${REVISION} is not readable on this device. Automatic three-way merging may be unavailable, but the live revisions remain available for explicit review.",
"No shared ancestor is available for this conflict. The live revisions remain available for explicit review.":
"No shared ancestor is available for this conflict. The live revisions remain available for explicit review.",
"Show revision history": "Show revision history",
"Use Vault file in local database": "Use Vault file in local database",
"Restore database winner to Vault": "Restore database winner to Vault",
"Copy database information": "Copy database information",
"Recreate chunks for current Vault files": "Recreate chunks for current Vault files",
"Recreate chunks from the files currently present in this Vault. This cannot reconstruct unavailable historical or conflict content.":
"Recreate chunks from the files currently present in this Vault. This cannot reconstruct unavailable historical or conflict content.",
"Recreate current chunks": "Recreate current chunks",
"Resolve every conflict by modification time? This logically deletes every version except the newest one and cannot recover content which is already unavailable.":
"Resolve every conflict by modification time? This logically deletes every version except the newest one and cannot recover content which is already unavailable.",
"Resolve all conflicts by the newest version": "Resolve all conflicts by the newest version",
"Verify and repair all files": "Verify and repair all files",
"Compare each Vault file with every live local-database revision. Unreadable conflict versions remain visible until you retry or explicitly discard an exact revision.":
"Compare each Vault file with every live local-database revision. Unreadable conflict versions remain visible until you retry or explicitly discard an exact revision.",
"Verify all": "Verify all",
"Connection settings": "Connection settings",
"Saved connections": "Saved connections",
} as const;
@@ -86,8 +86,11 @@ export class ModuleConflictResolver extends AbstractModule {
return MISSING_OR_ERROR;
}
if (rightLeaf == false) {
// Conflicted item could not load, delete this.
return await this.services.conflict.resolveByDeletingRevision(path, rightRev, "MISSING OLD REV");
// A locally unreadable conflict leaf may still be recoverable from another
// replica or backup. Keep it visible for explicit repair instead of treating
// missing chunks as evidence that the branch is obsolete.
this._log(`could not read conflicted revision ${rightRev}:${path}`, LOG_LEVEL_NOTICE);
return MISSING_OR_ERROR;
}
const isSame = leftLeaf.data == rightLeaf.data && leftLeaf.deleted == rightLeaf.deleted;
@@ -4,6 +4,7 @@ import {
DEFAULT_SETTINGS,
LOG_LEVEL_INFO,
LOG_LEVEL_NOTICE,
MISSING_OR_ERROR,
type FilePathWithPrefix,
type MetaEntry,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
@@ -189,6 +190,28 @@ describe("ModuleConflictResolver independent same-path creation", () => {
});
describe("ModuleConflictResolver sensible merge hand-off", () => {
it("keeps an unreadable non-winner revision unresolved", async () => {
const path = "missing-conflict-body.md" as FilePathWithPrefix;
const { module, resolveByDeletingRevision, tryAutoMerge } = createModule();
tryAutoMerge.mockResolvedValue({
leftRev: "3-current",
rightRev: "2-unreadable",
leftLeaf: {
rev: "3-current",
data: "Readable current body\n",
ctime: 1,
mtime: 3,
deleted: false,
},
rightLeaf: false,
});
const result = await module.checkConflictAndPerformAutoMerge(path);
expect(result).toBe(MISSING_OR_ERROR);
expect(resolveByDeletingRevision).not.toHaveBeenCalled();
});
it("stores the merged body and removes the resolved conflict leaf", async () => {
const path = "sensible.md" as FilePathWithPrefix;
const { module, resolveByDeletingRevision, tryAutoMerge } = createModule();
+8 -4
View File
@@ -3,6 +3,7 @@ import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
import { fireAndForget } from "octagonal-wheels/promises";
import { AbstractModule } from "@/modules/AbstractModule";
import { $msg } from "@/common/translation";
import { copyFileDatabaseInfo } from "@/serviceFeatures/fileDatabaseInfo";
// Separated Module for basic menu commands, which are not related to obsidian specific features. It is expected to be used in other platforms with minimal changes.
// However, it is odd that it has here at all; it really ought to be in each respective feature. It will likely be moved eventually. Until now, addCommand pointed to Obsidian's version.
export class ModuleBasicMenu extends AbstractModule {
@@ -16,11 +17,14 @@ export class ModuleBasicMenu extends AbstractModule {
});
this.addCommand({
id: "livesync-dump",
name: "Dump information of this doc ",
callback: () => {
name: $msg("Copy database information for the active file"),
checkCallback: (checking) => {
const file = this.services.vault.getActiveFilePath();
if (!file) return;
fireAndForget(() => this.localDatabase.getDBEntry(file, {}, true, false));
if (!file) return false;
if (!checking) {
fireAndForget(() => copyFileDatabaseInfo(this.core, file));
}
return true;
},
});
this.addCommand({
@@ -136,4 +136,35 @@ describe("ModuleBasicMenu command palette", () => {
expect(fixture.getCommand("livesync-abortsync").checkCallback?.(true)).toBe(true);
});
it("keeps active-file database information available and opens it in a copy dialogue", async () => {
const fixture = createFixture();
await fixture.module._everyOnloadStart();
const command = fixture.getCommand("livesync-dump");
expect(command.name).toBe("Copy database information for the active file");
expect(command.checkCallback?.(true)).toBe(true);
command.checkCallback?.(false);
await vi.waitFor(() => {
expect(fixture.services.UI.promptCopyToClipboard).toHaveBeenCalledOnce();
});
const [title, report] = fixture.services.UI.promptCopyToClipboard.mock.calls[0];
expect(title).toBe("Database information for note.md");
expect(report).toContain("note.md");
expect(report).toContain("2-current");
expect(report).toContain("h:private-chunk-id");
expect(report).toContain("1-chunk");
expect(fixture.core.localDatabase.getDBEntry).not.toHaveBeenCalled();
});
it("hides the active-file database report when no file is active", async () => {
const fixture = createFixture();
fixture.services.vault.getActiveFilePath.mockReturnValue(null);
await fixture.module._everyOnloadStart();
expect(fixture.getCommand("livesync-dump").checkCallback?.(true)).toBe(false);
});
});
+341 -158
View File
@@ -3,14 +3,13 @@ import {
type DocumentID,
LOG_LEVEL_NOTICE,
LOG_LEVEL_VERBOSE,
type LoadedEntry,
type MetaEntry,
type FilePath,
type EntryDoc,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { createBlob, getFileRegExp, isDocContentSame, readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger";
import { addPrefix, shouldBeIgnored, stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import { shouldBeIgnored } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import { $msg } from "@/common/translation";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
@@ -21,12 +20,24 @@ import {
EVENT_REQUEST_RUN_FIX_INCOMPLETE,
eventHub,
} from "@/common/events.ts";
import { ICHeader, ICXHeader, PSCHeader } from "@/common/types.ts";
import { ICHeader } from "@/common/types.ts";
import { HiddenFileSync } from "@/features/HiddenFileSync/CmdHiddenFileSync.ts";
import { EVENT_REQUEST_SHOW_HISTORY } from "@/common/obsidianEvents.ts";
import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts";
import type { PageFunctions } from "./SettingPane.ts";
import { isNotFoundError } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc";
import {
chooseAndCopyFileDatabaseInfo,
collectFileDatabaseInfoPaths,
copyFileDatabaseInfo,
retryReadFileDatabaseRevision,
} from "@/serviceFeatures/fileDatabaseInfo.ts";
import {
discardUnreadableLiveRevision,
inspectFileRepair,
type FileRepairInspection,
type FileRepairRevision,
} from "@/serviceFeatures/fileRepair.ts";
export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, { addPanel }: PageFunctions): void {
// const hatchWarn = this.createEl(paneEl, "div", { text: `To stop the boot up sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
// hatchWarn.addClass("op-warn-info");
@@ -67,6 +78,18 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
await this.app.commands.executeCommandById("obsidian-livesync:dump-debug-info");
})
);
new Setting(paneEl)
.setName($msg("Copy database information for a file"))
.setDesc(
$msg(
"Copy revision, conflict, and local chunk availability information, including document and chunk identifiers but not file contents."
)
)
.addButton((button) =>
button.setButtonText($msg("Choose file")).onClick(async () => {
await chooseAndCopyFileDatabaseInfo(this.core);
})
);
new Setting(paneEl)
.setName($msg("Analyse database usage"))
.setDesc(
@@ -99,132 +122,300 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
});
void addPanel(paneEl, "Recovery and Repair").then((paneEl) => {
const addResult = async (path: string, file: FilePathWithPrefix | false, fileOnDB: LoadedEntry | false) => {
const storageFileStat = file ? await this.core.storageAccess.statHidden(file) : null;
resultArea.appendChild(
this.createEl(resultArea, "div", {}, (el) => {
el.appendChild(this.createEl(el, "h6", { text: path }));
el.appendChild(
this.createEl(el, "div", {}, (infoGroupEl) => {
infoGroupEl.appendChild(
this.createEl(infoGroupEl, "div", {
text: `Storage : Modified: ${!storageFileStat ? `Missing:` : `${new Date(storageFileStat.mtime).toLocaleString()}, Size:${storageFileStat.size}`}`,
})
);
infoGroupEl.appendChild(
this.createEl(infoGroupEl, "div", {
text: `Database: Modified: ${!fileOnDB ? `Missing:` : `${new Date(fileOnDB.mtime).toLocaleString()}, Size:${fileOnDB.size} (actual size:${readAsBlob(fileOnDB).size})`}`,
})
);
})
const resultArea = paneEl.createDiv({ text: "", cls: "sls-repair-results" });
const addActionButton = (
parent: HTMLElement,
text: string,
action: (button: HTMLButtonElement) => Promise<void> | void,
warning = false
) => {
this.createEl(parent, "button", { text }, (button) => {
if (warning) {
button.addClass("mod-warning");
}
button.onClickEvent(async () => {
button.disabled = true;
try {
await action(button);
} finally {
if (button.isConnected) {
button.disabled = false;
}
}
});
});
};
const storeStorageInDatabase = async (path: string): Promise<boolean> => {
if (path.startsWith(".")) {
const addOn = this.core.getAddOn<HiddenFileSync>(HiddenFileSync.name);
if (!addOn) {
return false;
}
const file = (await addOn.scanInternalFiles()).find((entry) => entry.path === path);
if (!file) {
Logger(`Failed to find the file in the internal files: ${path}`, LOG_LEVEL_NOTICE);
return false;
}
return Boolean(await addOn.storeInternalFileToDatabase(file, true));
}
return Boolean(await this.core.fileHandler.storeFileToDB(path as FilePath, true));
};
const applyWinnerToStorage = async (
path: string,
revision: FileRepairRevision
): Promise<boolean> => {
if (revision.loadedEntry === false) {
return false;
}
if (revision.loadedEntry.path.startsWith(ICHeader)) {
const addOn = this.core.getAddOn<HiddenFileSync>(HiddenFileSync.name);
return addOn
? Boolean(await addOn.extractInternalFileFromDatabase(path as FilePath, true))
: false;
}
return Boolean(await this.core.fileHandler.dbToStorage(revision.loadedEntry as MetaEntry, null, true));
};
const addRepairResult = (inspection: FileRepairInspection) => {
const { information, revisions } = inspection;
const path = information.path;
const card = this.createEl(resultArea, "div", { cls: "sls-repair-result" });
const refresh = async () => {
card.remove();
const refreshed = await inspectFileRepair(this.core, path);
if (refreshed.requiresAttention) {
addRepairResult(refreshed);
} else {
Logger(`Verification no longer reports a problem for ${path}`, LOG_LEVEL_NOTICE);
}
};
this.createEl(card, "h6", { text: path });
if (information.storage.exists) {
this.createEl(card, "div", {
text: $msg("Vault file: modified ${TIME}, size ${SIZE}", {
TIME: new Date(information.storage.mtime ?? 0).toLocaleString(),
SIZE: `${information.storage.size ?? 0}`,
}),
});
} else {
this.createEl(card, "div", { text: $msg("Vault file: missing") });
}
if (!information.database.exists) {
this.createEl(card, "div", { text: $msg("Local database document: missing") });
}
const addRevision = (revision: FileRepairRevision) => {
const { metadata } = revision;
const revisionEl = this.createEl(card, "div", { cls: "sls-repair-revision" });
this.createEl(revisionEl, "div", {
text: $msg("${ROLE}: ${REVISION}", {
ROLE: revision.role === "winner" ? $msg("Winner revision") : $msg("Conflict revision"),
REVISION: metadata.revision ?? $msg("Unknown revision"),
}),
cls: "sls-repair-revision-title",
});
if (metadata.deleted) {
this.createEl(revisionEl, "div", { text: $msg("Logical deletion") });
} else if (revision.contentReadable) {
this.createEl(revisionEl, "div", {
text: $msg("Readable on this device; recorded size ${RECORDED}, decoded size ${ACTUAL}", {
RECORDED: `${metadata.recordedSize}`,
ACTUAL: `${revision.loadedEntry === false ? 0 : readAsBlob(revision.loadedEntry).size}`,
}),
});
} else {
const missing = metadata.chunks.filter(
({ embedded, localDatabaseState }) =>
!embedded && localDatabaseState !== "available"
);
if (fileOnDB && file) {
el.appendChild(
this.createEl(el, "button", { text: "Show history" }, (buttonEl) => {
buttonEl.onClickEvent(() => {
eventHub.emitEvent(EVENT_REQUEST_SHOW_HISTORY, {
file: file,
fileOnDB: fileOnDB,
});
});
})
);
this.createEl(revisionEl, "div", {
text: $msg("Unreadable on this device; ${COUNT} referenced chunks are missing or deleted", {
COUNT: `${missing.length}`,
}),
cls: "mod-warning",
});
if (missing.length > 0) {
this.createEl(revisionEl, "code", {
text: missing
.slice(0, 3)
.map(({ id }) => id)
.join(", ") + (missing.length > 3 ? ", …" : ""),
});
}
if (file) {
el.appendChild(
this.createEl(el, "button", { text: "Storage -> Database" }, (buttonEl) => {
buttonEl.onClickEvent(async () => {
if (file.startsWith(".")) {
const addOn = this.core.getAddOn<HiddenFileSync>(HiddenFileSync.name);
if (addOn) {
const file = (await addOn.scanInternalFiles()).find((e) => e.path == path);
if (!file) {
Logger(
`Failed to find the file in the internal files: ${path}`,
LOG_LEVEL_NOTICE
);
return;
}
if (!(await addOn.storeInternalFileToDatabase(file, true))) {
Logger(
`Failed to store the file to the database (Hidden file): ${file.path}`,
LOG_LEVEL_NOTICE
);
return;
}
}
} else {
if (!(await this.core.fileHandler.storeFileToDB(file, true))) {
Logger(
`Failed to store the file to the database: ${file}`,
LOG_LEVEL_NOTICE
);
return;
}
if (revision.contentMatchesStorage === true) {
this.createEl(revisionEl, "div", { text: $msg("Matches the current Vault file") });
} else if (revision.contentMatchesStorage === false) {
this.createEl(revisionEl, "div", { text: $msg("Differs from the current Vault file") });
}
if (!metadata.deleted && !revision.contentReadable && metadata.revision) {
const actions = this.createEl(revisionEl, "div", { cls: "sls-repair-actions" });
addActionButton(actions, $msg("Retry reading revision"), async () => {
const loaded = await retryReadFileDatabaseRevision(this.core, path, metadata.revision!);
Logger(
loaded
? `Revision ${metadata.revision} of ${path} is readable after retry`
: `Revision ${metadata.revision} of ${path} remains unreadable`,
LOG_LEVEL_NOTICE
);
await refresh();
});
addActionButton(
actions,
$msg("Discard unreadable revision"),
async () => {
const confirmed =
(await this.core.confirm.askYesNoDialog(
$msg(
"Discard database revision ${REVISION} of ${FILE}? This creates a logical deletion for that exact live revision. Missing content cannot be recovered by this action.",
{
REVISION: metadata.revision!,
FILE: path,
}
),
{
title: $msg("Discard unreadable revision"),
defaultOption: "No",
}
el.remove();
});
})
);
}
if (fileOnDB) {
el.appendChild(
this.createEl(el, "button", { text: "Database -> Storage" }, (buttonEl) => {
buttonEl.onClickEvent(async () => {
if (fileOnDB.path.startsWith(ICHeader)) {
const addOn = this.core.getAddOn<HiddenFileSync>(HiddenFileSync.name);
if (addOn) {
if (
!(await addOn.extractInternalFileFromDatabase(path as FilePath, true))
) {
Logger(
`Failed to store the file to the database (Hidden file): ${file}`,
LOG_LEVEL_NOTICE
);
return;
}
}
} else {
if (
!(await this.core.fileHandler.dbToStorage(
fileOnDB as MetaEntry,
null,
true
))
) {
Logger(
`Failed to store the file to the storage: ${fileOnDB.path}`,
LOG_LEVEL_NOTICE
);
return;
}
)) === "yes";
if (!confirmed) {
return;
}
const result = await discardUnreadableLiveRevision(
this.core,
path,
metadata.revision!
);
Logger(
`Discard unreadable revision ${metadata.revision} of ${path}: ${result}`,
result === "discarded" ? LOG_LEVEL_NOTICE : LOG_LEVEL_VERBOSE
);
await refresh();
},
true
);
}
};
revisions.forEach(addRevision);
for (const revision of information.database.unavailableConflictRevisions) {
const revisionEl = this.createEl(card, "div", { cls: "sls-repair-revision" });
this.createEl(revisionEl, "div", {
text: $msg("${ROLE}: ${REVISION}", {
ROLE: $msg("Conflict revision"),
REVISION: revision,
}),
cls: "sls-repair-revision-title",
});
this.createEl(revisionEl, "div", {
text: $msg("Revision metadata is unavailable on this device"),
cls: "mod-warning",
});
const actions = this.createEl(revisionEl, "div", { cls: "sls-repair-actions" });
addActionButton(actions, $msg("Retry reading revision"), async () => {
await retryReadFileDatabaseRevision(this.core, path, revision);
await refresh();
});
addActionButton(
actions,
$msg("Discard unreadable revision"),
async () => {
const confirmed =
(await this.core.confirm.askYesNoDialog(
$msg(
"Discard database revision ${REVISION} of ${FILE}? This creates a logical deletion for that exact live revision. Missing content cannot be recovered by this action.",
{
REVISION: revision,
FILE: path,
}
el.remove();
});
})
);
),
{
title: $msg("Discard unreadable revision"),
defaultOption: "No",
}
)) === "yes";
if (!confirmed) {
return;
}
await discardUnreadableLiveRevision(this.core, path, revision);
await refresh();
},
true
);
}
for (const base of information.database.mergeBases) {
if (base.contentAvailableLocally) {
continue;
}
this.createEl(card, "div", {
text: base.revision
? $msg(
"Shared ancestor ${REVISION} is not readable on this device. Automatic three-way merging may be unavailable, but the live revisions remain available for explicit review.",
{
REVISION: base.revision,
}
)
: $msg(
"No shared ancestor is available for this conflict. The live revisions remain available for explicit review."
),
cls: "sls-repair-ancestor-warning",
});
}
const winner = revisions.find(({ role }) => role === "winner");
const actions = this.createEl(card, "div", { cls: "sls-repair-actions" });
if (winner?.loadedEntry && information.storage.exists) {
const winnerEntry = winner.loadedEntry;
addActionButton(actions, $msg("Show revision history"), () => {
eventHub.emitEvent(EVENT_REQUEST_SHOW_HISTORY, {
file: path as FilePathWithPrefix,
fileOnDB: winnerEntry,
});
});
}
if (
information.storage.exists &&
information.database.conflictCount === 0 &&
(!winner || winner.contentReadable)
) {
addActionButton(actions, $msg("Use Vault file in local database"), async () => {
if (!(await storeStorageInDatabase(path))) {
Logger(`Failed to store the Vault file in the local database: ${path}`, LOG_LEVEL_NOTICE);
return;
}
return el;
})
);
await refresh();
});
}
if (
!information.storage.exists &&
information.database.conflictCount === 0 &&
winner?.loadedEntry
) {
addActionButton(actions, $msg("Restore database winner to Vault"), async () => {
if (!(await applyWinnerToStorage(path, winner))) {
Logger(`Failed to restore the database winner to the Vault: ${path}`, LOG_LEVEL_NOTICE);
return;
}
await refresh();
});
}
addActionButton(actions, $msg("Copy database information"), async () => {
await copyFileDatabaseInfo(this.core, path);
});
};
const checkBetweenStorageAndDatabase = async (file: FilePathWithPrefix, fileOnDB: LoadedEntry) => {
const dataContent = readAsBlob(fileOnDB);
const content = createBlob(await this.core.storageAccess.readHiddenFileBinary(file));
if (await isDocContentSame(content, dataContent)) {
Logger(`Compare: SAME: ${file}`);
} else {
Logger(`Compare: CONTENT IS NOT MATCHED! ${file}`, LOG_LEVEL_NOTICE);
void addResult(file, file, fileOnDB);
}
};
new Setting(paneEl)
.setName("Recreate missing chunks for all files")
.setDesc("This will recreate chunks for all files. If there were missing chunks, this may fix the errors.")
.setName($msg("Recreate chunks for current Vault files"))
.setDesc(
$msg(
"Recreate chunks from the files currently present in this Vault. This cannot reconstruct unavailable historical or conflict content."
)
)
.addButton((button) =>
button
.setButtonText("Recreate all")
.setButtonText($msg("Recreate current chunks"))
.setCta()
.onClick(async () => {
await this.core.fileHandler.createAllChunks(true);
@@ -240,40 +431,40 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
.setButtonText("Resolve All")
.setCta()
.onClick(async () => {
const confirmed =
(await this.core.confirm.askYesNoDialog(
$msg(
"Resolve every conflict by modification time? This logically deletes every version except the newest one and cannot recover content which is already unavailable."
),
{
title: $msg("Resolve all conflicts by the newest version"),
defaultOption: "No",
}
)) === "yes";
if (!confirmed) {
return;
}
await this.services.conflict.resolveAllConflictedFilesByNewerOnes();
})
);
new Setting(paneEl)
.setName("Verify and repair all files")
.setName($msg("Verify and repair all files"))
.setDesc(
"Compare the content of files between on local database and storage. If not matched, you will be asked which one you want to keep."
$msg(
"Compare each Vault file with every live local-database revision. Unreadable conflict versions remain visible until you retry or explicitly discard an exact revision."
)
)
.addButton((button) =>
button
.setButtonText("Verify all")
.setButtonText($msg("Verify all"))
.setDisabled(false)
.setCta()
.onClick(async () => {
resultArea.replaceChildren();
Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify");
const ignorePatterns = getFileRegExp(this.core.settings, "syncInternalFilesIgnorePatterns");
const targetPatterns = getFileRegExp(this.core.settings, "syncInternalFilesTargetPatterns");
this.core.localDatabase.clearCaches();
Logger("Start verifying all files", LOG_LEVEL_NOTICE, "verify");
const files = this.core.settings.syncInternalFiles
? await this.core.storageAccess.getFilesIncludeHidden("/", targetPatterns, ignorePatterns)
: await this.core.storageAccess.getFileNames();
const documents = [] as FilePath[];
const adn = this.core.localDatabase.findAllDocs();
for await (const i of adn) {
const path = this.services.path.getPath(i);
if (path.startsWith(ICXHeader)) continue;
if (path.startsWith(PSCHeader)) continue;
if (!this.core.settings.syncInternalFiles && path.startsWith(ICHeader)) continue;
documents.push(stripAllPrefixes(path));
}
const allPaths = [...new Set([...documents, ...files])];
const allPaths = await collectFileDatabaseInfoPaths(this.core);
let i = 0;
const incProc = () => {
i++;
@@ -295,28 +486,21 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
: false;
const fileOnStorage = stat != null ? stat : false;
if (!(await this.services.vault.isTargetFile(path))) return incProc();
const releaser = await semaphore.acquire(1);
if (fileOnStorage && this.services.vault.isFileSizeTooLarge(fileOnStorage.size))
return incProc();
const releaser = await semaphore.acquire(1);
try {
const isHiddenFile = path.startsWith(".");
const dbPath = isHiddenFile ? addPrefix(path, ICHeader) : path;
const fileOnDB = await this.core.localDatabase.getDBEntry(dbPath);
if (fileOnDB && this.services.vault.isFileSizeTooLarge(fileOnDB.size))
const inspection = await inspectFileRepair(this.core, path);
const winner = inspection.revisions.find(({ role }) => role === "winner");
if (
winner &&
this.services.vault.isFileSizeTooLarge(winner.metadata.recordedSize)
)
return incProc();
if (!fileOnDB && fileOnStorage) {
Logger(`Compare: Not found on the local database: ${path}`, LOG_LEVEL_NOTICE);
void addResult(path, path, false);
return incProc();
}
if (fileOnDB && !fileOnStorage) {
Logger(`Compare: Not found on the storage: ${path}`, LOG_LEVEL_NOTICE);
void addResult(path, false, fileOnDB);
return incProc();
}
if (fileOnStorage && fileOnDB) {
await checkBetweenStorageAndDatabase(path, fileOnDB);
if (inspection.requiresAttention) {
addRepairResult(inspection);
} else {
Logger(`Compare: SAME: ${path}`);
}
} catch (ex) {
Logger(`Error while processing ${path}`, LOG_LEVEL_NOTICE);
@@ -335,7 +519,6 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
// Logger(`${i}/${files.length}\n`, LOG_LEVEL_NOTICE, "verify-processed");
})
);
const resultArea = paneEl.createDiv({ text: "" });
new Setting(paneEl)
.setName("Check and convert non-path-obfuscated files")
.setDesc("")
+454
View File
@@ -0,0 +1,454 @@
import { $msg } from "@/common/translation";
import type {
FilePath,
FilePathWithPrefix,
LoadedEntry,
ObsidianLiveSyncSettings,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { getFileRegExp } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { isNotFoundError } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc";
import { ICHeader, ICXHeader, PSCHeader } from "@vrtmrz/livesync-commonlib/compat/common/models/fileaccess.const";
import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB";
import type { IPathService, IUIService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import { addPrefix, stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
type DatabaseMeta = LoadedEntry & {
_rawStorageType: string | null;
_legacyBodyPresent: boolean;
_revs_info?: Array<{
rev: string;
status: string;
}>;
};
export type FileDatabaseInfoCore = {
localDatabase: Pick<
LiveSyncLocalDB,
"allDocsRaw" | "findAllDocs" | "getDBEntryFromMeta" | "getDBEntry" | "localDatabase"
>;
services: {
path: Pick<IPathService, "path2id">;
UI: IUIService;
};
settings: ObsidianLiveSyncSettings;
storageAccess: Pick<
StorageAccess,
"getFileNames" | "getFilesIncludeHidden" | "isExistsIncludeHidden" | "statHidden"
>;
};
export type RevisionDatabaseInfo = {
documentId: string;
revision: string | null;
current: boolean;
deleted: boolean;
storageType: string;
storageLayout: "chunked" | "legacy-inline";
ctime: number;
mtime: number;
recordedSize: number;
revisionHistory: Array<{
revision: string;
status: string;
}>;
chunkReferences: number;
uniqueChunkReferences: number;
embeddedChunkReferences: number;
locallyStoredChunkReferences: number;
contentAvailableLocally: boolean;
chunks: Array<{
id: string;
referenceCount: number;
embedded: boolean;
storedInLocalDatabase: boolean;
localDatabaseState: "available" | "deleted" | "missing";
localDatabaseRevision: string | null;
}>;
};
export type FileDatabaseMergeBaseInfo = {
winnerRevision: string;
conflictRevision: string;
revision: string | null;
metadataAvailableLocally: boolean;
contentAvailableLocally: boolean;
missingChunkIds: string[];
unavailableSharedRevisions: string[];
};
export type FileDatabaseInfo = {
path: string;
databasePath: FilePathWithPrefix | FilePath;
storage: {
exists: boolean;
ctime?: number;
mtime?: number;
size?: number;
};
database: {
source: "local database on this device";
remoteQueried: false;
exists: boolean;
currentRevision: string | null;
conflictCount: number;
conflictRevisions: string[];
unavailableConflictRevisions: string[];
revisions: RevisionDatabaseInfo[];
mergeBases: FileDatabaseMergeBaseInfo[];
};
};
const REPORT_WARNING =
"All revisions and chunk availability below are a snapshot of this device's local database; the remote is not queried. Review the Vault-relative path, document identifier, content-derived chunk identifiers, and metadata before sharing this report. File contents are omitted.";
function toDatabasePath(path: string): FilePathWithPrefix | FilePath {
if (path.startsWith(".")) {
return addPrefix(path as FilePath, ICHeader);
}
return path as FilePath;
}
type RawDatabaseDocument = {
_id: string;
_rev?: string;
_conflicts?: string[];
_deleted?: boolean;
_revs_info?: Array<{
rev: string;
status: string;
}>;
children?: string[];
ctime?: number;
deleted?: boolean;
data?: string | string[];
eden?: Record<string, unknown>;
mtime?: number;
size?: number;
type?: string;
};
async function getLocalDatabaseMeta(
core: FileDatabaseInfoCore,
path: FilePathWithPrefix | FilePath,
options: PouchDB.Core.GetOptions
): Promise<DatabaseMeta | false> {
const documentId = await core.services.path.path2id(path);
let raw: RawDatabaseDocument;
try {
raw = await core.localDatabase.localDatabase.get(documentId, options);
} catch (error) {
if (isNotFoundError(error)) {
return false;
}
throw error;
}
if (raw.type === "leaf") {
return false;
}
if (raw.type && raw.type !== "notes" && raw.type !== "newnote" && raw.type !== "plain") {
return false;
}
const rawStorageType = raw.type ?? null;
const legacy = rawStorageType === null || rawStorageType === "notes";
const type = legacy ? "notes" : rawStorageType;
const legacyBodyPresent =
legacy && (typeof raw.data === "string" || (Array.isArray(raw.data) && raw.data.every((item) => typeof item === "string")));
return {
_id: raw._id,
_rev: raw._rev,
_conflicts: raw._conflicts,
_revs_info: raw._revs_info,
path,
data: legacyBodyPresent ? raw.data : "",
ctime: raw.ctime ?? 0,
mtime: raw.mtime ?? 0,
size: raw.size ?? 0,
children: type === "newnote" || type === "plain" ? (raw.children ?? []) : [],
datatype: type === "newnote" ? "newnote" : "plain",
deleted: raw.deleted ?? raw._deleted,
type,
eden: raw.eden ?? {},
_rawStorageType: rawStorageType,
_legacyBodyPresent: legacyBodyPresent,
} as DatabaseMeta;
}
async function collectRevisionDatabaseInfo(
core: FileDatabaseInfoCore,
meta: DatabaseMeta,
current: boolean
): Promise<RevisionDatabaseInfo> {
const legacy = meta._rawStorageType === null || meta._rawStorageType === "notes";
const children = legacy ? [] : "children" in meta ? meta.children : [];
const uniqueChildren = [...new Set(children)];
const referenceCounts = new Map<string, number>();
for (const child of children) {
referenceCounts.set(child, (referenceCounts.get(child) ?? 0) + 1);
}
const embeddedChildren = new Set(
Object.keys("eden" in meta && meta.eden ? meta.eden : {}).filter((id) => uniqueChildren.includes(id))
);
const localRows =
uniqueChildren.length === 0
? []
: (
await core.localDatabase.allDocsRaw({
keys: uniqueChildren,
include_docs: false,
})
).rows;
const localChunkStates = new Map(
localRows
.filter((row) => "value" in row)
.map(
(row) =>
[
row.key,
{
state: row.value.deleted ? ("deleted" as const) : ("available" as const),
revision: row.value.rev,
},
] as const
)
);
return {
documentId: meta._id,
revision: meta._rev ?? null,
current,
deleted: Boolean(meta.deleted ?? meta._deleted),
storageType: meta._rawStorageType ?? "absent",
storageLayout: legacy ? "legacy-inline" : "chunked",
ctime: meta.ctime,
mtime: meta.mtime,
recordedSize: meta.size,
revisionHistory: (meta._revs_info ?? []).map(({ rev, status }) => ({
revision: rev,
status,
})),
chunkReferences: children.length,
uniqueChunkReferences: uniqueChildren.length,
embeddedChunkReferences: children.filter((id) => embeddedChildren.has(id)).length,
locallyStoredChunkReferences: children.filter((id) => localChunkStates.get(id)?.state === "available").length,
contentAvailableLocally: legacy
? meta._legacyBodyPresent
: uniqueChildren.every(
(id) => embeddedChildren.has(id) || localChunkStates.get(id)?.state === "available"
),
chunks: uniqueChildren.map((id) => {
const localState = localChunkStates.get(id);
return {
id,
referenceCount: referenceCounts.get(id) ?? 0,
embedded: embeddedChildren.has(id),
storedInLocalDatabase: localState?.state === "available",
localDatabaseState: localState?.state ?? "missing",
localDatabaseRevision: localState?.revision ?? null,
};
}),
};
}
function revisionHistory(meta: DatabaseMeta): Array<{ revision: string; status: string }> {
const history = (meta._revs_info ?? []).map(({ rev, status }) => ({
revision: rev,
status,
}));
if (meta._rev && !history.some(({ revision }) => revision === meta._rev)) {
history.unshift({
revision: meta._rev,
status: "available",
});
}
return history;
}
function missingChunkIds(info: RevisionDatabaseInfo): string[] {
return info.chunks
.filter(({ embedded, localDatabaseState }) => !embedded && localDatabaseState !== "available")
.map(({ id }) => id);
}
export async function inspectFileDatabaseInfo(core: FileDatabaseInfoCore, path: string): Promise<FileDatabaseInfo> {
const storageExists = await core.storageAccess.isExistsIncludeHidden(path);
const storageStat = storageExists ? await core.storageAccess.statHidden(path) : null;
const databasePath = toDatabasePath(path);
const currentMeta = await getLocalDatabaseMeta(core, databasePath, {
conflicts: true,
revs: true,
revs_info: true,
});
const revisions: RevisionDatabaseInfo[] = [];
const conflictRevisions = currentMeta === false ? [] : (currentMeta._conflicts ?? []);
const unavailableConflictRevisions: string[] = [];
const mergeBases: FileDatabaseMergeBaseInfo[] = [];
const metadataByRevision = new Map<string, DatabaseMeta | false>();
if (currentMeta !== false && currentMeta._rev) {
metadataByRevision.set(currentMeta._rev, currentMeta);
}
const getRevisionMeta = async (revision: string): Promise<DatabaseMeta | false> => {
const cached = metadataByRevision.get(revision);
if (cached !== undefined) {
return cached;
}
const meta = await getLocalDatabaseMeta(core, databasePath, {
rev: revision,
revs: true,
revs_info: true,
});
metadataByRevision.set(revision, meta);
return meta;
};
if (currentMeta) {
revisions.push(await collectRevisionDatabaseInfo(core, currentMeta, true));
for (const revision of conflictRevisions) {
const conflictMeta = await getRevisionMeta(revision);
if (conflictMeta) {
revisions.push(await collectRevisionDatabaseInfo(core, conflictMeta, false));
const winnerHistory = revisionHistory(currentMeta);
const conflictHistory = revisionHistory(conflictMeta);
const conflictHistoryByRevision = new Map(
conflictHistory.map(({ revision: historyRevision, status }) => [historyRevision, status])
);
const sharedHistory = winnerHistory.filter(({ revision: historyRevision }) =>
conflictHistoryByRevision.has(historyRevision)
);
const sharedRevision = sharedHistory[0]?.revision ?? null;
const unavailableSharedRevisions = sharedHistory
.filter(
({ revision: historyRevision, status }) =>
status !== "available" ||
conflictHistoryByRevision.get(historyRevision) !== "available"
)
.map(({ revision: historyRevision }) => historyRevision);
const sharedMeta = sharedRevision ? await getRevisionMeta(sharedRevision) : false;
const sharedInfo = sharedMeta
? await collectRevisionDatabaseInfo(core, sharedMeta, false)
: undefined;
mergeBases.push({
winnerRevision: currentMeta._rev ?? "",
conflictRevision: revision,
revision: sharedRevision,
metadataAvailableLocally: Boolean(sharedMeta),
contentAvailableLocally: sharedInfo?.contentAvailableLocally ?? false,
missingChunkIds: sharedInfo ? missingChunkIds(sharedInfo) : [],
unavailableSharedRevisions,
});
} else {
unavailableConflictRevisions.push(revision);
}
}
}
const report: FileDatabaseInfo = {
path,
databasePath,
storage: storageStat
? {
exists: true,
ctime: storageStat.ctime,
mtime: storageStat.mtime,
size: storageStat.size,
}
: {
exists: false,
},
database: {
source: "local database on this device",
remoteQueried: false,
exists: currentMeta !== false,
currentRevision: currentMeta ? (currentMeta._rev ?? null) : null,
conflictCount: conflictRevisions.length,
conflictRevisions,
unavailableConflictRevisions,
revisions,
mergeBases,
},
};
return report;
}
export async function readFileDatabaseRevisionLocally(
core: FileDatabaseInfoCore,
path: string,
revision: string
): Promise<LoadedEntry | false> {
const databasePath = toDatabasePath(path);
const meta = await getLocalDatabaseMeta(core, databasePath, {
rev: revision,
revs: true,
revs_info: true,
});
if (!meta) {
return false;
}
const info = await collectRevisionDatabaseInfo(core, meta, false);
if (info.deleted || !info.contentAvailableLocally) {
return false;
}
return await core.localDatabase.getDBEntryFromMeta(meta, false, false);
}
export async function retryReadFileDatabaseRevision(
core: FileDatabaseInfoCore,
path: string,
revision: string
): Promise<LoadedEntry | false> {
return await core.localDatabase.getDBEntry(toDatabasePath(path), { rev: revision }, false, true, true);
}
export async function buildFileDatabaseInfoReport(core: FileDatabaseInfoCore, path: string): Promise<string> {
const report = await inspectFileDatabaseInfo(core, path);
return `${$msg(REPORT_WARNING)}
\`\`\`json
${JSON.stringify(report, null, 2)}
\`\`\``;
}
export async function copyFileDatabaseInfo(core: FileDatabaseInfoCore, path: string): Promise<boolean> {
const report = await buildFileDatabaseInfoReport(core, path);
return await core.services.UI.promptCopyToClipboard(
$msg("Database information for ${FILE}", { FILE: path }),
report
);
}
export async function collectFileDatabaseInfoPaths(core: FileDatabaseInfoCore): Promise<string[]> {
const ignorePatterns = getFileRegExp(core.settings, "syncInternalFilesIgnorePatterns");
const targetPatterns = getFileRegExp(core.settings, "syncInternalFilesTargetPatterns");
const storagePaths = core.settings.syncInternalFiles
? await core.storageAccess.getFilesIncludeHidden("/", targetPatterns, ignorePatterns)
: await core.storageAccess.getFileNames();
const databasePaths: string[] = [];
for await (const entry of core.localDatabase.findAllDocs()) {
const prefixedPath = entry.path;
if (prefixedPath.startsWith(ICXHeader) || prefixedPath.startsWith(PSCHeader)) {
continue;
}
if (!core.settings.syncInternalFiles && prefixedPath.startsWith(ICHeader)) {
continue;
}
databasePaths.push(stripAllPrefixes(prefixedPath));
}
return [...new Set([...storagePaths, ...databasePaths])].sort((left, right) =>
left < right ? -1 : left > right ? 1 : 0
);
}
export async function chooseAndCopyFileDatabaseInfo(core: FileDatabaseInfoCore): Promise<boolean> {
const paths = await collectFileDatabaseInfoPaths(core);
const selected = await core.services.UI.confirm.askSelectString($msg("Choose a file to inspect"), paths);
if (!selected) {
return false;
}
return await copyFileDatabaseInfo(core, selected);
}
@@ -0,0 +1,413 @@
import { describe, expect, it, vi } from "vitest";
import {
buildFileDatabaseInfoReport,
chooseAndCopyFileDatabaseInfo,
collectFileDatabaseInfoPaths,
inspectFileDatabaseInfo,
readFileDatabaseRevisionLocally,
retryReadFileDatabaseRevision,
} from "./fileDatabaseInfo";
async function* documents(paths: string[]) {
for (const path of paths) {
yield {
_id: `f:${path}`,
path,
};
}
}
function createCore() {
const current = {
_id: "f:note",
_rev: "3-current",
_conflicts: ["2-conflict"],
_revs_info: [
{ rev: "3-current", status: "available" },
{ rev: "2-parent", status: "missing" },
],
path: "note.md",
ctime: 100,
mtime: 300,
size: 42,
type: "plain",
datatype: "plain",
data: "secret current body",
children: ["h:private-current", "h:private-current", "h:private-embedded", "h:private-deleted"],
eden: {
"h:private-embedded": {
data: "secret embedded body",
epoch: 1,
},
},
};
const conflict = {
...current,
_rev: "2-conflict",
_conflicts: undefined,
_revs_info: [{ rev: "2-conflict", status: "available" }],
mtime: 200,
data: "secret conflict body",
children: ["h:private-missing"],
eden: {},
};
const promptCopyToClipboard = vi.fn(async (_title: string, _value: string) => true);
const askSelectString = vi.fn(async () => "db-only.md");
const core = {
settings: {
syncInternalFiles: false,
syncInternalFilesIgnorePatterns: "",
syncInternalFilesTargetPatterns: "",
},
storageAccess: {
isExistsIncludeHidden: vi.fn(async () => true),
statHidden: vi.fn(async () => ({
ctime: 90,
mtime: 310,
size: 45,
type: "file",
})),
getFileNames: vi.fn(async () => ["z.md", "a.md"]),
getFilesIncludeHidden: vi.fn(async () => [".obsidian/app.json", "a.md"]),
},
localDatabase: {
getDBEntryFromMeta: vi.fn(async (meta: typeof current) => ({
...meta,
data: ["loaded body"],
})),
getDBEntry: vi.fn(async () => current),
localDatabase: {
get: vi.fn(async (_id: string, options?: { rev?: string }) =>
options?.rev === "2-conflict" ? conflict : current
),
},
allDocsRaw: vi.fn(async ({ keys }: { keys: string[] }) => ({
rows: [
...(keys.includes("h:private-current")
? [
{
id: "h:private-current",
key: "h:private-current",
value: { rev: "1-chunk" },
},
]
: []),
...(keys.includes("h:private-deleted")
? [
{
id: "h:private-deleted",
key: "h:private-deleted",
value: { rev: "4-deleted-chunk", deleted: true },
},
]
: []),
],
})),
findAllDocs: vi.fn(() => documents(["db-only.md", "i:.obsidian/app.json", "ix:ignore", "ps:setting"])),
},
services: {
path: {
path2id: vi.fn(async () => "f:note"),
},
UI: {
promptCopyToClipboard,
confirm: {
askSelectString,
},
},
},
};
return {
askSelectString,
conflict,
core,
current,
promptCopyToClipboard,
};
}
describe("file database information", () => {
it("reports document and chunk revisions without exposing file contents", async () => {
const { core } = createCore();
const report = await buildFileDatabaseInfoReport(core as never, "note.md");
expect(report).toContain('"path": "note.md"');
expect(report).toContain('"documentId": "f:note"');
expect(report).toContain('"revision": "3-current"');
expect(report).toContain('"revision": "2-conflict"');
expect(report).toContain('"storageType": "plain"');
expect(report).toContain('"storageLayout": "chunked"');
expect(report).toContain('"contentAvailableLocally": false');
expect(report).toContain('"id": "h:private-current"');
expect(report).toContain('"localDatabaseRevision": "1-chunk"');
expect(report).toContain('"referenceCount": 2');
expect(report).toContain('"id": "h:private-embedded"');
expect(report).toContain('"embedded": true');
expect(report).toContain('"id": "h:private-deleted"');
expect(report).toContain('"localDatabaseState": "deleted"');
expect(report).toContain('"localDatabaseRevision": "4-deleted-chunk"');
expect(report).toContain('"id": "h:private-missing"');
expect(report).toContain('"localDatabaseState": "missing"');
expect(report).toContain('"localDatabaseRevision": null');
expect(report).not.toContain("secret current body");
expect(report).not.toContain("secret conflict body");
expect(report).not.toContain("secret embedded body");
});
it.each([
{
name: "notes",
document: {
type: "notes",
data: "secret legacy body",
},
storageType: "notes",
},
{
name: "an absent type",
document: {
type: undefined,
data: ["secret", " legacy body"],
},
storageType: "absent",
},
])("reports $name as legacy inline storage without exposing its body", async ({ document, storageType }) => {
const { core, current } = createCore();
core.localDatabase.localDatabase.get.mockResolvedValue({
...current,
...document,
_conflicts: [],
children: ["h:must-not-be-treated-as-a-chunk"],
} as never);
const info = await inspectFileDatabaseInfo(core as never, "note.md");
const report = await buildFileDatabaseInfoReport(core as never, "note.md");
expect(info.database.revisions).toEqual([
expect.objectContaining({
storageType,
storageLayout: "legacy-inline",
chunkReferences: 0,
contentAvailableLocally: true,
}),
]);
expect(report).not.toContain("secret legacy body");
expect(report).not.toContain("h:must-not-be-treated-as-a-chunk");
});
it("reports the exact shared ancestor and its missing chunks for each conflict", async () => {
const { conflict, core, current } = createCore();
const parent = {
...current,
_rev: "2-parent",
_conflicts: undefined,
_revs_info: [
{ rev: "2-parent", status: "available" },
{ rev: "1-root", status: "missing" },
],
children: ["h:missing-parent"],
eden: {},
};
core.localDatabase.localDatabase.get.mockImplementation(async (_id: string, options?: { rev?: string }) => {
if (options?.rev === "2-conflict") {
return {
...conflict,
_revs_info: [
{ rev: "2-conflict", status: "available" },
{ rev: "2-parent", status: "available" },
{ rev: "1-root", status: "missing" },
],
};
}
if (options?.rev === "2-parent") {
return parent;
}
return {
...current,
_revs_info: [
{ rev: "3-current", status: "available" },
{ rev: "2-parent", status: "available" },
{ rev: "1-root", status: "missing" },
],
};
});
const info = await inspectFileDatabaseInfo(core as never, "note.md");
expect(info.database.mergeBases).toEqual([
{
winnerRevision: "3-current",
conflictRevision: "2-conflict",
revision: "2-parent",
metadataAvailableLocally: true,
contentAvailableLocally: false,
missingChunkIds: ["h:missing-parent"],
unavailableSharedRevisions: ["1-root"],
},
]);
});
it("does not decode a revision whose chunks are not all available locally", async () => {
const { core } = createCore();
await expect(readFileDatabaseRevisionLocally(core as never, "note.md", "3-current")).resolves.toBe(false);
expect(core.localDatabase.getDBEntryFromMeta).not.toHaveBeenCalled();
});
it("decodes an exact revision after confirming that every chunk is available locally", async () => {
const { core, current } = createCore();
core.localDatabase.localDatabase.get.mockResolvedValue({
...current,
children: ["h:available"],
eden: {},
} as never);
core.localDatabase.allDocsRaw.mockResolvedValue({
rows: [
{
id: "h:available",
key: "h:available",
value: { rev: "1-available" },
},
],
});
await expect(readFileDatabaseRevisionLocally(core as never, "note.md", "3-current")).resolves.toEqual(
expect.objectContaining({
data: ["loaded body"],
})
);
expect(core.localDatabase.getDBEntryFromMeta).toHaveBeenCalledWith(
expect.objectContaining({
_rev: "3-current",
}),
false,
false
);
});
it("retries an exact revision through the configured chunk retrieval path", async () => {
const { core } = createCore();
await retryReadFileDatabaseRevision(core as never, "note.md", "2-conflict");
expect(core.localDatabase.getDBEntry).toHaveBeenCalledWith(
"note.md",
{ rev: "2-conflict" },
false,
true,
true
);
});
it("reports the exact revision as locally available after retry recovers its missing chunk", async () => {
const { conflict, core } = createCore();
let recovered = false;
core.localDatabase.getDBEntry.mockImplementation(async () => {
recovered = true;
return conflict as never;
});
core.localDatabase.allDocsRaw.mockImplementation(async ({ keys }: { keys: string[] }) => ({
rows:
recovered && keys.includes("h:private-missing")
? [
{
id: "h:private-missing",
key: "h:private-missing",
value: { rev: "1-recovered" },
},
]
: [],
}));
await expect(
retryReadFileDatabaseRevision(core as never, "note.md", "2-conflict")
).resolves.not.toBe(false);
const information = await inspectFileDatabaseInfo(core as never, "note.md");
expect(
information.database.revisions.find(({ revision }) => revision === "2-conflict")
).toEqual(
expect.objectContaining({
contentAvailableLocally: true,
chunks: [
expect.objectContaining({
id: "h:private-missing",
localDatabaseState: "available",
localDatabaseRevision: "1-recovered",
}),
],
})
);
});
it("keeps the exact revision identifiers when conflict metadata is unavailable", async () => {
const { conflict, core, current } = createCore();
core.localDatabase.localDatabase.get.mockImplementation(async (_id: string, options?: { rev?: string }) => {
if (options?.rev === "2-unavailable") {
throw Object.assign(new Error("missing"), { status: 404 });
}
if (options?.rev === "2-conflict") {
return conflict;
}
return { ...current, _conflicts: ["2-conflict", "2-unavailable"] };
});
const report = await buildFileDatabaseInfoReport(core as never, "note.md");
expect(report).toContain('"conflictRevisions"');
expect(report).toContain('"2-conflict"');
expect(report).toContain('"2-unavailable"');
expect(report).toContain('"unavailableConflictRevisions"');
});
it("reads an existing local document even when current synchronisation filters exclude its path", async () => {
const { core, current } = createCore();
core.services.path.path2id.mockResolvedValue("f:ignored");
core.localDatabase.localDatabase.get.mockResolvedValue({
...current,
_id: "f:ignored",
_rev: "5-ignored",
_conflicts: [],
_revs_info: [],
path: "ignored.md",
ctime: 10,
mtime: 20,
size: 30,
children: [],
});
const report = await buildFileDatabaseInfoReport(core as never, "ignored.md");
expect(report).toContain('"exists": true');
expect(report).toContain('"documentId": "f:ignored"');
expect(report).toContain('"revision": "5-ignored"');
});
it("offers the union of storage and database paths and excludes inactive internal namespaces", async () => {
const { core } = createCore();
await expect(collectFileDatabaseInfoPaths(core as never)).resolves.toEqual(["a.md", "db-only.md", "z.md"]);
core.settings.syncInternalFiles = true;
await expect(collectFileDatabaseInfoPaths(core as never)).resolves.toEqual([
".obsidian/app.json",
"a.md",
"db-only.md",
]);
});
it("copies the selected file report through the existing copy dialogue", async () => {
const { askSelectString, core, promptCopyToClipboard } = createCore();
await expect(chooseAndCopyFileDatabaseInfo(core as never)).resolves.toBe(true);
expect(askSelectString).toHaveBeenCalledWith("Choose a file to inspect", ["a.md", "db-only.md", "z.md"]);
expect(promptCopyToClipboard).toHaveBeenCalledWith(
"Database information for db-only.md",
expect.stringContaining('"path": "db-only.md"')
);
});
});
+109
View File
@@ -0,0 +1,109 @@
import type { LoadedEntry } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { createBlob, isDocContentSame, readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
import type { IFileHandler } from "@vrtmrz/livesync-commonlib/compat/interfaces/FileHandler";
import {
inspectFileDatabaseInfo,
readFileDatabaseRevisionLocally,
type FileDatabaseInfo,
type FileDatabaseInfoCore,
type RevisionDatabaseInfo,
} from "./fileDatabaseInfo";
export type FileRepairCore = FileDatabaseInfoCore & {
fileHandler: Pick<IFileHandler, "deleteRevisionFromDB">;
storageAccess: FileDatabaseInfoCore["storageAccess"] & Pick<StorageAccess, "readHiddenFileBinary">;
};
export type FileRepairRevision = {
role: "winner" | "conflict";
metadata: RevisionDatabaseInfo;
contentReadable: boolean;
contentMatchesStorage: boolean | null;
loadedEntry: LoadedEntry | false;
};
export type FileRepairInspection = {
information: FileDatabaseInfo;
revisions: FileRepairRevision[];
requiresAttention: boolean;
};
export type DiscardUnreadableRevisionResult =
| "discarded"
| "failed"
| "no-longer-live"
| "revision-is-readable";
export async function inspectFileRepair(core: FileRepairCore, path: string): Promise<FileRepairInspection> {
const information = await inspectFileDatabaseInfo(core, path);
const storageContent = information.storage.exists
? createBlob(await core.storageAccess.readHiddenFileBinary(path))
: undefined;
const revisions: FileRepairRevision[] = [];
for (const metadata of information.database.revisions) {
const loadedEntry =
metadata.deleted || !metadata.contentAvailableLocally
? false
: await readFileDatabaseRevisionLocally(core, path, metadata.revision ?? "");
const contentReadable = metadata.deleted || loadedEntry !== false;
const contentMatchesStorage =
storageContent && loadedEntry !== false
? await isDocContentSame(storageContent, readAsBlob(loadedEntry))
: null;
revisions.push({
role: metadata.current ? "winner" : "conflict",
metadata,
contentReadable,
contentMatchesStorage,
loadedEntry,
});
}
const winner = revisions.find(({ role }) => role === "winner");
const databaseAndStorageDiffer =
information.storage.exists !== information.database.exists ||
(information.storage.exists &&
winner !== undefined &&
(winner.metadata.deleted || winner.contentMatchesStorage === false)) ||
(!information.storage.exists && winner !== undefined && !winner.metadata.deleted);
const unreadableLiveRevision =
information.database.unavailableConflictRevisions.length > 0 ||
revisions.some(({ contentReadable }) => !contentReadable);
const requiresAttention =
databaseAndStorageDiffer ||
information.database.conflictCount > 0 ||
unreadableLiveRevision ||
(information.database.exists && winner === undefined);
return {
information,
revisions,
requiresAttention,
};
}
export async function discardUnreadableLiveRevision(
core: FileRepairCore,
path: string,
revision: string
): Promise<DiscardUnreadableRevisionResult> {
const latest = await inspectFileDatabaseInfo(core, path);
const liveRevisions = [
latest.database.currentRevision,
...latest.database.conflictRevisions,
].filter((candidate): candidate is string => candidate !== null);
if (!liveRevisions.includes(revision)) {
return "no-longer-live";
}
const metadata = latest.database.revisions.find((candidate) => candidate.revision === revision);
const metadataUnavailable = latest.database.unavailableConflictRevisions.includes(revision);
if (!metadataUnavailable && (metadata?.deleted || metadata?.contentAvailableLocally)) {
return "revision-is-readable";
}
const deleted = await core.fileHandler.deleteRevisionFromDB(latest.databasePath, revision);
return deleted ? "discarded" : "failed";
}
+172
View File
@@ -0,0 +1,172 @@
import { describe, expect, it, vi } from "vitest";
import {
discardUnreadableLiveRevision,
inspectFileRepair,
} from "./fileRepair";
function createCore() {
const current = {
_id: "f:note",
_rev: "3-current",
_conflicts: ["2-conflict"],
_revs_info: [{ rev: "3-current", status: "available" }],
path: "note.md",
ctime: 1,
mtime: 3,
size: 7,
type: "plain",
children: ["h:current"],
eden: {},
};
const conflict = {
...current,
_rev: "2-conflict",
_conflicts: undefined,
_revs_info: [{ rev: "2-conflict", status: "available" }],
mtime: 2,
children: ["h:missing-conflict"],
};
const deleteRevisionFromDB = vi.fn(async () => true);
const core = {
settings: {
syncInternalFiles: false,
syncInternalFilesIgnorePatterns: "",
syncInternalFilesTargetPatterns: "",
},
storageAccess: {
isExistsIncludeHidden: vi.fn(async () => true),
statHidden: vi.fn(async () => ({
ctime: 1,
mtime: 3,
size: 7,
type: "file",
})),
readHiddenFileBinary: vi.fn(async () => new TextEncoder().encode("current").buffer),
getFileNames: vi.fn(async () => ["note.md"]),
getFilesIncludeHidden: vi.fn(async () => ["note.md"]),
},
localDatabase: {
localDatabase: {
get: vi.fn(async (_id: string, options?: { rev?: string }) =>
options?.rev === "2-conflict" ? conflict : current
),
},
allDocsRaw: vi.fn(async ({ keys }: { keys: string[] }) => ({
rows: keys.includes("h:current")
? [
{
id: "h:current",
key: "h:current",
value: { rev: "1-current" },
},
]
: [],
})),
getDBEntryFromMeta: vi.fn(async (meta: typeof current) => ({
...meta,
data: [meta._rev === "3-current" ? "current" : "conflict"],
})),
getDBEntry: vi.fn(async () => false),
findAllDocs: vi.fn(async function* () {
yield current;
}),
},
fileHandler: {
deleteRevisionFromDB,
},
services: {
path: {
path2id: vi.fn(async () => "f:note"),
},
UI: {
confirm: {},
},
},
};
return {
conflict,
core,
current,
deleteRevisionFromDB,
};
}
describe("file repair inspection", () => {
it("shows the winner and every conflict revision independently", async () => {
const { core } = createCore();
const inspection = await inspectFileRepair(core as never, "note.md");
expect(inspection.revisions).toEqual([
expect.objectContaining({
role: "winner",
contentReadable: true,
contentMatchesStorage: true,
metadata: expect.objectContaining({
revision: "3-current",
}),
}),
expect.objectContaining({
role: "conflict",
contentReadable: false,
contentMatchesStorage: null,
metadata: expect.objectContaining({
revision: "2-conflict",
}),
}),
]);
expect(inspection.requiresAttention).toBe(true);
});
it("rechecks liveness and readability before discarding an exact revision", async () => {
const { core, deleteRevisionFromDB } = createCore();
await expect(
discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")
).resolves.toBe("discarded");
await expect(
discardUnreadableLiveRevision(core as never, "note.md", "3-current")
).resolves.toBe("revision-is-readable");
expect(deleteRevisionFromDB).toHaveBeenCalledOnce();
expect(deleteRevisionFromDB).toHaveBeenCalledWith("note.md", "2-conflict");
});
it("allows an exact unreadable generation-one winner to be discarded explicitly", async () => {
const { core, current, deleteRevisionFromDB } = createCore();
current._rev = "1-root";
current._conflicts = [];
current.children = ["h:missing-root"];
core.localDatabase.allDocsRaw.mockResolvedValue({ rows: [] });
const inspection = await inspectFileRepair(core as never, "note.md");
expect(inspection.revisions).toEqual([
expect.objectContaining({
role: "winner",
contentReadable: false,
metadata: expect.objectContaining({
revision: "1-root",
}),
}),
]);
await expect(
discardUnreadableLiveRevision(core as never, "note.md", "1-root")
).resolves.toBe("discarded");
expect(deleteRevisionFromDB).toHaveBeenCalledWith("note.md", "1-root");
});
it("refuses to discard a revision which stopped being a live leaf", async () => {
const { core, current, deleteRevisionFromDB } = createCore();
core.localDatabase.localDatabase.get.mockResolvedValue({
...current,
_conflicts: [],
});
await expect(
discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")
).resolves.toBe("no-longer-live");
expect(deleteRevisionFromDB).not.toHaveBeenCalled();
});
});