mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-08-27 22:07:07 +00:00
Keep unreadable revisions for explicit repair
This commit is contained in:
vorotamoroz
@@ -0,0 +1,454 @@
|
||||
import { $msg } from "@/common/translation";
|
||||
import type {
|
||||
FilePath,
|
||||
FilePathWithPrefix,
|
||||
LoadedEntry,
|
||||
ObsidianLiveSyncSettings,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { getFileRegExp } from "@vrtmrz/livesync-commonlib/compat/common/utils";
|
||||
import { isNotFoundError } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc";
|
||||
import { ICHeader, ICXHeader, PSCHeader } from "@vrtmrz/livesync-commonlib/compat/common/models/fileaccess.const";
|
||||
import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
|
||||
import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB";
|
||||
import type { IPathService, IUIService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
|
||||
import { addPrefix, stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
|
||||
|
||||
type DatabaseMeta = LoadedEntry & {
|
||||
_rawStorageType: string | null;
|
||||
_legacyBodyPresent: boolean;
|
||||
_revs_info?: Array<{
|
||||
rev: string;
|
||||
status: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
export type FileDatabaseInfoCore = {
|
||||
localDatabase: Pick<
|
||||
LiveSyncLocalDB,
|
||||
"allDocsRaw" | "findAllDocs" | "getDBEntryFromMeta" | "getDBEntry" | "localDatabase"
|
||||
>;
|
||||
services: {
|
||||
path: Pick<IPathService, "path2id">;
|
||||
UI: IUIService;
|
||||
};
|
||||
settings: ObsidianLiveSyncSettings;
|
||||
storageAccess: Pick<
|
||||
StorageAccess,
|
||||
"getFileNames" | "getFilesIncludeHidden" | "isExistsIncludeHidden" | "statHidden"
|
||||
>;
|
||||
};
|
||||
|
||||
export type RevisionDatabaseInfo = {
|
||||
documentId: string;
|
||||
revision: string | null;
|
||||
current: boolean;
|
||||
deleted: boolean;
|
||||
storageType: string;
|
||||
storageLayout: "chunked" | "legacy-inline";
|
||||
ctime: number;
|
||||
mtime: number;
|
||||
recordedSize: number;
|
||||
revisionHistory: Array<{
|
||||
revision: string;
|
||||
status: string;
|
||||
}>;
|
||||
chunkReferences: number;
|
||||
uniqueChunkReferences: number;
|
||||
embeddedChunkReferences: number;
|
||||
locallyStoredChunkReferences: number;
|
||||
contentAvailableLocally: boolean;
|
||||
chunks: Array<{
|
||||
id: string;
|
||||
referenceCount: number;
|
||||
embedded: boolean;
|
||||
storedInLocalDatabase: boolean;
|
||||
localDatabaseState: "available" | "deleted" | "missing";
|
||||
localDatabaseRevision: string | null;
|
||||
}>;
|
||||
};
|
||||
|
||||
export type FileDatabaseMergeBaseInfo = {
|
||||
winnerRevision: string;
|
||||
conflictRevision: string;
|
||||
revision: string | null;
|
||||
metadataAvailableLocally: boolean;
|
||||
contentAvailableLocally: boolean;
|
||||
missingChunkIds: string[];
|
||||
unavailableSharedRevisions: string[];
|
||||
};
|
||||
|
||||
export type FileDatabaseInfo = {
|
||||
path: string;
|
||||
databasePath: FilePathWithPrefix | FilePath;
|
||||
storage: {
|
||||
exists: boolean;
|
||||
ctime?: number;
|
||||
mtime?: number;
|
||||
size?: number;
|
||||
};
|
||||
database: {
|
||||
source: "local database on this device";
|
||||
remoteQueried: false;
|
||||
exists: boolean;
|
||||
currentRevision: string | null;
|
||||
conflictCount: number;
|
||||
conflictRevisions: string[];
|
||||
unavailableConflictRevisions: string[];
|
||||
revisions: RevisionDatabaseInfo[];
|
||||
mergeBases: FileDatabaseMergeBaseInfo[];
|
||||
};
|
||||
};
|
||||
|
||||
const REPORT_WARNING =
|
||||
"All revisions and chunk availability below are a snapshot of this device's local database; the remote is not queried. Review the Vault-relative path, document identifier, content-derived chunk identifiers, and metadata before sharing this report. File contents are omitted.";
|
||||
|
||||
function toDatabasePath(path: string): FilePathWithPrefix | FilePath {
|
||||
if (path.startsWith(".")) {
|
||||
return addPrefix(path as FilePath, ICHeader);
|
||||
}
|
||||
return path as FilePath;
|
||||
}
|
||||
|
||||
type RawDatabaseDocument = {
|
||||
_id: string;
|
||||
_rev?: string;
|
||||
_conflicts?: string[];
|
||||
_deleted?: boolean;
|
||||
_revs_info?: Array<{
|
||||
rev: string;
|
||||
status: string;
|
||||
}>;
|
||||
children?: string[];
|
||||
ctime?: number;
|
||||
deleted?: boolean;
|
||||
data?: string | string[];
|
||||
eden?: Record<string, unknown>;
|
||||
mtime?: number;
|
||||
size?: number;
|
||||
type?: string;
|
||||
};
|
||||
|
||||
async function getLocalDatabaseMeta(
|
||||
core: FileDatabaseInfoCore,
|
||||
path: FilePathWithPrefix | FilePath,
|
||||
options: PouchDB.Core.GetOptions
|
||||
): Promise<DatabaseMeta | false> {
|
||||
const documentId = await core.services.path.path2id(path);
|
||||
let raw: RawDatabaseDocument;
|
||||
try {
|
||||
raw = await core.localDatabase.localDatabase.get(documentId, options);
|
||||
} catch (error) {
|
||||
if (isNotFoundError(error)) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (raw.type === "leaf") {
|
||||
return false;
|
||||
}
|
||||
if (raw.type && raw.type !== "notes" && raw.type !== "newnote" && raw.type !== "plain") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const rawStorageType = raw.type ?? null;
|
||||
const legacy = rawStorageType === null || rawStorageType === "notes";
|
||||
const type = legacy ? "notes" : rawStorageType;
|
||||
const legacyBodyPresent =
|
||||
legacy && (typeof raw.data === "string" || (Array.isArray(raw.data) && raw.data.every((item) => typeof item === "string")));
|
||||
return {
|
||||
_id: raw._id,
|
||||
_rev: raw._rev,
|
||||
_conflicts: raw._conflicts,
|
||||
_revs_info: raw._revs_info,
|
||||
path,
|
||||
data: legacyBodyPresent ? raw.data : "",
|
||||
ctime: raw.ctime ?? 0,
|
||||
mtime: raw.mtime ?? 0,
|
||||
size: raw.size ?? 0,
|
||||
children: type === "newnote" || type === "plain" ? (raw.children ?? []) : [],
|
||||
datatype: type === "newnote" ? "newnote" : "plain",
|
||||
deleted: raw.deleted ?? raw._deleted,
|
||||
type,
|
||||
eden: raw.eden ?? {},
|
||||
_rawStorageType: rawStorageType,
|
||||
_legacyBodyPresent: legacyBodyPresent,
|
||||
} as DatabaseMeta;
|
||||
}
|
||||
|
||||
async function collectRevisionDatabaseInfo(
|
||||
core: FileDatabaseInfoCore,
|
||||
meta: DatabaseMeta,
|
||||
current: boolean
|
||||
): Promise<RevisionDatabaseInfo> {
|
||||
const legacy = meta._rawStorageType === null || meta._rawStorageType === "notes";
|
||||
const children = legacy ? [] : "children" in meta ? meta.children : [];
|
||||
const uniqueChildren = [...new Set(children)];
|
||||
const referenceCounts = new Map<string, number>();
|
||||
for (const child of children) {
|
||||
referenceCounts.set(child, (referenceCounts.get(child) ?? 0) + 1);
|
||||
}
|
||||
const embeddedChildren = new Set(
|
||||
Object.keys("eden" in meta && meta.eden ? meta.eden : {}).filter((id) => uniqueChildren.includes(id))
|
||||
);
|
||||
const localRows =
|
||||
uniqueChildren.length === 0
|
||||
? []
|
||||
: (
|
||||
await core.localDatabase.allDocsRaw({
|
||||
keys: uniqueChildren,
|
||||
include_docs: false,
|
||||
})
|
||||
).rows;
|
||||
const localChunkStates = new Map(
|
||||
localRows
|
||||
.filter((row) => "value" in row)
|
||||
.map(
|
||||
(row) =>
|
||||
[
|
||||
row.key,
|
||||
{
|
||||
state: row.value.deleted ? ("deleted" as const) : ("available" as const),
|
||||
revision: row.value.rev,
|
||||
},
|
||||
] as const
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
documentId: meta._id,
|
||||
revision: meta._rev ?? null,
|
||||
current,
|
||||
deleted: Boolean(meta.deleted ?? meta._deleted),
|
||||
storageType: meta._rawStorageType ?? "absent",
|
||||
storageLayout: legacy ? "legacy-inline" : "chunked",
|
||||
ctime: meta.ctime,
|
||||
mtime: meta.mtime,
|
||||
recordedSize: meta.size,
|
||||
revisionHistory: (meta._revs_info ?? []).map(({ rev, status }) => ({
|
||||
revision: rev,
|
||||
status,
|
||||
})),
|
||||
chunkReferences: children.length,
|
||||
uniqueChunkReferences: uniqueChildren.length,
|
||||
embeddedChunkReferences: children.filter((id) => embeddedChildren.has(id)).length,
|
||||
locallyStoredChunkReferences: children.filter((id) => localChunkStates.get(id)?.state === "available").length,
|
||||
contentAvailableLocally: legacy
|
||||
? meta._legacyBodyPresent
|
||||
: uniqueChildren.every(
|
||||
(id) => embeddedChildren.has(id) || localChunkStates.get(id)?.state === "available"
|
||||
),
|
||||
chunks: uniqueChildren.map((id) => {
|
||||
const localState = localChunkStates.get(id);
|
||||
return {
|
||||
id,
|
||||
referenceCount: referenceCounts.get(id) ?? 0,
|
||||
embedded: embeddedChildren.has(id),
|
||||
storedInLocalDatabase: localState?.state === "available",
|
||||
localDatabaseState: localState?.state ?? "missing",
|
||||
localDatabaseRevision: localState?.revision ?? null,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function revisionHistory(meta: DatabaseMeta): Array<{ revision: string; status: string }> {
|
||||
const history = (meta._revs_info ?? []).map(({ rev, status }) => ({
|
||||
revision: rev,
|
||||
status,
|
||||
}));
|
||||
if (meta._rev && !history.some(({ revision }) => revision === meta._rev)) {
|
||||
history.unshift({
|
||||
revision: meta._rev,
|
||||
status: "available",
|
||||
});
|
||||
}
|
||||
return history;
|
||||
}
|
||||
|
||||
function missingChunkIds(info: RevisionDatabaseInfo): string[] {
|
||||
return info.chunks
|
||||
.filter(({ embedded, localDatabaseState }) => !embedded && localDatabaseState !== "available")
|
||||
.map(({ id }) => id);
|
||||
}
|
||||
|
||||
export async function inspectFileDatabaseInfo(core: FileDatabaseInfoCore, path: string): Promise<FileDatabaseInfo> {
|
||||
const storageExists = await core.storageAccess.isExistsIncludeHidden(path);
|
||||
const storageStat = storageExists ? await core.storageAccess.statHidden(path) : null;
|
||||
const databasePath = toDatabasePath(path);
|
||||
const currentMeta = await getLocalDatabaseMeta(core, databasePath, {
|
||||
conflicts: true,
|
||||
revs: true,
|
||||
revs_info: true,
|
||||
});
|
||||
|
||||
const revisions: RevisionDatabaseInfo[] = [];
|
||||
const conflictRevisions = currentMeta === false ? [] : (currentMeta._conflicts ?? []);
|
||||
const unavailableConflictRevisions: string[] = [];
|
||||
const mergeBases: FileDatabaseMergeBaseInfo[] = [];
|
||||
const metadataByRevision = new Map<string, DatabaseMeta | false>();
|
||||
if (currentMeta !== false && currentMeta._rev) {
|
||||
metadataByRevision.set(currentMeta._rev, currentMeta);
|
||||
}
|
||||
const getRevisionMeta = async (revision: string): Promise<DatabaseMeta | false> => {
|
||||
const cached = metadataByRevision.get(revision);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
const meta = await getLocalDatabaseMeta(core, databasePath, {
|
||||
rev: revision,
|
||||
revs: true,
|
||||
revs_info: true,
|
||||
});
|
||||
metadataByRevision.set(revision, meta);
|
||||
return meta;
|
||||
};
|
||||
|
||||
if (currentMeta) {
|
||||
revisions.push(await collectRevisionDatabaseInfo(core, currentMeta, true));
|
||||
for (const revision of conflictRevisions) {
|
||||
const conflictMeta = await getRevisionMeta(revision);
|
||||
if (conflictMeta) {
|
||||
revisions.push(await collectRevisionDatabaseInfo(core, conflictMeta, false));
|
||||
const winnerHistory = revisionHistory(currentMeta);
|
||||
const conflictHistory = revisionHistory(conflictMeta);
|
||||
const conflictHistoryByRevision = new Map(
|
||||
conflictHistory.map(({ revision: historyRevision, status }) => [historyRevision, status])
|
||||
);
|
||||
const sharedHistory = winnerHistory.filter(({ revision: historyRevision }) =>
|
||||
conflictHistoryByRevision.has(historyRevision)
|
||||
);
|
||||
const sharedRevision = sharedHistory[0]?.revision ?? null;
|
||||
const unavailableSharedRevisions = sharedHistory
|
||||
.filter(
|
||||
({ revision: historyRevision, status }) =>
|
||||
status !== "available" ||
|
||||
conflictHistoryByRevision.get(historyRevision) !== "available"
|
||||
)
|
||||
.map(({ revision: historyRevision }) => historyRevision);
|
||||
const sharedMeta = sharedRevision ? await getRevisionMeta(sharedRevision) : false;
|
||||
const sharedInfo = sharedMeta
|
||||
? await collectRevisionDatabaseInfo(core, sharedMeta, false)
|
||||
: undefined;
|
||||
mergeBases.push({
|
||||
winnerRevision: currentMeta._rev ?? "",
|
||||
conflictRevision: revision,
|
||||
revision: sharedRevision,
|
||||
metadataAvailableLocally: Boolean(sharedMeta),
|
||||
contentAvailableLocally: sharedInfo?.contentAvailableLocally ?? false,
|
||||
missingChunkIds: sharedInfo ? missingChunkIds(sharedInfo) : [],
|
||||
unavailableSharedRevisions,
|
||||
});
|
||||
} else {
|
||||
unavailableConflictRevisions.push(revision);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const report: FileDatabaseInfo = {
|
||||
path,
|
||||
databasePath,
|
||||
storage: storageStat
|
||||
? {
|
||||
exists: true,
|
||||
ctime: storageStat.ctime,
|
||||
mtime: storageStat.mtime,
|
||||
size: storageStat.size,
|
||||
}
|
||||
: {
|
||||
exists: false,
|
||||
},
|
||||
database: {
|
||||
source: "local database on this device",
|
||||
remoteQueried: false,
|
||||
exists: currentMeta !== false,
|
||||
currentRevision: currentMeta ? (currentMeta._rev ?? null) : null,
|
||||
conflictCount: conflictRevisions.length,
|
||||
conflictRevisions,
|
||||
unavailableConflictRevisions,
|
||||
revisions,
|
||||
mergeBases,
|
||||
},
|
||||
};
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
export async function readFileDatabaseRevisionLocally(
|
||||
core: FileDatabaseInfoCore,
|
||||
path: string,
|
||||
revision: string
|
||||
): Promise<LoadedEntry | false> {
|
||||
const databasePath = toDatabasePath(path);
|
||||
const meta = await getLocalDatabaseMeta(core, databasePath, {
|
||||
rev: revision,
|
||||
revs: true,
|
||||
revs_info: true,
|
||||
});
|
||||
if (!meta) {
|
||||
return false;
|
||||
}
|
||||
const info = await collectRevisionDatabaseInfo(core, meta, false);
|
||||
if (info.deleted || !info.contentAvailableLocally) {
|
||||
return false;
|
||||
}
|
||||
return await core.localDatabase.getDBEntryFromMeta(meta, false, false);
|
||||
}
|
||||
|
||||
export async function retryReadFileDatabaseRevision(
|
||||
core: FileDatabaseInfoCore,
|
||||
path: string,
|
||||
revision: string
|
||||
): Promise<LoadedEntry | false> {
|
||||
return await core.localDatabase.getDBEntry(toDatabasePath(path), { rev: revision }, false, true, true);
|
||||
}
|
||||
|
||||
export async function buildFileDatabaseInfoReport(core: FileDatabaseInfoCore, path: string): Promise<string> {
|
||||
const report = await inspectFileDatabaseInfo(core, path);
|
||||
return `${$msg(REPORT_WARNING)}
|
||||
|
||||
\`\`\`json
|
||||
${JSON.stringify(report, null, 2)}
|
||||
\`\`\``;
|
||||
}
|
||||
|
||||
export async function copyFileDatabaseInfo(core: FileDatabaseInfoCore, path: string): Promise<boolean> {
|
||||
const report = await buildFileDatabaseInfoReport(core, path);
|
||||
return await core.services.UI.promptCopyToClipboard(
|
||||
$msg("Database information for ${FILE}", { FILE: path }),
|
||||
report
|
||||
);
|
||||
}
|
||||
|
||||
export async function collectFileDatabaseInfoPaths(core: FileDatabaseInfoCore): Promise<string[]> {
|
||||
const ignorePatterns = getFileRegExp(core.settings, "syncInternalFilesIgnorePatterns");
|
||||
const targetPatterns = getFileRegExp(core.settings, "syncInternalFilesTargetPatterns");
|
||||
const storagePaths = core.settings.syncInternalFiles
|
||||
? await core.storageAccess.getFilesIncludeHidden("/", targetPatterns, ignorePatterns)
|
||||
: await core.storageAccess.getFileNames();
|
||||
const databasePaths: string[] = [];
|
||||
|
||||
for await (const entry of core.localDatabase.findAllDocs()) {
|
||||
const prefixedPath = entry.path;
|
||||
if (prefixedPath.startsWith(ICXHeader) || prefixedPath.startsWith(PSCHeader)) {
|
||||
continue;
|
||||
}
|
||||
if (!core.settings.syncInternalFiles && prefixedPath.startsWith(ICHeader)) {
|
||||
continue;
|
||||
}
|
||||
databasePaths.push(stripAllPrefixes(prefixedPath));
|
||||
}
|
||||
|
||||
return [...new Set([...storagePaths, ...databasePaths])].sort((left, right) =>
|
||||
left < right ? -1 : left > right ? 1 : 0
|
||||
);
|
||||
}
|
||||
|
||||
export async function chooseAndCopyFileDatabaseInfo(core: FileDatabaseInfoCore): Promise<boolean> {
|
||||
const paths = await collectFileDatabaseInfoPaths(core);
|
||||
const selected = await core.services.UI.confirm.askSelectString($msg("Choose a file to inspect"), paths);
|
||||
if (!selected) {
|
||||
return false;
|
||||
}
|
||||
return await copyFileDatabaseInfo(core, selected);
|
||||
}
|
||||
@@ -0,0 +1,413 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildFileDatabaseInfoReport,
|
||||
chooseAndCopyFileDatabaseInfo,
|
||||
collectFileDatabaseInfoPaths,
|
||||
inspectFileDatabaseInfo,
|
||||
readFileDatabaseRevisionLocally,
|
||||
retryReadFileDatabaseRevision,
|
||||
} from "./fileDatabaseInfo";
|
||||
|
||||
async function* documents(paths: string[]) {
|
||||
for (const path of paths) {
|
||||
yield {
|
||||
_id: `f:${path}`,
|
||||
path,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function createCore() {
|
||||
const current = {
|
||||
_id: "f:note",
|
||||
_rev: "3-current",
|
||||
_conflicts: ["2-conflict"],
|
||||
_revs_info: [
|
||||
{ rev: "3-current", status: "available" },
|
||||
{ rev: "2-parent", status: "missing" },
|
||||
],
|
||||
path: "note.md",
|
||||
ctime: 100,
|
||||
mtime: 300,
|
||||
size: 42,
|
||||
type: "plain",
|
||||
datatype: "plain",
|
||||
data: "secret current body",
|
||||
children: ["h:private-current", "h:private-current", "h:private-embedded", "h:private-deleted"],
|
||||
eden: {
|
||||
"h:private-embedded": {
|
||||
data: "secret embedded body",
|
||||
epoch: 1,
|
||||
},
|
||||
},
|
||||
};
|
||||
const conflict = {
|
||||
...current,
|
||||
_rev: "2-conflict",
|
||||
_conflicts: undefined,
|
||||
_revs_info: [{ rev: "2-conflict", status: "available" }],
|
||||
mtime: 200,
|
||||
data: "secret conflict body",
|
||||
children: ["h:private-missing"],
|
||||
eden: {},
|
||||
};
|
||||
const promptCopyToClipboard = vi.fn(async (_title: string, _value: string) => true);
|
||||
const askSelectString = vi.fn(async () => "db-only.md");
|
||||
const core = {
|
||||
settings: {
|
||||
syncInternalFiles: false,
|
||||
syncInternalFilesIgnorePatterns: "",
|
||||
syncInternalFilesTargetPatterns: "",
|
||||
},
|
||||
storageAccess: {
|
||||
isExistsIncludeHidden: vi.fn(async () => true),
|
||||
statHidden: vi.fn(async () => ({
|
||||
ctime: 90,
|
||||
mtime: 310,
|
||||
size: 45,
|
||||
type: "file",
|
||||
})),
|
||||
getFileNames: vi.fn(async () => ["z.md", "a.md"]),
|
||||
getFilesIncludeHidden: vi.fn(async () => [".obsidian/app.json", "a.md"]),
|
||||
},
|
||||
localDatabase: {
|
||||
getDBEntryFromMeta: vi.fn(async (meta: typeof current) => ({
|
||||
...meta,
|
||||
data: ["loaded body"],
|
||||
})),
|
||||
getDBEntry: vi.fn(async () => current),
|
||||
localDatabase: {
|
||||
get: vi.fn(async (_id: string, options?: { rev?: string }) =>
|
||||
options?.rev === "2-conflict" ? conflict : current
|
||||
),
|
||||
},
|
||||
allDocsRaw: vi.fn(async ({ keys }: { keys: string[] }) => ({
|
||||
rows: [
|
||||
...(keys.includes("h:private-current")
|
||||
? [
|
||||
{
|
||||
id: "h:private-current",
|
||||
key: "h:private-current",
|
||||
value: { rev: "1-chunk" },
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(keys.includes("h:private-deleted")
|
||||
? [
|
||||
{
|
||||
id: "h:private-deleted",
|
||||
key: "h:private-deleted",
|
||||
value: { rev: "4-deleted-chunk", deleted: true },
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
})),
|
||||
findAllDocs: vi.fn(() => documents(["db-only.md", "i:.obsidian/app.json", "ix:ignore", "ps:setting"])),
|
||||
},
|
||||
services: {
|
||||
path: {
|
||||
path2id: vi.fn(async () => "f:note"),
|
||||
},
|
||||
UI: {
|
||||
promptCopyToClipboard,
|
||||
confirm: {
|
||||
askSelectString,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return {
|
||||
askSelectString,
|
||||
conflict,
|
||||
core,
|
||||
current,
|
||||
promptCopyToClipboard,
|
||||
};
|
||||
}
|
||||
|
||||
describe("file database information", () => {
|
||||
it("reports document and chunk revisions without exposing file contents", async () => {
|
||||
const { core } = createCore();
|
||||
|
||||
const report = await buildFileDatabaseInfoReport(core as never, "note.md");
|
||||
|
||||
expect(report).toContain('"path": "note.md"');
|
||||
expect(report).toContain('"documentId": "f:note"');
|
||||
expect(report).toContain('"revision": "3-current"');
|
||||
expect(report).toContain('"revision": "2-conflict"');
|
||||
expect(report).toContain('"storageType": "plain"');
|
||||
expect(report).toContain('"storageLayout": "chunked"');
|
||||
expect(report).toContain('"contentAvailableLocally": false');
|
||||
expect(report).toContain('"id": "h:private-current"');
|
||||
expect(report).toContain('"localDatabaseRevision": "1-chunk"');
|
||||
expect(report).toContain('"referenceCount": 2');
|
||||
expect(report).toContain('"id": "h:private-embedded"');
|
||||
expect(report).toContain('"embedded": true');
|
||||
expect(report).toContain('"id": "h:private-deleted"');
|
||||
expect(report).toContain('"localDatabaseState": "deleted"');
|
||||
expect(report).toContain('"localDatabaseRevision": "4-deleted-chunk"');
|
||||
expect(report).toContain('"id": "h:private-missing"');
|
||||
expect(report).toContain('"localDatabaseState": "missing"');
|
||||
expect(report).toContain('"localDatabaseRevision": null');
|
||||
expect(report).not.toContain("secret current body");
|
||||
expect(report).not.toContain("secret conflict body");
|
||||
expect(report).not.toContain("secret embedded body");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "notes",
|
||||
document: {
|
||||
type: "notes",
|
||||
data: "secret legacy body",
|
||||
},
|
||||
storageType: "notes",
|
||||
},
|
||||
{
|
||||
name: "an absent type",
|
||||
document: {
|
||||
type: undefined,
|
||||
data: ["secret", " legacy body"],
|
||||
},
|
||||
storageType: "absent",
|
||||
},
|
||||
])("reports $name as legacy inline storage without exposing its body", async ({ document, storageType }) => {
|
||||
const { core, current } = createCore();
|
||||
core.localDatabase.localDatabase.get.mockResolvedValue({
|
||||
...current,
|
||||
...document,
|
||||
_conflicts: [],
|
||||
children: ["h:must-not-be-treated-as-a-chunk"],
|
||||
} as never);
|
||||
|
||||
const info = await inspectFileDatabaseInfo(core as never, "note.md");
|
||||
const report = await buildFileDatabaseInfoReport(core as never, "note.md");
|
||||
|
||||
expect(info.database.revisions).toEqual([
|
||||
expect.objectContaining({
|
||||
storageType,
|
||||
storageLayout: "legacy-inline",
|
||||
chunkReferences: 0,
|
||||
contentAvailableLocally: true,
|
||||
}),
|
||||
]);
|
||||
expect(report).not.toContain("secret legacy body");
|
||||
expect(report).not.toContain("h:must-not-be-treated-as-a-chunk");
|
||||
});
|
||||
|
||||
it("reports the exact shared ancestor and its missing chunks for each conflict", async () => {
|
||||
const { conflict, core, current } = createCore();
|
||||
const parent = {
|
||||
...current,
|
||||
_rev: "2-parent",
|
||||
_conflicts: undefined,
|
||||
_revs_info: [
|
||||
{ rev: "2-parent", status: "available" },
|
||||
{ rev: "1-root", status: "missing" },
|
||||
],
|
||||
children: ["h:missing-parent"],
|
||||
eden: {},
|
||||
};
|
||||
core.localDatabase.localDatabase.get.mockImplementation(async (_id: string, options?: { rev?: string }) => {
|
||||
if (options?.rev === "2-conflict") {
|
||||
return {
|
||||
...conflict,
|
||||
_revs_info: [
|
||||
{ rev: "2-conflict", status: "available" },
|
||||
{ rev: "2-parent", status: "available" },
|
||||
{ rev: "1-root", status: "missing" },
|
||||
],
|
||||
};
|
||||
}
|
||||
if (options?.rev === "2-parent") {
|
||||
return parent;
|
||||
}
|
||||
return {
|
||||
...current,
|
||||
_revs_info: [
|
||||
{ rev: "3-current", status: "available" },
|
||||
{ rev: "2-parent", status: "available" },
|
||||
{ rev: "1-root", status: "missing" },
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
const info = await inspectFileDatabaseInfo(core as never, "note.md");
|
||||
|
||||
expect(info.database.mergeBases).toEqual([
|
||||
{
|
||||
winnerRevision: "3-current",
|
||||
conflictRevision: "2-conflict",
|
||||
revision: "2-parent",
|
||||
metadataAvailableLocally: true,
|
||||
contentAvailableLocally: false,
|
||||
missingChunkIds: ["h:missing-parent"],
|
||||
unavailableSharedRevisions: ["1-root"],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not decode a revision whose chunks are not all available locally", async () => {
|
||||
const { core } = createCore();
|
||||
|
||||
await expect(readFileDatabaseRevisionLocally(core as never, "note.md", "3-current")).resolves.toBe(false);
|
||||
|
||||
expect(core.localDatabase.getDBEntryFromMeta).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("decodes an exact revision after confirming that every chunk is available locally", async () => {
|
||||
const { core, current } = createCore();
|
||||
core.localDatabase.localDatabase.get.mockResolvedValue({
|
||||
...current,
|
||||
children: ["h:available"],
|
||||
eden: {},
|
||||
} as never);
|
||||
core.localDatabase.allDocsRaw.mockResolvedValue({
|
||||
rows: [
|
||||
{
|
||||
id: "h:available",
|
||||
key: "h:available",
|
||||
value: { rev: "1-available" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(readFileDatabaseRevisionLocally(core as never, "note.md", "3-current")).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
data: ["loaded body"],
|
||||
})
|
||||
);
|
||||
|
||||
expect(core.localDatabase.getDBEntryFromMeta).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
_rev: "3-current",
|
||||
}),
|
||||
false,
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
it("retries an exact revision through the configured chunk retrieval path", async () => {
|
||||
const { core } = createCore();
|
||||
|
||||
await retryReadFileDatabaseRevision(core as never, "note.md", "2-conflict");
|
||||
|
||||
expect(core.localDatabase.getDBEntry).toHaveBeenCalledWith(
|
||||
"note.md",
|
||||
{ rev: "2-conflict" },
|
||||
false,
|
||||
true,
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
it("reports the exact revision as locally available after retry recovers its missing chunk", async () => {
|
||||
const { conflict, core } = createCore();
|
||||
let recovered = false;
|
||||
core.localDatabase.getDBEntry.mockImplementation(async () => {
|
||||
recovered = true;
|
||||
return conflict as never;
|
||||
});
|
||||
core.localDatabase.allDocsRaw.mockImplementation(async ({ keys }: { keys: string[] }) => ({
|
||||
rows:
|
||||
recovered && keys.includes("h:private-missing")
|
||||
? [
|
||||
{
|
||||
id: "h:private-missing",
|
||||
key: "h:private-missing",
|
||||
value: { rev: "1-recovered" },
|
||||
},
|
||||
]
|
||||
: [],
|
||||
}));
|
||||
|
||||
await expect(
|
||||
retryReadFileDatabaseRevision(core as never, "note.md", "2-conflict")
|
||||
).resolves.not.toBe(false);
|
||||
const information = await inspectFileDatabaseInfo(core as never, "note.md");
|
||||
|
||||
expect(
|
||||
information.database.revisions.find(({ revision }) => revision === "2-conflict")
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
contentAvailableLocally: true,
|
||||
chunks: [
|
||||
expect.objectContaining({
|
||||
id: "h:private-missing",
|
||||
localDatabaseState: "available",
|
||||
localDatabaseRevision: "1-recovered",
|
||||
}),
|
||||
],
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the exact revision identifiers when conflict metadata is unavailable", async () => {
|
||||
const { conflict, core, current } = createCore();
|
||||
core.localDatabase.localDatabase.get.mockImplementation(async (_id: string, options?: { rev?: string }) => {
|
||||
if (options?.rev === "2-unavailable") {
|
||||
throw Object.assign(new Error("missing"), { status: 404 });
|
||||
}
|
||||
if (options?.rev === "2-conflict") {
|
||||
return conflict;
|
||||
}
|
||||
return { ...current, _conflicts: ["2-conflict", "2-unavailable"] };
|
||||
});
|
||||
|
||||
const report = await buildFileDatabaseInfoReport(core as never, "note.md");
|
||||
|
||||
expect(report).toContain('"conflictRevisions"');
|
||||
expect(report).toContain('"2-conflict"');
|
||||
expect(report).toContain('"2-unavailable"');
|
||||
expect(report).toContain('"unavailableConflictRevisions"');
|
||||
});
|
||||
|
||||
it("reads an existing local document even when current synchronisation filters exclude its path", async () => {
|
||||
const { core, current } = createCore();
|
||||
core.services.path.path2id.mockResolvedValue("f:ignored");
|
||||
core.localDatabase.localDatabase.get.mockResolvedValue({
|
||||
...current,
|
||||
_id: "f:ignored",
|
||||
_rev: "5-ignored",
|
||||
_conflicts: [],
|
||||
_revs_info: [],
|
||||
path: "ignored.md",
|
||||
ctime: 10,
|
||||
mtime: 20,
|
||||
size: 30,
|
||||
children: [],
|
||||
});
|
||||
|
||||
const report = await buildFileDatabaseInfoReport(core as never, "ignored.md");
|
||||
|
||||
expect(report).toContain('"exists": true');
|
||||
expect(report).toContain('"documentId": "f:ignored"');
|
||||
expect(report).toContain('"revision": "5-ignored"');
|
||||
});
|
||||
|
||||
it("offers the union of storage and database paths and excludes inactive internal namespaces", async () => {
|
||||
const { core } = createCore();
|
||||
|
||||
await expect(collectFileDatabaseInfoPaths(core as never)).resolves.toEqual(["a.md", "db-only.md", "z.md"]);
|
||||
|
||||
core.settings.syncInternalFiles = true;
|
||||
await expect(collectFileDatabaseInfoPaths(core as never)).resolves.toEqual([
|
||||
".obsidian/app.json",
|
||||
"a.md",
|
||||
"db-only.md",
|
||||
]);
|
||||
});
|
||||
|
||||
it("copies the selected file report through the existing copy dialogue", async () => {
|
||||
const { askSelectString, core, promptCopyToClipboard } = createCore();
|
||||
|
||||
await expect(chooseAndCopyFileDatabaseInfo(core as never)).resolves.toBe(true);
|
||||
|
||||
expect(askSelectString).toHaveBeenCalledWith("Choose a file to inspect", ["a.md", "db-only.md", "z.md"]);
|
||||
expect(promptCopyToClipboard).toHaveBeenCalledWith(
|
||||
"Database information for db-only.md",
|
||||
expect.stringContaining('"path": "db-only.md"')
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
import type { LoadedEntry } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { createBlob, isDocContentSame, readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
|
||||
import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
|
||||
import type { IFileHandler } from "@vrtmrz/livesync-commonlib/compat/interfaces/FileHandler";
|
||||
import {
|
||||
inspectFileDatabaseInfo,
|
||||
readFileDatabaseRevisionLocally,
|
||||
type FileDatabaseInfo,
|
||||
type FileDatabaseInfoCore,
|
||||
type RevisionDatabaseInfo,
|
||||
} from "./fileDatabaseInfo";
|
||||
|
||||
export type FileRepairCore = FileDatabaseInfoCore & {
|
||||
fileHandler: Pick<IFileHandler, "deleteRevisionFromDB">;
|
||||
storageAccess: FileDatabaseInfoCore["storageAccess"] & Pick<StorageAccess, "readHiddenFileBinary">;
|
||||
};
|
||||
|
||||
export type FileRepairRevision = {
|
||||
role: "winner" | "conflict";
|
||||
metadata: RevisionDatabaseInfo;
|
||||
contentReadable: boolean;
|
||||
contentMatchesStorage: boolean | null;
|
||||
loadedEntry: LoadedEntry | false;
|
||||
};
|
||||
|
||||
export type FileRepairInspection = {
|
||||
information: FileDatabaseInfo;
|
||||
revisions: FileRepairRevision[];
|
||||
requiresAttention: boolean;
|
||||
};
|
||||
|
||||
export type DiscardUnreadableRevisionResult =
|
||||
| "discarded"
|
||||
| "failed"
|
||||
| "no-longer-live"
|
||||
| "revision-is-readable";
|
||||
|
||||
export async function inspectFileRepair(core: FileRepairCore, path: string): Promise<FileRepairInspection> {
|
||||
const information = await inspectFileDatabaseInfo(core, path);
|
||||
const storageContent = information.storage.exists
|
||||
? createBlob(await core.storageAccess.readHiddenFileBinary(path))
|
||||
: undefined;
|
||||
const revisions: FileRepairRevision[] = [];
|
||||
|
||||
for (const metadata of information.database.revisions) {
|
||||
const loadedEntry =
|
||||
metadata.deleted || !metadata.contentAvailableLocally
|
||||
? false
|
||||
: await readFileDatabaseRevisionLocally(core, path, metadata.revision ?? "");
|
||||
const contentReadable = metadata.deleted || loadedEntry !== false;
|
||||
const contentMatchesStorage =
|
||||
storageContent && loadedEntry !== false
|
||||
? await isDocContentSame(storageContent, readAsBlob(loadedEntry))
|
||||
: null;
|
||||
revisions.push({
|
||||
role: metadata.current ? "winner" : "conflict",
|
||||
metadata,
|
||||
contentReadable,
|
||||
contentMatchesStorage,
|
||||
loadedEntry,
|
||||
});
|
||||
}
|
||||
|
||||
const winner = revisions.find(({ role }) => role === "winner");
|
||||
const databaseAndStorageDiffer =
|
||||
information.storage.exists !== information.database.exists ||
|
||||
(information.storage.exists &&
|
||||
winner !== undefined &&
|
||||
(winner.metadata.deleted || winner.contentMatchesStorage === false)) ||
|
||||
(!information.storage.exists && winner !== undefined && !winner.metadata.deleted);
|
||||
const unreadableLiveRevision =
|
||||
information.database.unavailableConflictRevisions.length > 0 ||
|
||||
revisions.some(({ contentReadable }) => !contentReadable);
|
||||
const requiresAttention =
|
||||
databaseAndStorageDiffer ||
|
||||
information.database.conflictCount > 0 ||
|
||||
unreadableLiveRevision ||
|
||||
(information.database.exists && winner === undefined);
|
||||
|
||||
return {
|
||||
information,
|
||||
revisions,
|
||||
requiresAttention,
|
||||
};
|
||||
}
|
||||
|
||||
export async function discardUnreadableLiveRevision(
|
||||
core: FileRepairCore,
|
||||
path: string,
|
||||
revision: string
|
||||
): Promise<DiscardUnreadableRevisionResult> {
|
||||
const latest = await inspectFileDatabaseInfo(core, path);
|
||||
const liveRevisions = [
|
||||
latest.database.currentRevision,
|
||||
...latest.database.conflictRevisions,
|
||||
].filter((candidate): candidate is string => candidate !== null);
|
||||
if (!liveRevisions.includes(revision)) {
|
||||
return "no-longer-live";
|
||||
}
|
||||
|
||||
const metadata = latest.database.revisions.find((candidate) => candidate.revision === revision);
|
||||
const metadataUnavailable = latest.database.unavailableConflictRevisions.includes(revision);
|
||||
if (!metadataUnavailable && (metadata?.deleted || metadata?.contentAvailableLocally)) {
|
||||
return "revision-is-readable";
|
||||
}
|
||||
|
||||
const deleted = await core.fileHandler.deleteRevisionFromDB(latest.databasePath, revision);
|
||||
return deleted ? "discarded" : "failed";
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
discardUnreadableLiveRevision,
|
||||
inspectFileRepair,
|
||||
} from "./fileRepair";
|
||||
|
||||
function createCore() {
|
||||
const current = {
|
||||
_id: "f:note",
|
||||
_rev: "3-current",
|
||||
_conflicts: ["2-conflict"],
|
||||
_revs_info: [{ rev: "3-current", status: "available" }],
|
||||
path: "note.md",
|
||||
ctime: 1,
|
||||
mtime: 3,
|
||||
size: 7,
|
||||
type: "plain",
|
||||
children: ["h:current"],
|
||||
eden: {},
|
||||
};
|
||||
const conflict = {
|
||||
...current,
|
||||
_rev: "2-conflict",
|
||||
_conflicts: undefined,
|
||||
_revs_info: [{ rev: "2-conflict", status: "available" }],
|
||||
mtime: 2,
|
||||
children: ["h:missing-conflict"],
|
||||
};
|
||||
const deleteRevisionFromDB = vi.fn(async () => true);
|
||||
const core = {
|
||||
settings: {
|
||||
syncInternalFiles: false,
|
||||
syncInternalFilesIgnorePatterns: "",
|
||||
syncInternalFilesTargetPatterns: "",
|
||||
},
|
||||
storageAccess: {
|
||||
isExistsIncludeHidden: vi.fn(async () => true),
|
||||
statHidden: vi.fn(async () => ({
|
||||
ctime: 1,
|
||||
mtime: 3,
|
||||
size: 7,
|
||||
type: "file",
|
||||
})),
|
||||
readHiddenFileBinary: vi.fn(async () => new TextEncoder().encode("current").buffer),
|
||||
getFileNames: vi.fn(async () => ["note.md"]),
|
||||
getFilesIncludeHidden: vi.fn(async () => ["note.md"]),
|
||||
},
|
||||
localDatabase: {
|
||||
localDatabase: {
|
||||
get: vi.fn(async (_id: string, options?: { rev?: string }) =>
|
||||
options?.rev === "2-conflict" ? conflict : current
|
||||
),
|
||||
},
|
||||
allDocsRaw: vi.fn(async ({ keys }: { keys: string[] }) => ({
|
||||
rows: keys.includes("h:current")
|
||||
? [
|
||||
{
|
||||
id: "h:current",
|
||||
key: "h:current",
|
||||
value: { rev: "1-current" },
|
||||
},
|
||||
]
|
||||
: [],
|
||||
})),
|
||||
getDBEntryFromMeta: vi.fn(async (meta: typeof current) => ({
|
||||
...meta,
|
||||
data: [meta._rev === "3-current" ? "current" : "conflict"],
|
||||
})),
|
||||
getDBEntry: vi.fn(async () => false),
|
||||
findAllDocs: vi.fn(async function* () {
|
||||
yield current;
|
||||
}),
|
||||
},
|
||||
fileHandler: {
|
||||
deleteRevisionFromDB,
|
||||
},
|
||||
services: {
|
||||
path: {
|
||||
path2id: vi.fn(async () => "f:note"),
|
||||
},
|
||||
UI: {
|
||||
confirm: {},
|
||||
},
|
||||
},
|
||||
};
|
||||
return {
|
||||
conflict,
|
||||
core,
|
||||
current,
|
||||
deleteRevisionFromDB,
|
||||
};
|
||||
}
|
||||
|
||||
describe("file repair inspection", () => {
|
||||
it("shows the winner and every conflict revision independently", async () => {
|
||||
const { core } = createCore();
|
||||
|
||||
const inspection = await inspectFileRepair(core as never, "note.md");
|
||||
|
||||
expect(inspection.revisions).toEqual([
|
||||
expect.objectContaining({
|
||||
role: "winner",
|
||||
contentReadable: true,
|
||||
contentMatchesStorage: true,
|
||||
metadata: expect.objectContaining({
|
||||
revision: "3-current",
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
role: "conflict",
|
||||
contentReadable: false,
|
||||
contentMatchesStorage: null,
|
||||
metadata: expect.objectContaining({
|
||||
revision: "2-conflict",
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
expect(inspection.requiresAttention).toBe(true);
|
||||
});
|
||||
|
||||
it("rechecks liveness and readability before discarding an exact revision", async () => {
|
||||
const { core, deleteRevisionFromDB } = createCore();
|
||||
|
||||
await expect(
|
||||
discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")
|
||||
).resolves.toBe("discarded");
|
||||
await expect(
|
||||
discardUnreadableLiveRevision(core as never, "note.md", "3-current")
|
||||
).resolves.toBe("revision-is-readable");
|
||||
|
||||
expect(deleteRevisionFromDB).toHaveBeenCalledOnce();
|
||||
expect(deleteRevisionFromDB).toHaveBeenCalledWith("note.md", "2-conflict");
|
||||
});
|
||||
|
||||
it("allows an exact unreadable generation-one winner to be discarded explicitly", async () => {
|
||||
const { core, current, deleteRevisionFromDB } = createCore();
|
||||
current._rev = "1-root";
|
||||
current._conflicts = [];
|
||||
current.children = ["h:missing-root"];
|
||||
core.localDatabase.allDocsRaw.mockResolvedValue({ rows: [] });
|
||||
|
||||
const inspection = await inspectFileRepair(core as never, "note.md");
|
||||
|
||||
expect(inspection.revisions).toEqual([
|
||||
expect.objectContaining({
|
||||
role: "winner",
|
||||
contentReadable: false,
|
||||
metadata: expect.objectContaining({
|
||||
revision: "1-root",
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
await expect(
|
||||
discardUnreadableLiveRevision(core as never, "note.md", "1-root")
|
||||
).resolves.toBe("discarded");
|
||||
expect(deleteRevisionFromDB).toHaveBeenCalledWith("note.md", "1-root");
|
||||
});
|
||||
|
||||
it("refuses to discard a revision which stopped being a live leaf", async () => {
|
||||
const { core, current, deleteRevisionFromDB } = createCore();
|
||||
core.localDatabase.localDatabase.get.mockResolvedValue({
|
||||
...current,
|
||||
_conflicts: [],
|
||||
});
|
||||
|
||||
await expect(
|
||||
discardUnreadableLiveRevision(core as never, "note.md", "2-conflict")
|
||||
).resolves.toBe("no-longer-live");
|
||||
|
||||
expect(deleteRevisionFromDB).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user