Add safe Metadata document ID repair

This commit is contained in:
vorotamoroz
2026-08-12 18:56:17 +00:00
parent eda78dee36
commit c933674a0d
13 changed files with 715 additions and 9 deletions
@@ -150,9 +150,38 @@ export const liveSyncProvisionalEnglishMessages = {
"Resolve every conflict by modification time? This logically deletes every version except the newest one and cannot recover content which is already unavailable.",
"Resolve all conflicts by the newest version": "Resolve all conflicts by the newest version",
"Inspect conflicts and file/database differences": "Inspect conflicts and file/database differences",
"Scan every Vault file and live local-database revision for conflicts, missing chunks, and differences. Each result provides actions for the exact revision.":
"Scan every Vault file and live local-database revision for conflicts, missing chunks, and differences. Each result provides actions for the exact revision.",
"Scan Vault files and local-database Metadata for conflicts, missing chunks, identity mismatches, and differences. Each result provides actions for one exact entry or revision.":
"Scan Vault files and local-database Metadata for conflicts, missing chunks, identity mismatches, and differences. Each result provides actions for one exact entry or revision.",
"Begin inspection": "Begin inspection",
"Metadata entry requires review and was left unchanged": "Metadata entry requires review and was left unchanged",
"The stored document ID does not match the ID derived from its recorded path.":
"The stored document ID does not match the ID derived from its recorded path.",
"The stored document ID and recorded path are handled by different synchronisation features.":
"The stored document ID and recorded path are handled by different synchronisation features.",
"Stored document ID: ${ID}": "Stored document ID: ${ID}",
"Expected document ID: ${ID}": "Expected document ID: ${ID}",
"Source revision: ${REVISION}": "Source revision: ${REVISION}",
"One-step repair is unavailable because this entry is ambiguous, no longer current, or unsafe to change.":
"One-step repair is unavailable because this entry is ambiguous, no longer current, or unsafe to change.",
"An exact target is already present; repair can remove the obsolete ID.":
"An exact target is already present; repair can remove the obsolete ID.",
"Repair is available for this entry.": "Repair is available for this entry.",
"Repair this Metadata document ID": "Repair this Metadata document ID",
"Repair Metadata ID": "Repair Metadata ID",
"Keep unchanged": "Keep unchanged",
"Repair Metadata document ID": "Repair Metadata document ID",
"This moves one local Metadata entry to the ID derived from its recorded path.\n\n**File:** `${FILE}` \n**Source:** `${SOURCE}@${REVISION}` \n**Target:** `${TARGET}`\n\nThe target is verified before the source is removed. Its CouchDB revision ancestry cannot be preserved.\n\n> [!warning] Before repairing\n> - Back up this device.\n> - If file-name case or path obfuscation was intentionally changed for the whole database, use Rebuild instead.\n> - If other devices share this database, pause them, allow this device to upload the repair, then resume them one at a time.":
"This moves one local Metadata entry to the ID derived from its recorded path.\n\n**File:** `${FILE}` \n**Source:** `${SOURCE}@${REVISION}` \n**Target:** `${TARGET}`\n\nThe target is verified before the source is removed. Its CouchDB revision ancestry cannot be preserved.\n\n> [!warning] Before repairing\n> - Back up this device.\n> - If file-name case or path obfuscation was intentionally changed for the whole database, use Rebuild instead.\n> - If other devices share this database, pause them, allow this device to upload the repair, then resume them one at a time.",
"Metadata document ID repair and the ordinary Vault scan completed. Run this inspection again after synchronisation.":
"Metadata document ID repair and the ordinary Vault scan completed. Run this inspection again after synchronisation.",
"Metadata document ID repair completed, but the ordinary Vault scan did not run. Keep synchronisation paused, resolve the scan condition, then run 'Scan storage and database again'.":
"Metadata document ID repair completed, but the ordinary Vault scan did not run. Keep synchronisation paused, resolve the scan condition, then run 'Scan storage and database again'.",
"The inspected state changed. No repair was performed; run inspection again.":
"The inspected state changed. No repair was performed; run inspection again.",
"Repair stopped after creating the target. The source was retained. Run inspection again before retrying.":
"Repair stopped after creating the target. The source was retained. Run inspection again before retrying.",
"Repair failed before the source was removed. Run inspection again before retrying.":
"Repair failed before the source was removed. Run inspection again before retrying.",
"Connection settings": "Connection settings",
"Saved connections": "Saved connections",
} as const;
@@ -7,7 +7,7 @@ import {
type EntryDoc,
type diff_result,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { createBlob, readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { createBlob, escapeMarkdownValue, readAsBlob } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger";
import { shouldBeIgnored } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import { Menu, diff_match_patch, setIcon } from "@/deps.ts";
@@ -44,6 +44,21 @@ import {
getFileRepairRevisionComparison,
} from "@/serviceFeatures/fileRepairPresentation.ts";
import { ConflictResolveModal } from "@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts";
import {
inspectMetadataDocumentIdentities,
MetadataDocumentRepairResults,
OfflineScanUnresolvedReasons,
repairMetadataDocumentIdentity,
type MetadataDocumentIdentityIssue,
} from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import {
metadataIdentityPathKey,
selectUnresolvedMetadataIdentityEntries,
} from "@/serviceFeatures/metadataIdentityInspection.ts";
import {
executeMetadataIdentityRepair,
MetadataIdentityRepairExecutions,
} from "@/serviceFeatures/metadataIdentityRepair.ts";
export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, { addPanel }: PageFunctions): void {
// const hatchWarn = this.createEl(paneEl, "div", { text: `To stop the boot up sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
// hatchWarn.addClass("op-warn-info");
@@ -178,6 +193,150 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
});
});
};
const addMetadataIdentityResult = (entry: MetadataDocumentIdentityIssue) => {
const { diagnostic } = entry.inspection;
const card = this.createEl(resultArea, "div", { cls: "sls-repair-result" });
this.createEl(card, "h6", { text: diagnostic.declaredPath });
this.createEl(card, "div", {
text: $msg("Metadata entry requires review and was left unchanged"),
cls: "sls-repair-status-warning",
});
this.createEl(card, "div", {
text:
diagnostic.reason === OfflineScanUnresolvedReasons.DOCUMENT_ID_MISMATCH
? $msg("The stored document ID does not match the ID derived from its recorded path.")
: $msg(
"The stored document ID and recorded path are handled by different synchronisation features."
),
cls: "sls-repair-metric",
});
this.createEl(card, "div", {
text: $msg("Stored document ID: ${ID}", { ID: diagnostic.actualDocumentId }),
cls: "sls-repair-metric",
});
if (diagnostic.expectedDocumentId !== undefined) {
this.createEl(card, "div", {
text: $msg("Expected document ID: ${ID}", { ID: diagnostic.expectedDocumentId }),
cls: "sls-repair-metric",
});
}
this.createEl(card, "div", {
text: $msg("Source revision: ${REVISION}", {
REVISION: entry.sourceRevision ?? $msg("Unknown revision"),
}),
cls: "sls-repair-metric",
});
if (entry.logicallyDeleted) {
this.createEl(card, "div", {
text: $msg("🗑️ Logical deletion"),
cls: "sls-repair-metric mod-warning",
});
}
if (entry.conflictRevisions.length > 0) {
this.createEl(card, "div", {
text: $msg("⚠️ Conflicts: ${COUNT}", { COUNT: `${entry.conflictRevisions.length}` }),
cls: "sls-repair-metric mod-warning",
});
}
if (
!entry.repairAvailable ||
diagnostic.expectedDocumentId === undefined ||
entry.sourceRevision === null
) {
this.createEl(card, "div", {
text: $msg(
"One-step repair is unavailable because this entry is ambiguous, no longer current, or unsafe to change."
),
cls: "sls-repair-metric mod-warning",
});
return;
}
this.createEl(card, "div", {
text: entry.targetAlreadyPresent
? $msg("An exact target is already present; repair can remove the obsolete ID.")
: $msg("Repair is available for this entry."),
cls: "sls-repair-status-ok",
});
const request = {
actualDocumentId: diagnostic.actualDocumentId,
expectedDocumentId: diagnostic.expectedDocumentId,
sourceRevision: entry.sourceRevision,
};
const repairAction = $msg("Repair Metadata ID");
const keepAction = $msg("Keep unchanged");
addActionMenu(card, $msg("More actions for ${FILE}", { FILE: diagnostic.declaredPath }), [
{
title: $msg("Repair this Metadata document ID"),
warning: true,
run: async () => {
const execution = await executeMetadataIdentityRepair(request, {
confirm: async () =>
(await this.core.confirm.confirmWithMessage(
$msg("Repair Metadata document ID"),
$msg(
"This moves one local Metadata entry to the ID derived from its recorded path.\n\n**File:** `${FILE}` \n**Source:** `${SOURCE}@${REVISION}` \n**Target:** `${TARGET}`\n\nThe target is verified before the source is removed. Its CouchDB revision ancestry cannot be preserved.\n\n> [!warning] Before repairing\n> - Back up this device.\n> - If file-name case or path obfuscation was intentionally changed for the whole database, use Rebuild instead.\n> - If other devices share this database, pause them, allow this device to upload the repair, then resume them one at a time.",
{
FILE: escapeMarkdownValue(diagnostic.declaredPath),
SOURCE: escapeMarkdownValue(diagnostic.actualDocumentId),
REVISION: escapeMarkdownValue(entry.sourceRevision!),
TARGET: escapeMarkdownValue(diagnostic.expectedDocumentId!),
}
),
[repairAction, keepAction],
keepAction,
undefined,
"vertical"
)) === repairAction,
repair: async (repairRequest) =>
await repairMetadataDocumentIdentity(this.core, repairRequest),
requestOrdinaryScan: async () => await this.services.vault.scanVault(true, false),
});
if (execution.status === MetadataIdentityRepairExecutions.CANCELLED) return;
const result = execution.result;
if (result.message) Logger(result.message, LOG_LEVEL_VERBOSE);
if (result.status === MetadataDocumentRepairResults.COMPLETED) {
if (execution.scanError !== undefined) {
Logger(execution.scanError, LOG_LEVEL_VERBOSE);
}
resultArea.replaceChildren();
this.createEl(resultArea, "div", {
text: execution.scanCompleted
? $msg(
"Metadata document ID repair and the ordinary Vault scan completed. Run this inspection again after synchronisation."
)
: $msg(
"Metadata document ID repair completed, but the ordinary Vault scan did not run. Keep synchronisation paused, resolve the scan condition, then run 'Scan storage and database again'."
),
cls: execution.scanCompleted
? "sls-repair-status-ok"
: "sls-repair-metric mod-warning",
});
return;
}
const resultMessage =
result.status === MetadataDocumentRepairResults.STALE ||
result.status === MetadataDocumentRepairResults.BLOCKED
? $msg("The inspected state changed. No repair was performed; run inspection again.")
: result.targetCreated
? $msg(
"Repair stopped after creating the target. The source was retained. Run inspection again before retrying."
)
: $msg(
"Repair failed before the source was removed. Run inspection again before retrying."
);
this.createEl(card, "div", {
text: resultMessage,
cls: "sls-repair-metric mod-warning",
});
},
},
]);
};
const findHiddenFile = async (path: string) => {
const addOn = this.core.getAddOn<HiddenFileSync>(HiddenFileSync.name);
if (!addOn) {
@@ -827,7 +986,7 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
.setName($msg("Inspect conflicts and file/database differences"))
.setDesc(
$msg(
"Scan every Vault file and live local-database revision for conflicts, missing chunks, and differences. Each result provides actions for the exact revision."
"Scan Vault files and local-database Metadata for conflicts, missing chunks, identity mismatches, and differences. Each result provides actions for one exact entry or revision."
)
)
.addButton((button) =>
@@ -839,6 +998,13 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
resultArea.replaceChildren();
Logger("Start inspecting file/database state", LOG_LEVEL_NOTICE, "verify");
this.core.localDatabase.clearCaches();
const identityEntries = await inspectMetadataDocumentIdentities(this.core);
const handleFilenameCaseSensitive = this.core.settings.handleFilenameCaseSensitive;
const unresolvedIdentity = selectUnresolvedMetadataIdentityEntries(
identityEntries,
handleFilenameCaseSensitive
);
unresolvedIdentity.entries.forEach(addMetadataIdentityResult);
const allPaths = await collectFileDatabaseInfoPaths(this.core);
let i = 0;
const incProc = () => {
@@ -853,6 +1019,13 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
const semaphore = Semaphore(10);
const processes = allPaths.map(async (path) => {
try {
if (
unresolvedIdentity.unresolvedPathKeys.has(
metadataIdentityPathKey(path, handleFilenameCaseSensitive)
)
) {
return incProc();
}
if (shouldBeIgnored(path)) {
return incProc();
}
@@ -0,0 +1,31 @@
import type { MetadataDocumentIdentityIssue } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
export function metadataIdentityPathKey(path: string, handleFilenameCaseSensitive: boolean): string {
const vaultPath = stripAllPrefixes(path as FilePathWithPrefix);
return handleFilenameCaseSensitive ? vaultPath : vaultPath.toLowerCase();
}
/**
* Select unresolved identity evidence for read-only presentation and create
* the path keys which must be withheld from ordinary path-based repair.
*/
export function selectUnresolvedMetadataIdentityEntries(
entries: readonly MetadataDocumentIdentityIssue[],
handleFilenameCaseSensitive: boolean
): {
entries: MetadataDocumentIdentityIssue[];
unresolvedPathKeys: ReadonlySet<string>;
} {
return {
entries: [...entries],
unresolvedPathKeys: new Set(
entries
.filter(({ ordinaryPathAvailable }) => !ordinaryPathAvailable)
.map(({ inspection }) =>
metadataIdentityPathKey(inspection.diagnostic.declaredPath, handleFilenameCaseSensitive)
)
),
};
}
@@ -0,0 +1,77 @@
import { describe, expect, it } from "vitest";
import type { MetadataDocumentIdentityIssue } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import { metadataIdentityPathKey, selectUnresolvedMetadataIdentityEntries } from "./metadataIdentityInspection";
function createEntries(): MetadataDocumentIdentityIssue[] {
return [
{
inspection: {
status: "unresolved",
diagnostic: {
reason: "document-id-mismatch",
actualDocumentId: "f:stale",
declaredPath: "Folder/Renamed.md",
expectedDocumentId: "f:renamed",
actualNamespace: "normal",
declaredPathNamespace: "normal",
},
},
sourceRevision: "3-stale",
logicallyDeleted: false,
conflictRevisions: [],
repairAvailable: false,
targetAlreadyPresent: false,
ordinaryPathAvailable: false,
},
{
inspection: {
status: "unresolved",
diagnostic: {
reason: "namespace-mismatch",
actualDocumentId: "f:stale-internal-path",
declaredPath: "i:.Obsidian/App.json",
actualNamespace: "normal",
declaredPathNamespace: "internal",
},
},
sourceRevision: "2-stale",
logicallyDeleted: false,
conflictRevisions: [],
repairAvailable: false,
targetAlreadyPresent: false,
ordinaryPathAvailable: false,
},
] as unknown as MetadataDocumentIdentityIssue[];
}
describe("Metadata identity inspection presentation", () => {
it("derives case-insensitive Vault path keys for unresolved evidence", () => {
const result = selectUnresolvedMetadataIdentityEntries(createEntries(), false);
expect(result.entries.map(({ sourceRevision }) => sourceRevision)).toEqual(["3-stale", "2-stale"]);
expect([...result.unresolvedPathKeys]).toEqual(["folder/renamed.md", ".obsidian/app.json"]);
expect(metadataIdentityPathKey("folder/RENAMED.md", false)).toBe("folder/renamed.md");
expect(result.unresolvedPathKeys.has(metadataIdentityPathKey("folder/RENAMED.md", false))).toBe(true);
});
it("retains case distinctions when filename handling is case-sensitive", () => {
const result = selectUnresolvedMetadataIdentityEntries(createEntries(), true);
expect(result.unresolvedPathKeys.has("Folder/Renamed.md")).toBe(true);
expect(result.unresolvedPathKeys.has("folder/renamed.md")).toBe(false);
});
it("does not suppress ordinary inspection when the path has resolvable Metadata", () => {
const entries = createEntries();
entries[0] = {
...entries[0],
ordinaryPathAvailable: true,
} as MetadataDocumentIdentityIssue;
const result = selectUnresolvedMetadataIdentityEntries(entries, false);
expect(result.entries).toHaveLength(2);
expect(result.unresolvedPathKeys.has("folder/renamed.md")).toBe(false);
expect(result.unresolvedPathKeys.has(".obsidian/app.json")).toBe(true);
});
});
@@ -0,0 +1,71 @@
import type {
MetadataDocumentRepairRequest,
MetadataDocumentRepairResult,
} from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import { MetadataDocumentRepairResults } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
export const MetadataIdentityRepairExecutions = {
CANCELLED: "cancelled",
REPAIR_RESULT: "repair-result",
} as const;
export type MetadataIdentityRepairExecution =
| { status: typeof MetadataIdentityRepairExecutions.CANCELLED }
| {
status: typeof MetadataIdentityRepairExecutions.REPAIR_RESULT;
result: MetadataDocumentRepairResult;
scanCompleted: boolean;
scanError?: unknown;
};
export interface MetadataIdentityRepairDependencies {
confirm: () => Promise<boolean>;
repair: (request: MetadataDocumentRepairRequest) => Promise<MetadataDocumentRepairResult>;
requestOrdinaryScan: () => Promise<boolean>;
}
/**
* Coordinate one explicitly confirmed Metadata identity repair.
*
* This consumer boundary deliberately keeps inspection approval, Commonlib
* mutation, and the subsequent ordinary Vault scan as separate operations.
* Cancellation cannot reach the mutation, and only a completed repair hands
* reconciliation back to the Offline Scanner. Commonlib re-inspects the
* source and expected ID under the current local path settings immediately
* before mutation, so remote replication state is not part of this boundary.
*/
export async function executeMetadataIdentityRepair(
request: MetadataDocumentRepairRequest,
dependencies: MetadataIdentityRepairDependencies
): Promise<MetadataIdentityRepairExecution> {
if (!(await dependencies.confirm())) {
return { status: MetadataIdentityRepairExecutions.CANCELLED };
}
const result = await dependencies.repair(request);
if (result.status !== MetadataDocumentRepairResults.COMPLETED) {
return {
status: MetadataIdentityRepairExecutions.REPAIR_RESULT,
result,
scanCompleted: false,
};
}
try {
const scanCompleted = await dependencies.requestOrdinaryScan();
return {
status: MetadataIdentityRepairExecutions.REPAIR_RESULT,
result,
scanCompleted,
};
} catch (scanError) {
// The Metadata identity mutation has already completed. Preserve that
// result separately so a follow-up scan failure cannot be mistaken for
// a failed or rolled-back repair.
return {
status: MetadataIdentityRepairExecutions.REPAIR_RESULT,
result,
scanCompleted: false,
scanError,
};
}
}
@@ -0,0 +1,108 @@
import { describe, expect, it, vi } from "vitest";
import type { DocumentID } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type {
MetadataDocumentRepairRequest,
MetadataDocumentRepairResult,
} from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import { executeMetadataIdentityRepair } from "./metadataIdentityRepair";
const request: MetadataDocumentRepairRequest = {
actualDocumentId: "f:stale" as DocumentID,
expectedDocumentId: "f:expected" as DocumentID,
sourceRevision: "4-source",
};
function createDependencies() {
const events: string[] = [];
return {
events,
confirm: vi.fn(async () => true),
repair: vi.fn(async (): Promise<MetadataDocumentRepairResult> => {
events.push("repair");
return {
status: "completed" as const,
...request,
targetCreated: true,
};
}),
requestOrdinaryScan: vi.fn(async () => {
events.push("scan");
return true;
}),
};
}
describe("executeMetadataIdentityRepair", () => {
it("performs no mutation when the separate confirmation is cancelled", async () => {
const dependencies = createDependencies();
dependencies.confirm.mockResolvedValue(false);
await expect(executeMetadataIdentityRepair(request, dependencies)).resolves.toEqual({
status: "cancelled",
});
expect(dependencies.repair).not.toHaveBeenCalled();
expect(dependencies.requestOrdinaryScan).not.toHaveBeenCalled();
});
it("requests an ordinary scan only after Commonlib completes the exact repair", async () => {
const dependencies = createDependencies();
await expect(executeMetadataIdentityRepair(request, dependencies)).resolves.toMatchObject({
status: "repair-result",
result: { status: "completed" },
scanCompleted: true,
});
expect(dependencies.repair).toHaveBeenCalledWith(request);
expect(dependencies.requestOrdinaryScan).toHaveBeenCalledOnce();
expect(dependencies.repair.mock.invocationCallOrder[0]).toBeLessThan(
dependencies.requestOrdinaryScan.mock.invocationCallOrder[0]
);
expect(dependencies.events).toEqual(["repair", "scan"]);
});
it("keeps a completed repair distinct when the ordinary scan cannot start", async () => {
const dependencies = createDependencies();
dependencies.requestOrdinaryScan.mockResolvedValue(false);
await expect(executeMetadataIdentityRepair(request, dependencies)).resolves.toMatchObject({
status: "repair-result",
result: { status: "completed" },
scanCompleted: false,
});
expect(dependencies.requestOrdinaryScan).toHaveBeenCalledOnce();
expect(dependencies.events).toEqual(["repair"]);
});
it("keeps a completed repair distinct when requesting the ordinary scan throws", async () => {
const dependencies = createDependencies();
const error = new Error("scan unavailable");
dependencies.requestOrdinaryScan.mockRejectedValue(error);
await expect(executeMetadataIdentityRepair(request, dependencies)).resolves.toMatchObject({
status: "repair-result",
result: { status: "completed" },
scanCompleted: false,
scanError: error,
});
expect(dependencies.events).toEqual(["repair"]);
});
it("does not scan after a stale, blocked, or failed repair result", async () => {
for (const status of ["stale", "blocked", "failed"] as const) {
const dependencies = createDependencies();
dependencies.repair.mockResolvedValue({
status,
...request,
targetCreated: false,
});
await expect(executeMetadataIdentityRepair(request, dependencies)).resolves.toMatchObject({
status: "repair-result",
result: { status },
scanCompleted: false,
});
expect(dependencies.requestOrdinaryScan).not.toHaveBeenCalled();
}
});
});