Add read-only database diagnostics

This commit is contained in:
vorotamoroz committed 2026-10-19 12:49:02 +00:00
1 parent f0ea5cb3a0
commit 8aeb7c60ef
19 files changed
+1049 -236

No files matched your search

+2
View File
@@ -19,6 +19,7 @@ export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor";
export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete";
export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage";
export const EVENT_INSPECT_DB_INTEGRITY = "inspect-db-integrity";
export const EVENT_REQUEST_PERFORM_GC_V3 = "request-perform-gc-v3";
// export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size";
// export const EVENT_FILE_CHANGED = "file-changed";
@@ -37,6 +38,7 @@ declare global {
[EVENT_REQUEST_RUN_DOCTOR]: string;
[EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined;
[EVENT_ANALYSE_DB_USAGE]: undefined;
[EVENT_INSPECT_DB_INTEGRITY]: undefined;
[EVENT_REQUEST_PERFORM_GC_V3]: undefined;
}
}
@@ -251,6 +251,47 @@ export const liveSyncProvisionalEnglishMessages = {
"An ID source is required to enable this option.": "An ID source is required to enable this option.",
"The ID source or recovery code is invalid. Check it and try again.":
"The ID source or recovery code is invalid. Check it and try again.",
"Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.":
"Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.",
Inspect: "Inspect",
"Inspect database integrity": "Inspect database integrity",
"Inspection completed. Review the findings and coverage before sharing.":
"Inspection completed. Review the findings and coverage before sharing.",
"Inspection partially completed. Some information could not be checked; review the reported reasons.":
"Inspection partially completed. Some information could not be checked; review the reported reasons.",
"Inspection failed. The report records the available evidence and reasons.":
"Inspection failed. The report records the available evidence and reasons.",
"Inspection cancelled. The report contains only the evidence collected so far.":
"Inspection cancelled. The report contains only the evidence collected so far.",
"Report copied.": "Report copied.",
"Copy failed. Select the report text and copy it manually.":
"Copy failed. Select the report text and copy it manually.",
"Download requested. You can also copy the report if downloads are unavailable on this device.":
"Download requested. You can also copy the report if downloads are unavailable on this device.",
"Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.":
"Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.",
"Read available local revisions and report their Chunk references. Settings are left unchanged.":
"Read available local revisions and report their Chunk references. Settings are left unchanged.",
"Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.":
"Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.",
"Names in this report": "Names in this report",
"Pseudonyms (recommended)": "Pseudonyms (recommended)",
"Original names": "Original names",
"Cancelling inspection…": "Cancelling inspection…",
"Items inspected": "Items inspected",
"Cancel inspection": "Cancel inspection",
"This report uses temporary pseudonyms. Counts and relationships remain visible.":
"This report uses temporary pseudonyms. Counts and relationships remain visible.",
"This report contains original paths and identifiers. Review them before sharing.":
"This report contains original paths and identifiers. Review them before sharing.",
"Partial Usage results are available as JSON so their coverage and reasons stay with the rows.":
"Partial Usage results are available as JSON so their coverage and reasons stay with the rows.",
"Report format": "Report format",
"Diagnostic report": "Diagnostic report",
"Copy report": "Copy report",
"Save report": "Save report",
"The inspection could not finish. Close this dialogue and try again after database startup.":
"The inspection could not finish. Close this dialogue and try again after database startup.",
} as const;
export type LiveSyncProvisionalMessageKey = keyof typeof liveSyncProvisionalEnglishMessages;
@@ -8,14 +8,13 @@ import {
type DocumentID,
type EntryDoc,
type EntryLeaf,
type FilePathWithPrefix,
type MetaEntry,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { getNoFromRev } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB";
import { LiveSyncCommands } from "@/features/LiveSyncCommands";
import { serialized } from "octagonal-wheels/concurrency/lock_v2";
import { arrayToChunkedArray } from "octagonal-wheels/collection";
import { EVENT_ANALYSE_DB_USAGE, EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events";
import { EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events";
import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
import type { ReplicatorInstance } from "@vrtmrz/livesync-commonlib/replication";
import { delay } from "@vrtmrz/livesync-commonlib/compat/common/utils";
@@ -61,18 +60,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands {
}
onload(): void | Promise<void> {
// NO OP.
this.services.API.addCommand({
id: "analyse-database",
name: "Analyse Database Usage (advanced)",
icon: "database-search",
checkCallback: (checking) => {
if (!this.settings.useAdvancedMode || !this._isDatabaseReady()) return false;
if (!checking) {
void this.analyseDatabase();
}
return true;
},
});
this.services.API.addCommand({
id: "gc-v3",
name: "Garbage Collection V3 (advanced, beta)",
@@ -88,7 +75,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands {
return true;
},
});
eventHub.onEvent(EVENT_ANALYSE_DB_USAGE, () => this.analyseDatabase());
eventHub.onEvent(EVENT_REQUEST_PERFORM_GC_V3, () => this.gcv3());
}
async allChunks(includeDeleted: boolean = false) {
@@ -550,218 +536,6 @@ Success: ${successCount}, Errored: ${errored}`;
await kvDB.set(DB_KEY_CHUNK_SET, chunkSet);
}
// Analyse the database and report chunk usage.
async analyseDatabase() {
if (!(await this.ensureAvailable("Analyse Database Usage"))) return;
const db = this.localDatabase.localDatabase;
// Map of chunk ID to its info
type ChunkInfo = {
id: DocumentID;
refCount: number;
length: number;
};
const chunkMap = new Map<DocumentID, Set<ChunkInfo>>();
// Map of document ID to its info
type DocumentInfo = {
id: DocumentID;
rev: Rev;
chunks: Set<ChunkID>;
uniqueChunks: Set<ChunkID>;
sharedChunks: Set<ChunkID>;
path: FilePathWithPrefix;
};
const docMap = new Map<DocumentID, Set<DocumentInfo>>();
const info = await db.info();
// Total number of revisions to process (approximate)
const maxSeq = Number.parseInt(`${info.update_seq ?? 0}`, 10);
let processed = 0;
let read = 0;
let errored = 0;
// Fetch Tasks
const ft = [] as ReturnType<typeof fetchRevision>[];
// Fetch a specific revision of a document and make note of its chunks, or add chunk info.
const fetchRevision = async (id: DocumentID, rev: Rev, seq: string | number) => {
try {
processed++;
const doc = await db.get(id, { rev: rev });
if (doc) {
if ("children" in doc) {
const id = doc._id;
const rev = doc._rev;
const children = (doc.children || []) as DocumentID[];
const set = docMap.get(id) || new Set();
set.add({
id,
rev,
chunks: new Set(children),
uniqueChunks: new Set(),
sharedChunks: new Set(),
path: doc.path,
});
docMap.set(id, set);
} else if (doc.type === EntryTypes.CHUNK) {
const id = doc._id;
if (chunkMap.has(id)) {
return;
}
if (doc._deleted) {
// Deleted chunk, skip (possibly resurrected later)
return;
}
const length = doc.data.length;
const set = chunkMap.get(id) || new Set();
set.add({ id, length, refCount: 0 });
chunkMap.set(id, set);
}
read++;
} else {
this._log(`Analysing Database: not found: ${id} / ${rev}`);
errored++;
}
} catch (error) {
this._log(`Error fetching document ${id} / ${rev}: $`, LOG_LEVEL_NOTICE);
this._log(error, LOG_LEVEL_VERBOSE);
errored++;
}
if (processed % 100 == 0) {
this._log(`Analysing database: ${read} (${errored}) / ${maxSeq} `, LOG_LEVEL_NOTICE, "db-analyse");
}
};
// Enumerate all documents and their revisions.
const IDs = this.localDatabase.findEntryNames("", "", {});
for await (const id of IDs) {
const revList = await this.localDatabase.getRaw(id as DocumentID, {
revs: true,
revs_info: true,
conflicts: true,
});
const revInfos = revList._revs_info || [];
for (const revInfo of revInfos) {
// All available revisions should be processed.
// If the revision is not available, it means the revision is already tombstoned.
if (revInfo.status == "available") {
// Schedule fetch task
ft.push(fetchRevision(id as DocumentID, revInfo.rev, 0));
}
}
}
// Wait for all fetch tasks to complete.
await Promise.all(ft);
// Reference count marking and unique/shared chunk classification.
for (const [, docRevs] of docMap) {
for (const docRev of docRevs) {
for (const chunkId of docRev.chunks) {
const chunkInfos = chunkMap.get(chunkId);
if (chunkInfos) {
for (const chunkInfo of chunkInfos) {
if (chunkInfo.refCount === 0) {
docRev.uniqueChunks.add(chunkId);
} else {
docRev.sharedChunks.add(chunkId);
}
chunkInfo.refCount++;
}
}
}
}
}
// Prepare results
const result = [];
// Calculate total size of chunks in the given set.
const getTotalSize = (ids: Set<DocumentID>) => {
return [...ids].reduce((acc, chunkId) => {
const chunkInfos = chunkMap.get(chunkId);
if (chunkInfos) {
for (const chunkInfo of chunkInfos) {
acc += chunkInfo.length;
}
}
return acc;
}, 0);
};
// Compile results for each document revision
for (const doc of docMap.values()) {
for (const rev of doc) {
const title = `${rev.path} (${rev.rev})`;
const id = rev.id;
const revStr = `${getNoFromRev(rev.rev)}`;
const revHash = rev.rev.split("-")[1].substring(0, 6);
const path = rev.path;
const uniqueChunkCount = rev.uniqueChunks.size;
const sharedChunkCount = rev.sharedChunks.size;
const uniqueChunkSize = getTotalSize(rev.uniqueChunks);
const sharedChunkSize = getTotalSize(rev.sharedChunks);
result.push({
title,
path,
rev: revStr,
revHash,
id,
uniqueChunkCount: uniqueChunkCount,
sharedChunkCount,
uniqueChunkSize: uniqueChunkSize,
sharedChunkSize: sharedChunkSize,
});
}
}
const titleMap = {
title: "Title",
id: "Document ID",
path: "Path",
rev: "Revision No",
revHash: "Revision Hash",
uniqueChunkCount: "Unique Chunk Count",
sharedChunkCount: "Shared Chunk Count",
uniqueChunkSize: "Unique Chunk Size",
sharedChunkSize: "Shared Chunk Size",
} as const;
// Enumerate orphan chunks (not referenced by any document)
const orphanChunks = [...chunkMap.entries()].filter(([chunkId, infos]) => {
const totalRefCount = [...infos].reduce((acc, info) => acc + info.refCount, 0);
return totalRefCount === 0;
});
const orphanChunkSize = orphanChunks.reduce((acc, [chunkId, infos]) => {
for (const info of infos) {
acc += info.length;
}
return acc;
}, 0);
result.push({
title: "__orphan",
id: "__orphan",
path: "__orphan",
rev: "1",
revHash: "xxxxx",
uniqueChunkCount: orphanChunks.length,
sharedChunkCount: 0,
uniqueChunkSize: orphanChunkSize,
sharedChunkSize: 0,
} as const);
const csvSrc = result.map((e) => {
return [
`${e.title.replace(/"/g, '""')}"`,
`${e.id}`,
`${e.path}`,
`${e.rev}`,
`${e.revHash}`,
`${e.uniqueChunkCount}`,
`${e.sharedChunkCount}`,
`${e.uniqueChunkSize}`,
`${e.sharedChunkSize}`,
].join("\t");
});
// Add title row
csvSrc.unshift(Object.values(titleMap).join("\t"));
const csv = csvSrc.join("\n");
// Prompt to copy to clipboard
await this.services.UI.promptCopyToClipboard("Database Analysis data (TSV):", csv);
}
async compactDatabase() {
const replicator = this.core.replicator;
if (!canCompactCouchDBRemote(replicator)) return;
@@ -42,6 +42,7 @@ import {
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { LocalDatabaseMaintenance } from "./CmdLocalDatabaseMainte";
import { ensureLocalDatabaseMaintenancePrerequisites } from "./maintenancePrerequisites";
import { eventHub, EVENT_REQUEST_PERFORM_GC_V3 } from "@/common/events";
function createPrerequisites(settingsOverride: Partial<typeof DEFAULT_SETTINGS> = {}) {
const askSelectStringDialogue = vi.fn(
@@ -63,7 +64,7 @@ function createPrerequisites(settingsOverride: Partial<typeof DEFAULT_SETTINGS>
}
describe("LocalDatabaseMaintenance prerequisites", () => {
it("shows database analysis in Advanced mode and Garbage Collection only in applicable Edge Case mode", () => {
it("shows Garbage Collection only in applicable Edge Case mode", () => {
const commands: Array<{
id: string;
checkCallback?: (checking: boolean) => boolean | void;
@@ -90,15 +91,15 @@ describe("LocalDatabaseMaintenance prerequisites", () => {
_isDatabaseReady: vi.fn(() => true),
});
vi.mocked(eventHub.onEvent).mockClear();
maintenance.onload();
expect(eventHub.onEvent).toHaveBeenCalledTimes(1);
expect(eventHub.onEvent).toHaveBeenCalledWith(EVENT_REQUEST_PERFORM_GC_V3, expect.any(Function));
const analyse = commands.find(({ id }) => id === "analyse-database");
const garbageCollect = commands.find(({ id }) => id === "gc-v3");
expect(analyse?.checkCallback?.(true)).toBe(false);
expect(garbageCollect?.checkCallback?.(true)).toBe(false);
settings.useAdvancedMode = true;
expect(analyse?.checkCallback?.(true)).toBe(true);
expect(garbageCollect?.checkCallback?.(true)).toBe(false);
settings.useEdgeCaseMode = true;
+2
View File
@@ -51,6 +51,7 @@ import { useInteractiveConflictResolutionFeature } from "./serviceFeatures/inter
import { ConflictResolveModal } from "./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts";
import { useObsidianReplicationRibbonFeature } from "./serviceFeatures/obsidianReplicationRibbon.ts";
import { useStartupLifecycleFeature } from "./serviceFeatures/startupLifecycle";
import { useDatabaseDiagnosticsFeature } from "./serviceFeatures/useDatabaseDiagnostics";
export type LiveSyncCore = LiveSyncBaseCore<ObsidianServiceContext, LiveSyncCommands>;
export default class ObsidianLiveSyncPlugin extends Plugin {
core: LiveSyncCore;
@@ -195,6 +196,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
useP2PReplicatorCommands(core, replicator);
useP2PReplicatorUI(core, core, replicator, createInteractiveP2PReplication(replicator));
useObsidianReplicationRibbonFeature(core);
useDatabaseDiagnosticsFeature(core);
useRemoteConfiguration(core);
useSetupProtocolFeature(core, setupManager);
@@ -16,6 +16,7 @@ import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
import {
EVENT_ANALYSE_DB_USAGE,
EVENT_INSPECT_DB_INTEGRITY,
EVENT_REQUEST_CHECK_REMOTE_SIZE,
EVENT_REQUEST_RUN_DOCTOR,
EVENT_REQUEST_RUN_FIX_INCOMPLETE,
@@ -113,6 +114,14 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
await chooseAndCopyFileDatabaseInfo(this.core);
})
);
new Setting(paneEl)
.setName($msg("Inspect database integrity"))
.setDesc(
$msg("Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.")
)
.addButton((button) =>
button.setButtonText($msg("Inspect")).onClick(() => eventHub.emitEvent(EVENT_INSPECT_DB_INTEGRITY))
);
new Setting(paneEl)
.setName($msg("Analyse database usage"))
.setDesc(
@@ -0,0 +1,148 @@
<script lang="ts">
import { onDestroy, onMount } from "svelte";
import type { GuestDialogProps } from "@/modules/services/LiveSyncUI/svelteDialog";
import DialogHeader from "@/modules/services/LiveSyncUI/components/DialogHeader.svelte";
import { $msg as translate } from "@/common/translation";
import type { DatabaseDiagnosticDialogOptions } from "@/serviceFeatures/databaseDiagnosticDialogTypes";
import type { DatabaseDiagnosticOutput } from "@/serviceFeatures/databaseDiagnostics";
import type { DiagnosticNameMode } from "@vrtmrz/livesync-commonlib/diagnostics";
const { setResult, getInitialData }: GuestDialogProps<"close", DatabaseDiagnosticDialogOptions> = $props();
let options = $state<DatabaseDiagnosticDialogOptions>();
let phase = $state<"choice" | "running" | "result" | "failed">("choice");
let nameMode = $state<DiagnosticNameMode>("pseudonym");
let processed = $state(0);
let cancelling = $state(false);
let output = $state<DatabaseDiagnosticOutput>();
let selectedFormat = $state("");
let actionStatus = $state("");
let disposed = false;
const selected = $derived(output?.artifacts.find((artifact) => artifact.format === selectedFormat));
const title = $derived(
options?.kind === "integrity" ? translate("Inspect database integrity") : translate("Analyse database usage")
);
const statusText = $derived(
output
? {
complete: translate("Inspection completed. Review the findings and coverage before sharing."),
partial: translate("Inspection partially completed. Some information could not be checked; review the reported reasons."),
failed: translate("Inspection failed. The report records the available evidence and reasons."),
cancelled: translate("Inspection cancelled. The report contains only the evidence collected so far."),
}[output.status]
: ""
);
onMount(() => {
options = getInitialData?.();
});
onDestroy(() => {
disposed = true;
options?.cancel();
});
async function begin() {
if (!options || phase !== "choice") return;
phase = "running";
try {
const result = await options.run(nameMode, (count) => {
if (!disposed) processed = count;
});
if (disposed) return;
output = result;
selectedFormat = result.artifacts[0].format;
phase = "result";
} catch {
if (!disposed) phase = "failed";
}
}
async function copyReport(event: MouseEvent) {
if (!selected) return;
const owner = (event.currentTarget as HTMLElement).ownerDocument.defaultView;
try {
await owner?.navigator.clipboard.writeText(selected.content);
actionStatus = translate("Report copied.");
} catch {
actionStatus = translate("Copy failed. Select the report text and copy it manually.");
}
}
function saveReport(event: MouseEvent) {
if (!selected) return;
const document = (event.currentTarget as HTMLElement).ownerDocument;
const url = URL.createObjectURL(new Blob([selected.content], { type: "text/plain;charset=utf-8" }));
const link = document.createElement("a");
link.href = url;
link.download = selected.filename;
document.body.appendChild(link);
link.click();
link.remove();
setTimeout(() => URL.revokeObjectURL(url), 1000);
actionStatus = translate("Download requested. You can also copy the report if downloads are unavailable on this device.");
}
</script>
<DialogHeader {title} />
<div class="database-diagnostic" data-diagnostic-phase={phase}>
{#if phase === "choice"}
<p>{translate("Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.")}</p>
<p>
{options?.kind === "usage"
? translate("Read available local revisions and report their Chunk references. Settings are left unchanged.")
: translate("Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.")}
</p>
<label>
{translate("Names in this report")}
<select bind:value={nameMode}>
<option value="pseudonym">{translate("Pseudonyms (recommended)")}</option>
<option value="original">{translate("Original names")}</option>
</select>
</label>
<div class="diagnostic-controls">
<button onclick={() => setResult("close")}>{translate("Cancel")}</button>
<button class="mod-cta" disabled={!options} onclick={() => void begin()}>{translate("Begin inspection")}</button>
</div>
{:else if phase === "running"}
<p role="status">
{cancelling ? translate("Cancelling inspection…") : `${translate("Items inspected")}: ${processed}`}
</p>
<button disabled={cancelling} onclick={() => { cancelling = true; options?.cancel(); }}>
{translate("Cancel inspection")}
</button>
{:else if phase === "result" && output}
<p role="status">{statusText}</p>
<p>
{output.nameMode === "pseudonym"
? translate("This report uses temporary pseudonyms. Counts and relationships remain visible.")
: translate("This report contains original paths and identifiers. Review them before sharing.")}
</p>
{#if options?.kind === "usage" && output.status !== "complete"}
<p>{translate("Partial Usage results are available as JSON so their coverage and reasons stay with the rows.")}</p>
{/if}
<label>
{translate("Report format")}
<select bind:value={selectedFormat}>
{#each output.artifacts as artifact}
<option value={artifact.format}>{artifact.format.toUpperCase()}</option>
{/each}
</select>
</label>
<textarea readonly rows="16" aria-label={translate("Diagnostic report")} value={selected?.content ?? ""}></textarea>
<p role="status">{actionStatus}</p>
<div class="diagnostic-controls">
<button onclick={(event) => void copyReport(event)}>{translate("Copy report")}</button>
<button onclick={saveReport}>{translate("Save report")}</button>
<button onclick={() => setResult("close")}>{translate("Close")}</button>
</div>
{:else if phase === "failed"}
<p role="status">{translate("The inspection could not finish. Close this dialogue and try again after database startup.")}</p>
<button onclick={() => setResult("close")}>{translate("Close")}</button>
{/if}
</div>
<style>
.database-diagnostic { display: flex; flex-direction: column; gap: var(--size-4-2); }
label { display: flex; align-items: center; flex-wrap: wrap; gap: var(--size-4-3); }
.diagnostic-controls { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: var(--size-4-2); }
button, select { min-height: 44px; }
textarea { width: 100%; resize: vertical; font-family: var(--font-monospace); font-size: var(--font-ui-small); }
</style>
@@ -0,0 +1,9 @@
import type { DiagnosticNameMode } from "@vrtmrz/livesync-commonlib/diagnostics";
import type { DatabaseDiagnosticKind, DatabaseDiagnosticOutput } from "./databaseDiagnostics";
/** The dialogue can run or cancel an inspection without receiving the service container. */
export interface DatabaseDiagnosticDialogOptions {
kind: DatabaseDiagnosticKind;
run(nameMode: DiagnosticNameMode, onProgress: (processed: number) => void): Promise<DatabaseDiagnosticOutput>;
cancel(): void;
}
+163
View File
@@ -0,0 +1,163 @@
import {
analyseDatabaseIntegrity,
analyseDatabaseUsage,
createCouchDBDiagnosticSource,
createDiagnosticNameMapper,
createLocalDiagnosticSource,
formatDiagnosticJSON,
formatIntegrityMarkdown,
formatUsageTSV,
projectIntegrityReport,
projectUsageReport,
type DiagnosticContent,
type DiagnosticNameMode,
type DiagnosticStatus,
} from "@vrtmrz/livesync-commonlib/diagnostics";
import { REMOTE_COUCHDB, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
import type { NecessaryServicesInterfaces } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule";
import commonlibPackage from "@vrtmrz/livesync-commonlib/package.json";
export type DatabaseDiagnosticKind = "usage" | "integrity";
export type DatabaseDiagnosticHost = NecessaryServicesInterfaces<
"setting" | "database" | "path" | "API",
"storageAccess"
>;
export interface DatabaseDiagnosticArtifact {
format: "json" | "markdown" | "tsv";
filename: string;
content: string;
}
export interface DatabaseDiagnosticOutput {
status: DiagnosticStatus;
nameMode: DiagnosticNameMode;
artifacts: DatabaseDiagnosticArtifact[];
}
export async function readDiagnosticStorage(
storage: Pick<StorageAccess, "statHidden" | "readHiddenFileBinary">,
path: string
): Promise<DiagnosticContent> {
if (/^(?:[/\\]|[a-zA-Z]:)/.test(path) || path.includes("\0") || path.split(/[/\\]/).includes("..")) {
return { status: "unreadable", reason: "content-unreadable" };
}
try {
const before = await storage.statHidden(path);
if (!before) return { status: "missing" };
if (before.type !== "file") return { status: "unreadable", reason: "content-unreadable" };
const bytes = new Uint8Array(await storage.readHiddenFileBinary(path));
const after = await storage.statHidden(path);
if (
!after ||
after.type !== "file" ||
before.mtime !== after.mtime ||
before.ctime !== after.ctime ||
before.size !== after.size ||
bytes.byteLength !== after.size
) {
return { status: "unreadable", reason: "storage-changed" };
}
return { status: "available", bytes };
} catch {
return { status: "unreadable", reason: "content-unreadable" };
}
}
/** Run only after the host has collected this run's name-display choice. */
export async function runDatabaseDiagnostic(
host: DatabaseDiagnosticHost,
kind: DatabaseDiagnosticKind,
nameMode: DiagnosticNameMode,
options: { signal?: AbortSignal; onProgress?(processed: number): void } = {}
): Promise<DatabaseDiagnosticOutput> {
const configuration = JSON.stringify(host.services.setting.currentSettings());
const settings = JSON.parse(configuration) as ObsidianLiveSyncSettings;
const database = host.services.database.localDatabase.localDatabase;
const isConfigurationCurrent = () =>
JSON.stringify(host.services.setting.currentSettings()) === configuration &&
host.services.database.localDatabase.localDatabase === database;
const mapper = createDiagnosticNameMapper(nameMode);
const versions = {
commonlib: commonlibPackage.version,
plugin: host.services.API.getPluginVersion(),
application: host.services.API.getAppVersion(),
};
const local = createLocalDiagnosticSource(database);
if (kind === "usage") {
const result = await analyseDatabaseUsage({
source: local,
signal: options.signal,
onProgress: (progress) => options.onProgress?.(progress.revisionsProcessed),
});
if (!isConfigurationCurrent()) {
result.errors.push({ code: "configuration_changed" });
if (result.status === "complete") result.status = "partial";
}
const report = projectUsageReport(result, mapper, versions);
const artifacts: DatabaseDiagnosticArtifact[] = [
{
format: "json",
filename: `database-usage-${result.status}.json`,
content: formatDiagnosticJSON(report),
},
];
// A standalone legacy TSV has no coverage fields. Partial runs retain
// their rows in JSON together with the status and reasons.
if (result.status === "complete") {
artifacts.unshift({ format: "tsv", filename: "database-usage.tsv", content: formatUsageTSV(report) });
}
return { status: result.status, nameMode, artifacts };
}
const result = await analyseDatabaseIntegrity({
local,
remote:
settings.remoteType === REMOTE_COUCHDB
? () =>
Promise.resolve(
createCouchDBDiagnosticSource({
settings,
signal: options.signal,
fetch: (request, requestOptions) =>
settings.useRequestAPI
? host.services.API.nativeFetch(request, requestOptions)
: host.services.API.webCompatFetch(request, requestOptions),
})
)
: undefined,
remoteUnavailableReason: settings.remoteType === REMOTE_COUCHDB ? "source-unavailable" : "unsupported-remote",
path: {
getPath: (entry) => host.services.path.getPath(entry),
path2id: (path, prefix) => host.services.path.path2idWithSettings(path, settings, prefix),
},
settings: {
encrypt: settings.encrypt,
usePathObfuscation: settings.usePathObfuscation,
handleFilenameCaseSensitive: settings.handleFilenameCaseSensitive,
idDerivationVersion: settings.idDerivationVersion ?? 0,
pathNormalizationVersion: settings.pathNormalizationVersion ?? 0,
},
readStorage: (path) => readDiagnosticStorage(host.serviceModules.storageAccess, path),
isConfigurationCurrent,
signal: options.signal,
onProgress: (progress) => options.onProgress?.(progress.processed),
});
const report = projectIntegrityReport(result, mapper, versions);
return {
status: result.status,
nameMode,
artifacts: [
{
format: "markdown",
filename: `database-integrity-${result.status}.md`,
content: formatIntegrityMarkdown(report),
},
{
format: "json",
filename: `database-integrity-${result.status}.json`,
content: formatDiagnosticJSON(report),
},
],
};
}
@@ -0,0 +1,213 @@
import { describe, expect, it, vi } from "vitest";
import { DEFAULT_SETTINGS, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { readDiagnosticStorage, runDatabaseDiagnostic, type DatabaseDiagnosticHost } from "./databaseDiagnostics";
function fixture() {
const path = "Private/日本語.md";
const body = "private file contents";
const bytes = new TextEncoder().encode(body);
const settings = {
...DEFAULT_SETTINGS,
remoteType: REMOTE_P2P,
readChunksOnline: true,
passphrase: "private encryption passphrase",
couchDB_PASSWORD: "private credential",
couchDB_URI: "https://private-server.invalid",
};
const documents = {
"f:private-actual-id": {
_id: "f:private-actual-id",
_rev: "1-private-revision",
_revs_info: [{ rev: "1-private-revision", status: "available" }],
type: "plain",
path,
ctime: 1,
mtime: 1,
size: bytes.byteLength,
children: ["h:private-chunk"],
},
"h:private-chunk": {
_id: "h:private-chunk",
_rev: "1-private-chunk-revision",
_revs_info: [{ rev: "1-private-chunk-revision", status: "available" }],
type: "leaf",
data: body,
},
};
const get = vi.fn(async (id: string) => {
const doc = documents[id as keyof typeof documents];
if (!doc) throw { status: 404 };
return structuredClone(doc);
});
const info = vi.fn(async () => ({ update_seq: "1-stable" }));
const put = vi.fn(() => {
throw new Error("Diagnostic attempted a write");
});
const applyPartial = vi.fn();
const fetch = vi.fn();
const storage = {
statHidden: vi.fn(async () => ({ type: "file" as const, ctime: 1, mtime: 1, size: bytes.byteLength })),
readHiddenFileBinary: vi.fn(async () => bytes.buffer),
};
const host = {
services: {
setting: { currentSettings: () => settings, applyPartial },
database: {
localDatabase: {
localDatabase: {
info,
get,
put,
allDocs: vi.fn(async () => ({
rows: Object.keys(documents).map((id) => ({ id, value: { rev: "1-current" } })),
})),
},
},
},
path: {
getPath: (doc: { path: string }) => doc.path,
path2idWithSettings: vi.fn(async (rawPath: string) => `f:expected:${rawPath}`),
},
API: {
webCompatFetch: fetch,
nativeFetch: fetch,
getPluginVersion: () => "1.0.38",
getAppVersion: () => "1.12.7",
},
},
serviceModules: { storageAccess: storage },
} as unknown as DatabaseDiagnosticHost;
return { host, settings, path, body, bytes, get, info, put, applyPartial, fetch, storage };
}
describe("database diagnostic host composition", () => {
it("keeps on-demand fetching enabled while analysing only existing local revisions", async () => {
const data = fixture();
const before = JSON.stringify(data.settings);
const result = await runDatabaseDiagnostic(data.host, "usage", "pseudonym");
expect(result.status).toBe("complete");
expect(result.artifacts.map((artifact) => artifact.format)).toEqual(["tsv", "json"]);
expect(data.settings.readChunksOnline).toBe(true);
expect(JSON.stringify(data.settings)).toBe(before);
expect(data.applyPartial).not.toHaveBeenCalled();
expect(data.fetch).not.toHaveBeenCalled();
expect(data.put).not.toHaveBeenCalled();
expect(data.get).toHaveBeenCalledWith("f:private-actual-id", { rev: "1-private-revision" });
const report = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content);
expect(report.versions).toMatchObject({ plugin: "1.0.38", application: "1.12.7" });
expect(report.versions.commonlib).toMatch(/^\d+\.\d+\.\d+/);
for (const artifact of result.artifacts) {
for (const secret of [
data.path,
data.body,
data.settings.passphrase,
"private-actual-id",
"private-revision",
"private-server.invalid",
]) {
expect(artifact.content).not.toContain(secret);
}
}
});
it("keeps local findings when the remote type is unsupported and projects both formats consistently", async () => {
const data = fixture();
const result = await runDatabaseDiagnostic(data.host, "integrity", "pseudonym");
expect(result.status).toBe("partial");
expect(data.fetch).not.toHaveBeenCalled();
expect(data.put).not.toHaveBeenCalled();
expect(data.storage.readHiddenFileBinary).toHaveBeenCalledWith(data.path);
const json = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content);
expect(json.errors).toContainEqual({ source: "remote", reason: "unsupported-remote" });
expect(json.observations[0]).toMatchObject({
identity: "document-id-mismatch",
content: { status: "available" },
});
expect(json.observations[0].comparisons).toContainEqual({
against: json.observations[0].vault.reference,
relation: "equal",
});
for (const artifact of result.artifacts) {
expect(artifact.content).toContain(json.observations[0].path);
expect(artifact.content).not.toContain(data.path);
expect(artifact.content).not.toContain(data.body);
expect(artifact.content).not.toContain(data.settings.passphrase);
}
});
it("includes names only when requested and never includes credentials or file contents", async () => {
const data = fixture();
const result = await runDatabaseDiagnostic(data.host, "usage", "original");
for (const artifact of result.artifacts) {
expect(artifact.content).toContain(data.path);
expect(artifact.content).toContain("f:private-actual-id");
expect(artifact.content).not.toContain(data.body);
expect(artifact.content).not.toContain(data.settings.couchDB_PASSWORD);
expect(artifact.content).not.toContain(data.settings.couchDB_URI);
}
});
it("retains cancellation and configuration changes in JSON instead of presenting a complete TSV", async () => {
const cancelled = fixture();
const controller = new AbortController();
controller.abort();
const stopped = await runDatabaseDiagnostic(cancelled.host, "usage", "pseudonym", {
signal: controller.signal,
});
expect(stopped.status).toBe("cancelled");
expect(stopped.artifacts.map((artifact) => artifact.format)).toEqual(["json"]);
expect(cancelled.get).not.toHaveBeenCalled();
const changed = fixture();
changed.info.mockImplementationOnce(async () => ({ update_seq: "1-stable" }));
changed.info.mockImplementationOnce(async () => {
changed.settings.couchDB_URI = "https://changed.invalid";
return { update_seq: "1-stable" };
});
const output = await runDatabaseDiagnostic(changed.host, "usage", "original");
expect(output.status).toBe("partial");
expect(output.artifacts.map((artifact) => artifact.format)).toEqual(["json"]);
expect(JSON.parse(output.artifacts[0].content).errors).toContainEqual({ code: "configuration_changed" });
});
});
describe("diagnostic Vault reads", () => {
it("distinguishes an absent file, a read error, and a file changing during its read", async () => {
const data = fixture();
data.storage.statHidden.mockResolvedValueOnce(null as never);
expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ status: "missing" });
data.storage.readHiddenFileBinary.mockRejectedValueOnce(new Error("private filename and content"));
expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({
status: "unreadable",
reason: "content-unreadable",
});
data.storage.statHidden.mockResolvedValueOnce({
type: "file",
ctime: 1,
mtime: 1,
size: data.bytes.byteLength,
});
data.storage.statHidden.mockResolvedValueOnce({
type: "file",
ctime: 1,
mtime: 2,
size: data.bytes.byteLength,
});
expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({
status: "unreadable",
reason: "storage-changed",
});
});
it("rejects paths outside the Vault before touching storage", async () => {
const data = fixture();
for (const path of ["../private", "/private", "C:\\private", "x/../private", "x\0y"]) {
expect(await readDiagnosticStorage(data.storage, path)).toEqual({
status: "unreadable",
reason: "content-unreadable",
});
}
expect(data.storage.statHidden).not.toHaveBeenCalled();
expect(data.storage.readHiddenFileBinary).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,76 @@
import {
createServiceFeature,
type NecessaryServices,
} from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule";
import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events";
import { $msg } from "@/common/translation";
import DatabaseDiagnosticDialog from "@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte";
import { runDatabaseDiagnostic, type DatabaseDiagnosticKind } from "./databaseDiagnostics";
import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes";
type DatabaseDiagnosticsFeatureHost = NecessaryServices<
"setting" | "database" | "path" | "API" | "UI" | "appLifecycle",
"storageAccess"
>;
export const useDatabaseDiagnosticsFeature = createServiceFeature((host: DatabaseDiagnosticsFeatureHost) => {
const { services } = host;
let active: AbortController | undefined;
const open = async (kind: DatabaseDiagnosticKind): Promise<void> => {
if (active) return;
const controller = new AbortController();
active = controller;
try {
await services.UI.dialogManager.open<"close", DatabaseDiagnosticDialogOptions>(DatabaseDiagnosticDialog, {
kind,
run: async (nameMode, onProgress) => {
if (!services.database.isDatabaseReady()) throw new Error("local-database-unavailable");
return await runDatabaseDiagnostic(host, kind, nameMode, {
signal: controller.signal,
onProgress,
});
},
cancel: () => controller.abort(),
});
} catch {
// Closing the host during unload can reject the dialogue result.
// Diagnostic errors are presented inside the dialogue without raw logs.
} finally {
controller.abort();
if (active === controller) active = undefined;
}
};
services.appLifecycle.onInitialise.addHandler(() => {
services.API.addCommand({
id: "analyse-database",
name: "Analyse Database Usage (advanced)",
icon: "database-search",
checkCallback: (checking) => {
if (!services.setting.currentSettings().useAdvancedMode || !services.database.isDatabaseReady())
return false;
if (!checking) void open("usage");
return true;
},
});
services.API.addCommand({
id: "inspect-database-integrity",
name: $msg("Inspect database integrity"),
icon: "database-search",
checkCallback: (checking) => {
if (!services.database.isDatabaseReady()) return false;
if (!checking) void open("integrity");
return true;
},
});
return Promise.resolve(true);
});
const stopUsage = services.context.events.onEvent(EVENT_ANALYSE_DB_USAGE, () => void open("usage"));
const stopIntegrity = services.context.events.onEvent(EVENT_INSPECT_DB_INTEGRITY, () => void open("integrity"));
services.appLifecycle.onUnload.addHandler(() => {
active?.abort();
stopUsage();
stopIntegrity();
return Promise.resolve(true);
});
return { open };
});
@@ -0,0 +1,122 @@
import { describe, expect, it, vi } from "vitest";
import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes";
vi.mock("@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte", () => ({ default: {} }));
vi.mock("./databaseDiagnostics", () => ({
runDatabaseDiagnostic: vi.fn(async () => ({ status: "complete", nameMode: "pseudonym", artifacts: [] })),
}));
vi.mock("@/common/translation", () => ({ $msg: (value: string) => value }));
import { runDatabaseDiagnostic } from "./databaseDiagnostics";
import { useDatabaseDiagnosticsFeature } from "./useDatabaseDiagnostics";
import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events";
function fixture() {
const initialise: Array<() => Promise<boolean>> = [];
const unload: Array<() => Promise<boolean>> = [];
const commands: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = [];
const listeners = new Map<string, () => void>();
const unsubscribe = vi.fn((event: string) => listeners.delete(event));
const settings = { useAdvancedMode: false };
let ready = true;
let close!: () => void;
const openDialog = vi.fn(
(_component: unknown, _options: DatabaseDiagnosticDialogOptions) =>
new Promise<void>((resolve) => {
close = resolve;
})
);
const host = {
services: {
setting: { currentSettings: () => settings },
database: { isDatabaseReady: () => ready },
path: {},
API: { addCommand: (command: (typeof commands)[number]) => commands.push(command) },
UI: { dialogManager: { open: openDialog } },
appLifecycle: {
onInitialise: { addHandler: (handler: (typeof initialise)[number]) => initialise.push(handler) },
onUnload: { addHandler: (handler: (typeof unload)[number]) => unload.push(handler) },
},
context: {
events: {
onEvent: (event: string, handler: () => void) => {
listeners.set(event, handler);
return () => unsubscribe(event);
},
},
},
},
serviceModules: { storageAccess: {} },
} as unknown as Parameters<typeof useDatabaseDiagnosticsFeature>[0];
const view = useDatabaseDiagnosticsFeature(host);
return {
host,
view,
initialise,
unload,
commands,
listeners,
settings,
openDialog,
unsubscribe,
close: () => close(),
setReady: (value: boolean) => {
ready = value;
},
};
}
describe("database diagnostics serviceFeature", () => {
it("retains the existing Advanced-mode Usage command gate in its new owner", async () => {
const data = fixture();
for (const initialise of data.initialise) await initialise();
const usage = data.commands.find((command) => command.id === "analyse-database")!;
const integrity = data.commands.find((command) => command.id === "inspect-database-integrity")!;
expect(usage.checkCallback(true)).toBe(false);
expect(integrity.checkCallback(true)).toBe(true);
data.settings.useAdvancedMode = true;
expect(usage.checkCallback(true)).toBe(true);
data.setReady(false);
expect(usage.checkCallback(true)).toBe(false);
expect(integrity.checkCallback(true)).toBe(false);
expect(data.openDialog).not.toHaveBeenCalled();
});
it("uses the existing dialogue manager and performs no analysis until the run choice is submitted", async () => {
vi.mocked(runDatabaseDiagnostic).mockClear();
const data = fixture();
const opened = data.view.open("usage");
expect(data.openDialog).toHaveBeenCalledTimes(1);
expect(runDatabaseDiagnostic).not.toHaveBeenCalled();
const options = data.openDialog.mock.calls[0][1];
expect(Object.keys(options).sort()).toEqual(["cancel", "kind", "run"]);
await options.run("original", () => {});
expect(runDatabaseDiagnostic).toHaveBeenCalledWith(
data.host,
"usage",
"original",
expect.objectContaining({ signal: expect.any(AbortSignal) })
);
const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!;
await data.view.open("integrity");
expect(data.openDialog).toHaveBeenCalledTimes(1);
data.close();
await opened;
expect(signal.aborted).toBe(true);
});
it("wires both Hatch actions and releases its listeners and active request on unload", async () => {
const data = fixture();
expect(data.listeners.has(EVENT_ANALYSE_DB_USAGE)).toBe(true);
expect(data.listeners.has(EVENT_INSPECT_DB_INTEGRITY)).toBe(true);
const opened = data.view.open("integrity");
await data.openDialog.mock.calls[0][1].run("pseudonym", () => {});
const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!;
for (const unload of data.unload) await unload();
expect(signal.aborted).toBe(true);
expect(data.listeners.size).toBe(0);
expect(data.unsubscribe).toHaveBeenCalledTimes(2);
data.close();
await opened;
});
});