Add read-only database diagnostics

This commit is contained in:
vorotamoroz
2026-10-19 12:49:02 +00:00
parent f0ea5cb3a0
commit 8aeb7c60ef
19 changed files with 1049 additions and 236 deletions
+6
View File
@@ -163,6 +163,12 @@ Enable Obsidian's `Detect all file extensions`, then check LiveSync selectors, i
## Collect a report ## Collect a report
For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. The inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete.
The result dialogue provides Markdown and JSON reports with the same name policy. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. They are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled.
**Analyse database usage** offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads.
Follow [How to report an issue](to_issue_reporting.md) for the complete reporting checklist, including Obsidian debug information and the privacy review required before sharing evidence. Follow [How to report an issue](to_issue_reporting.md) for the complete reporting checklist, including Obsidian debug information and the privacy review required before sharing evidence.
Run `Generate full report for opening the issue with debug info` to copy the current settings summary and recent verbose log lines. Remove credentials, remote URLs, Vault names, file contents, and other private information before sharing it. Run `Generate full report for opening the issue with debug info` to copy the current settings summary and recent verbose log lines. Remove credentials, remote URLs, Vault names, file contents, and other private information before sharing it.
+4 -4
View File
@@ -23,7 +23,7 @@
"@smithy/types": "^4.14.3", "@smithy/types": "^4.14.3",
"@smithy/util-retry": "^4.4.5", "@smithy/util-retry": "^4.4.5",
"@vrtmrz/browser-ui-kit": "0.1.0", "@vrtmrz/browser-ui-kit": "0.1.0",
"@vrtmrz/livesync-commonlib": "0.1.42", "@vrtmrz/livesync-commonlib": "0.1.43",
"@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/obsidian-plugin-kit": "0.1.4",
"@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/ui-interactions": "0.1.2",
"diff-match-patch": "^1.0.5", "diff-match-patch": "^1.0.5",
@@ -4389,9 +4389,9 @@
} }
}, },
"node_modules/@vrtmrz/livesync-commonlib": { "node_modules/@vrtmrz/livesync-commonlib": {
"version": "0.1.42", "version": "0.1.43",
"resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.42.tgz", "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.43.tgz",
"integrity": "sha512-LyWl0Mkrj9UsQXV0nOUaYJybcN3wLkpAAxCeHMZYOEizaUTQ7ezN+FgzN+OQxDRA8UXWnoatxh719NWzOwBpBw==", "integrity": "sha512-WM2k545hT0ctLv64WdKZ0PkHwyw9Fnlf9xssH7Uv9C0HEMm8CDtj6dfJsPOHs0+Z41Kfco1U/y6R5f5+xWZ00g==",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.808.0", "@aws-sdk/client-s3": "^3.808.0",
+2 -1
View File
@@ -57,6 +57,7 @@
"test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts", "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts",
"test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts",
"test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts",
"test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts",
"test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts",
"test:e2e:obsidian:revision-repair": "tsx test/e2e-obsidian/scripts/revision-repair.ts", "test:e2e:obsidian:revision-repair": "tsx test/e2e-obsidian/scripts/revision-repair.ts",
"test:e2e:obsidian:document-history-nav": "tsx test/e2e-obsidian/scripts/document-history-nav.ts", "test:e2e:obsidian:document-history-nav": "tsx test/e2e-obsidian/scripts/document-history-nav.ts",
@@ -191,7 +192,7 @@
"@smithy/types": "^4.14.3", "@smithy/types": "^4.14.3",
"@smithy/util-retry": "^4.4.5", "@smithy/util-retry": "^4.4.5",
"@vrtmrz/browser-ui-kit": "0.1.0", "@vrtmrz/browser-ui-kit": "0.1.0",
"@vrtmrz/livesync-commonlib": "0.1.42", "@vrtmrz/livesync-commonlib": "0.1.43",
"@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/obsidian-plugin-kit": "0.1.4",
"@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/ui-interactions": "0.1.2",
"diff-match-patch": "^1.0.5", "diff-match-patch": "^1.0.5",
+2
View File
@@ -19,6 +19,7 @@ export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor";
export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete"; export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete";
export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage"; export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage";
export const EVENT_INSPECT_DB_INTEGRITY = "inspect-db-integrity";
export const EVENT_REQUEST_PERFORM_GC_V3 = "request-perform-gc-v3"; export const EVENT_REQUEST_PERFORM_GC_V3 = "request-perform-gc-v3";
// export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size"; // export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size";
// export const EVENT_FILE_CHANGED = "file-changed"; // export const EVENT_FILE_CHANGED = "file-changed";
@@ -37,6 +38,7 @@ declare global {
[EVENT_REQUEST_RUN_DOCTOR]: string; [EVENT_REQUEST_RUN_DOCTOR]: string;
[EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined; [EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined;
[EVENT_ANALYSE_DB_USAGE]: undefined; [EVENT_ANALYSE_DB_USAGE]: undefined;
[EVENT_INSPECT_DB_INTEGRITY]: undefined;
[EVENT_REQUEST_PERFORM_GC_V3]: undefined; [EVENT_REQUEST_PERFORM_GC_V3]: undefined;
} }
} }
@@ -251,6 +251,47 @@ export const liveSyncProvisionalEnglishMessages = {
"An ID source is required to enable this option.": "An ID source is required to enable this option.", "An ID source is required to enable this option.": "An ID source is required to enable this option.",
"The ID source or recovery code is invalid. Check it and try again.": "The ID source or recovery code is invalid. Check it and try again.":
"The ID source or recovery code is invalid. Check it and try again.", "The ID source or recovery code is invalid. Check it and try again.",
"Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.":
"Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.",
Inspect: "Inspect",
"Inspect database integrity": "Inspect database integrity",
"Inspection completed. Review the findings and coverage before sharing.":
"Inspection completed. Review the findings and coverage before sharing.",
"Inspection partially completed. Some information could not be checked; review the reported reasons.":
"Inspection partially completed. Some information could not be checked; review the reported reasons.",
"Inspection failed. The report records the available evidence and reasons.":
"Inspection failed. The report records the available evidence and reasons.",
"Inspection cancelled. The report contains only the evidence collected so far.":
"Inspection cancelled. The report contains only the evidence collected so far.",
"Report copied.": "Report copied.",
"Copy failed. Select the report text and copy it manually.":
"Copy failed. Select the report text and copy it manually.",
"Download requested. You can also copy the report if downloads are unavailable on this device.":
"Download requested. You can also copy the report if downloads are unavailable on this device.",
"Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.":
"Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.",
"Read available local revisions and report their Chunk references. Settings are left unchanged.":
"Read available local revisions and report their Chunk references. Settings are left unchanged.",
"Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.":
"Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.",
"Names in this report": "Names in this report",
"Pseudonyms (recommended)": "Pseudonyms (recommended)",
"Original names": "Original names",
"Cancelling inspection…": "Cancelling inspection…",
"Items inspected": "Items inspected",
"Cancel inspection": "Cancel inspection",
"This report uses temporary pseudonyms. Counts and relationships remain visible.":
"This report uses temporary pseudonyms. Counts and relationships remain visible.",
"This report contains original paths and identifiers. Review them before sharing.":
"This report contains original paths and identifiers. Review them before sharing.",
"Partial Usage results are available as JSON so their coverage and reasons stay with the rows.":
"Partial Usage results are available as JSON so their coverage and reasons stay with the rows.",
"Report format": "Report format",
"Diagnostic report": "Diagnostic report",
"Copy report": "Copy report",
"Save report": "Save report",
"The inspection could not finish. Close this dialogue and try again after database startup.":
"The inspection could not finish. Close this dialogue and try again after database startup.",
} as const; } as const;
export type LiveSyncProvisionalMessageKey = keyof typeof liveSyncProvisionalEnglishMessages; export type LiveSyncProvisionalMessageKey = keyof typeof liveSyncProvisionalEnglishMessages;
@@ -8,14 +8,13 @@ import {
type DocumentID, type DocumentID,
type EntryDoc, type EntryDoc,
type EntryLeaf, type EntryLeaf,
type FilePathWithPrefix,
type MetaEntry, type MetaEntry,
} from "@vrtmrz/livesync-commonlib/compat/common/types"; } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { getNoFromRev } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; import { getNoFromRev } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB";
import { LiveSyncCommands } from "@/features/LiveSyncCommands"; import { LiveSyncCommands } from "@/features/LiveSyncCommands";
import { serialized } from "octagonal-wheels/concurrency/lock_v2"; import { serialized } from "octagonal-wheels/concurrency/lock_v2";
import { arrayToChunkedArray } from "octagonal-wheels/collection"; import { arrayToChunkedArray } from "octagonal-wheels/collection";
import { EVENT_ANALYSE_DB_USAGE, EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events"; import { EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events";
import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
import type { ReplicatorInstance } from "@vrtmrz/livesync-commonlib/replication"; import type { ReplicatorInstance } from "@vrtmrz/livesync-commonlib/replication";
import { delay } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { delay } from "@vrtmrz/livesync-commonlib/compat/common/utils";
@@ -61,18 +60,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands {
} }
onload(): void | Promise<void> { onload(): void | Promise<void> {
// NO OP. // NO OP.
this.services.API.addCommand({
id: "analyse-database",
name: "Analyse Database Usage (advanced)",
icon: "database-search",
checkCallback: (checking) => {
if (!this.settings.useAdvancedMode || !this._isDatabaseReady()) return false;
if (!checking) {
void this.analyseDatabase();
}
return true;
},
});
this.services.API.addCommand({ this.services.API.addCommand({
id: "gc-v3", id: "gc-v3",
name: "Garbage Collection V3 (advanced, beta)", name: "Garbage Collection V3 (advanced, beta)",
@@ -88,7 +75,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands {
return true; return true;
}, },
}); });
eventHub.onEvent(EVENT_ANALYSE_DB_USAGE, () => this.analyseDatabase());
eventHub.onEvent(EVENT_REQUEST_PERFORM_GC_V3, () => this.gcv3()); eventHub.onEvent(EVENT_REQUEST_PERFORM_GC_V3, () => this.gcv3());
} }
async allChunks(includeDeleted: boolean = false) { async allChunks(includeDeleted: boolean = false) {
@@ -550,218 +536,6 @@ Success: ${successCount}, Errored: ${errored}`;
await kvDB.set(DB_KEY_CHUNK_SET, chunkSet); await kvDB.set(DB_KEY_CHUNK_SET, chunkSet);
} }
// Analyse the database and report chunk usage.
async analyseDatabase() {
if (!(await this.ensureAvailable("Analyse Database Usage"))) return;
const db = this.localDatabase.localDatabase;
// Map of chunk ID to its info
type ChunkInfo = {
id: DocumentID;
refCount: number;
length: number;
};
const chunkMap = new Map<DocumentID, Set<ChunkInfo>>();
// Map of document ID to its info
type DocumentInfo = {
id: DocumentID;
rev: Rev;
chunks: Set<ChunkID>;
uniqueChunks: Set<ChunkID>;
sharedChunks: Set<ChunkID>;
path: FilePathWithPrefix;
};
const docMap = new Map<DocumentID, Set<DocumentInfo>>();
const info = await db.info();
// Total number of revisions to process (approximate)
const maxSeq = Number.parseInt(`${info.update_seq ?? 0}`, 10);
let processed = 0;
let read = 0;
let errored = 0;
// Fetch Tasks
const ft = [] as ReturnType<typeof fetchRevision>[];
// Fetch a specific revision of a document and make note of its chunks, or add chunk info.
const fetchRevision = async (id: DocumentID, rev: Rev, seq: string | number) => {
try {
processed++;
const doc = await db.get(id, { rev: rev });
if (doc) {
if ("children" in doc) {
const id = doc._id;
const rev = doc._rev;
const children = (doc.children || []) as DocumentID[];
const set = docMap.get(id) || new Set();
set.add({
id,
rev,
chunks: new Set(children),
uniqueChunks: new Set(),
sharedChunks: new Set(),
path: doc.path,
});
docMap.set(id, set);
} else if (doc.type === EntryTypes.CHUNK) {
const id = doc._id;
if (chunkMap.has(id)) {
return;
}
if (doc._deleted) {
// Deleted chunk, skip (possibly resurrected later)
return;
}
const length = doc.data.length;
const set = chunkMap.get(id) || new Set();
set.add({ id, length, refCount: 0 });
chunkMap.set(id, set);
}
read++;
} else {
this._log(`Analysing Database: not found: ${id} / ${rev}`);
errored++;
}
} catch (error) {
this._log(`Error fetching document ${id} / ${rev}: $`, LOG_LEVEL_NOTICE);
this._log(error, LOG_LEVEL_VERBOSE);
errored++;
}
if (processed % 100 == 0) {
this._log(`Analysing database: ${read} (${errored}) / ${maxSeq} `, LOG_LEVEL_NOTICE, "db-analyse");
}
};
// Enumerate all documents and their revisions.
const IDs = this.localDatabase.findEntryNames("", "", {});
for await (const id of IDs) {
const revList = await this.localDatabase.getRaw(id as DocumentID, {
revs: true,
revs_info: true,
conflicts: true,
});
const revInfos = revList._revs_info || [];
for (const revInfo of revInfos) {
// All available revisions should be processed.
// If the revision is not available, it means the revision is already tombstoned.
if (revInfo.status == "available") {
// Schedule fetch task
ft.push(fetchRevision(id as DocumentID, revInfo.rev, 0));
}
}
}
// Wait for all fetch tasks to complete.
await Promise.all(ft);
// Reference count marking and unique/shared chunk classification.
for (const [, docRevs] of docMap) {
for (const docRev of docRevs) {
for (const chunkId of docRev.chunks) {
const chunkInfos = chunkMap.get(chunkId);
if (chunkInfos) {
for (const chunkInfo of chunkInfos) {
if (chunkInfo.refCount === 0) {
docRev.uniqueChunks.add(chunkId);
} else {
docRev.sharedChunks.add(chunkId);
}
chunkInfo.refCount++;
}
}
}
}
}
// Prepare results
const result = [];
// Calculate total size of chunks in the given set.
const getTotalSize = (ids: Set<DocumentID>) => {
return [...ids].reduce((acc, chunkId) => {
const chunkInfos = chunkMap.get(chunkId);
if (chunkInfos) {
for (const chunkInfo of chunkInfos) {
acc += chunkInfo.length;
}
}
return acc;
}, 0);
};
// Compile results for each document revision
for (const doc of docMap.values()) {
for (const rev of doc) {
const title = `${rev.path} (${rev.rev})`;
const id = rev.id;
const revStr = `${getNoFromRev(rev.rev)}`;
const revHash = rev.rev.split("-")[1].substring(0, 6);
const path = rev.path;
const uniqueChunkCount = rev.uniqueChunks.size;
const sharedChunkCount = rev.sharedChunks.size;
const uniqueChunkSize = getTotalSize(rev.uniqueChunks);
const sharedChunkSize = getTotalSize(rev.sharedChunks);
result.push({
title,
path,
rev: revStr,
revHash,
id,
uniqueChunkCount: uniqueChunkCount,
sharedChunkCount,
uniqueChunkSize: uniqueChunkSize,
sharedChunkSize: sharedChunkSize,
});
}
}
const titleMap = {
title: "Title",
id: "Document ID",
path: "Path",
rev: "Revision No",
revHash: "Revision Hash",
uniqueChunkCount: "Unique Chunk Count",
sharedChunkCount: "Shared Chunk Count",
uniqueChunkSize: "Unique Chunk Size",
sharedChunkSize: "Shared Chunk Size",
} as const;
// Enumerate orphan chunks (not referenced by any document)
const orphanChunks = [...chunkMap.entries()].filter(([chunkId, infos]) => {
const totalRefCount = [...infos].reduce((acc, info) => acc + info.refCount, 0);
return totalRefCount === 0;
});
const orphanChunkSize = orphanChunks.reduce((acc, [chunkId, infos]) => {
for (const info of infos) {
acc += info.length;
}
return acc;
}, 0);
result.push({
title: "__orphan",
id: "__orphan",
path: "__orphan",
rev: "1",
revHash: "xxxxx",
uniqueChunkCount: orphanChunks.length,
sharedChunkCount: 0,
uniqueChunkSize: orphanChunkSize,
sharedChunkSize: 0,
} as const);
const csvSrc = result.map((e) => {
return [
`${e.title.replace(/"/g, '""')}"`,
`${e.id}`,
`${e.path}`,
`${e.rev}`,
`${e.revHash}`,
`${e.uniqueChunkCount}`,
`${e.sharedChunkCount}`,
`${e.uniqueChunkSize}`,
`${e.sharedChunkSize}`,
].join("\t");
});
// Add title row
csvSrc.unshift(Object.values(titleMap).join("\t"));
const csv = csvSrc.join("\n");
// Prompt to copy to clipboard
await this.services.UI.promptCopyToClipboard("Database Analysis data (TSV):", csv);
}
async compactDatabase() { async compactDatabase() {
const replicator = this.core.replicator; const replicator = this.core.replicator;
if (!canCompactCouchDBRemote(replicator)) return; if (!canCompactCouchDBRemote(replicator)) return;
@@ -42,6 +42,7 @@ import {
} from "@vrtmrz/livesync-commonlib/compat/common/types"; } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { LocalDatabaseMaintenance } from "./CmdLocalDatabaseMainte"; import { LocalDatabaseMaintenance } from "./CmdLocalDatabaseMainte";
import { ensureLocalDatabaseMaintenancePrerequisites } from "./maintenancePrerequisites"; import { ensureLocalDatabaseMaintenancePrerequisites } from "./maintenancePrerequisites";
import { eventHub, EVENT_REQUEST_PERFORM_GC_V3 } from "@/common/events";
function createPrerequisites(settingsOverride: Partial<typeof DEFAULT_SETTINGS> = {}) { function createPrerequisites(settingsOverride: Partial<typeof DEFAULT_SETTINGS> = {}) {
const askSelectStringDialogue = vi.fn( const askSelectStringDialogue = vi.fn(
@@ -63,7 +64,7 @@ function createPrerequisites(settingsOverride: Partial<typeof DEFAULT_SETTINGS>
} }
describe("LocalDatabaseMaintenance prerequisites", () => { describe("LocalDatabaseMaintenance prerequisites", () => {
it("shows database analysis in Advanced mode and Garbage Collection only in applicable Edge Case mode", () => { it("shows Garbage Collection only in applicable Edge Case mode", () => {
const commands: Array<{ const commands: Array<{
id: string; id: string;
checkCallback?: (checking: boolean) => boolean | void; checkCallback?: (checking: boolean) => boolean | void;
@@ -90,15 +91,15 @@ describe("LocalDatabaseMaintenance prerequisites", () => {
_isDatabaseReady: vi.fn(() => true), _isDatabaseReady: vi.fn(() => true),
}); });
vi.mocked(eventHub.onEvent).mockClear();
maintenance.onload(); maintenance.onload();
expect(eventHub.onEvent).toHaveBeenCalledTimes(1);
expect(eventHub.onEvent).toHaveBeenCalledWith(EVENT_REQUEST_PERFORM_GC_V3, expect.any(Function));
const analyse = commands.find(({ id }) => id === "analyse-database");
const garbageCollect = commands.find(({ id }) => id === "gc-v3"); const garbageCollect = commands.find(({ id }) => id === "gc-v3");
expect(analyse?.checkCallback?.(true)).toBe(false);
expect(garbageCollect?.checkCallback?.(true)).toBe(false); expect(garbageCollect?.checkCallback?.(true)).toBe(false);
settings.useAdvancedMode = true; settings.useAdvancedMode = true;
expect(analyse?.checkCallback?.(true)).toBe(true);
expect(garbageCollect?.checkCallback?.(true)).toBe(false); expect(garbageCollect?.checkCallback?.(true)).toBe(false);
settings.useEdgeCaseMode = true; settings.useEdgeCaseMode = true;
+2
View File
@@ -51,6 +51,7 @@ import { useInteractiveConflictResolutionFeature } from "./serviceFeatures/inter
import { ConflictResolveModal } from "./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts"; import { ConflictResolveModal } from "./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts";
import { useObsidianReplicationRibbonFeature } from "./serviceFeatures/obsidianReplicationRibbon.ts"; import { useObsidianReplicationRibbonFeature } from "./serviceFeatures/obsidianReplicationRibbon.ts";
import { useStartupLifecycleFeature } from "./serviceFeatures/startupLifecycle"; import { useStartupLifecycleFeature } from "./serviceFeatures/startupLifecycle";
import { useDatabaseDiagnosticsFeature } from "./serviceFeatures/useDatabaseDiagnostics";
export type LiveSyncCore = LiveSyncBaseCore<ObsidianServiceContext, LiveSyncCommands>; export type LiveSyncCore = LiveSyncBaseCore<ObsidianServiceContext, LiveSyncCommands>;
export default class ObsidianLiveSyncPlugin extends Plugin { export default class ObsidianLiveSyncPlugin extends Plugin {
core: LiveSyncCore; core: LiveSyncCore;
@@ -195,6 +196,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
useP2PReplicatorCommands(core, replicator); useP2PReplicatorCommands(core, replicator);
useP2PReplicatorUI(core, core, replicator, createInteractiveP2PReplication(replicator)); useP2PReplicatorUI(core, core, replicator, createInteractiveP2PReplication(replicator));
useObsidianReplicationRibbonFeature(core); useObsidianReplicationRibbonFeature(core);
useDatabaseDiagnosticsFeature(core);
useRemoteConfiguration(core); useRemoteConfiguration(core);
useSetupProtocolFeature(core, setupManager); useSetupProtocolFeature(core, setupManager);
@@ -16,6 +16,7 @@ import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts";
import { import {
EVENT_ANALYSE_DB_USAGE, EVENT_ANALYSE_DB_USAGE,
EVENT_INSPECT_DB_INTEGRITY,
EVENT_REQUEST_CHECK_REMOTE_SIZE, EVENT_REQUEST_CHECK_REMOTE_SIZE,
EVENT_REQUEST_RUN_DOCTOR, EVENT_REQUEST_RUN_DOCTOR,
EVENT_REQUEST_RUN_FIX_INCOMPLETE, EVENT_REQUEST_RUN_FIX_INCOMPLETE,
@@ -113,6 +114,14 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement,
await chooseAndCopyFileDatabaseInfo(this.core); await chooseAndCopyFileDatabaseInfo(this.core);
}) })
); );
new Setting(paneEl)
.setName($msg("Inspect database integrity"))
.setDesc(
$msg("Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.")
)
.addButton((button) =>
button.setButtonText($msg("Inspect")).onClick(() => eventHub.emitEvent(EVENT_INSPECT_DB_INTEGRITY))
);
new Setting(paneEl) new Setting(paneEl)
.setName($msg("Analyse database usage")) .setName($msg("Analyse database usage"))
.setDesc( .setDesc(
@@ -0,0 +1,148 @@
<script lang="ts">
import { onDestroy, onMount } from "svelte";
import type { GuestDialogProps } from "@/modules/services/LiveSyncUI/svelteDialog";
import DialogHeader from "@/modules/services/LiveSyncUI/components/DialogHeader.svelte";
import { $msg as translate } from "@/common/translation";
import type { DatabaseDiagnosticDialogOptions } from "@/serviceFeatures/databaseDiagnosticDialogTypes";
import type { DatabaseDiagnosticOutput } from "@/serviceFeatures/databaseDiagnostics";
import type { DiagnosticNameMode } from "@vrtmrz/livesync-commonlib/diagnostics";
const { setResult, getInitialData }: GuestDialogProps<"close", DatabaseDiagnosticDialogOptions> = $props();
let options = $state<DatabaseDiagnosticDialogOptions>();
let phase = $state<"choice" | "running" | "result" | "failed">("choice");
let nameMode = $state<DiagnosticNameMode>("pseudonym");
let processed = $state(0);
let cancelling = $state(false);
let output = $state<DatabaseDiagnosticOutput>();
let selectedFormat = $state("");
let actionStatus = $state("");
let disposed = false;
const selected = $derived(output?.artifacts.find((artifact) => artifact.format === selectedFormat));
const title = $derived(
options?.kind === "integrity" ? translate("Inspect database integrity") : translate("Analyse database usage")
);
const statusText = $derived(
output
? {
complete: translate("Inspection completed. Review the findings and coverage before sharing."),
partial: translate("Inspection partially completed. Some information could not be checked; review the reported reasons."),
failed: translate("Inspection failed. The report records the available evidence and reasons."),
cancelled: translate("Inspection cancelled. The report contains only the evidence collected so far."),
}[output.status]
: ""
);
onMount(() => {
options = getInitialData?.();
});
onDestroy(() => {
disposed = true;
options?.cancel();
});
async function begin() {
if (!options || phase !== "choice") return;
phase = "running";
try {
const result = await options.run(nameMode, (count) => {
if (!disposed) processed = count;
});
if (disposed) return;
output = result;
selectedFormat = result.artifacts[0].format;
phase = "result";
} catch {
if (!disposed) phase = "failed";
}
}
async function copyReport(event: MouseEvent) {
if (!selected) return;
const owner = (event.currentTarget as HTMLElement).ownerDocument.defaultView;
try {
await owner?.navigator.clipboard.writeText(selected.content);
actionStatus = translate("Report copied.");
} catch {
actionStatus = translate("Copy failed. Select the report text and copy it manually.");
}
}
function saveReport(event: MouseEvent) {
if (!selected) return;
const document = (event.currentTarget as HTMLElement).ownerDocument;
const url = URL.createObjectURL(new Blob([selected.content], { type: "text/plain;charset=utf-8" }));
const link = document.createElement("a");
link.href = url;
link.download = selected.filename;
document.body.appendChild(link);
link.click();
link.remove();
setTimeout(() => URL.revokeObjectURL(url), 1000);
actionStatus = translate("Download requested. You can also copy the report if downloads are unavailable on this device.");
}
</script>
<DialogHeader {title} />
<div class="database-diagnostic" data-diagnostic-phase={phase}>
{#if phase === "choice"}
<p>{translate("Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.")}</p>
<p>
{options?.kind === "usage"
? translate("Read available local revisions and report their Chunk references. Settings are left unchanged.")
: translate("Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.")}
</p>
<label>
{translate("Names in this report")}
<select bind:value={nameMode}>
<option value="pseudonym">{translate("Pseudonyms (recommended)")}</option>
<option value="original">{translate("Original names")}</option>
</select>
</label>
<div class="diagnostic-controls">
<button onclick={() => setResult("close")}>{translate("Cancel")}</button>
<button class="mod-cta" disabled={!options} onclick={() => void begin()}>{translate("Begin inspection")}</button>
</div>
{:else if phase === "running"}
<p role="status">
{cancelling ? translate("Cancelling inspection…") : `${translate("Items inspected")}: ${processed}`}
</p>
<button disabled={cancelling} onclick={() => { cancelling = true; options?.cancel(); }}>
{translate("Cancel inspection")}
</button>
{:else if phase === "result" && output}
<p role="status">{statusText}</p>
<p>
{output.nameMode === "pseudonym"
? translate("This report uses temporary pseudonyms. Counts and relationships remain visible.")
: translate("This report contains original paths and identifiers. Review them before sharing.")}
</p>
{#if options?.kind === "usage" && output.status !== "complete"}
<p>{translate("Partial Usage results are available as JSON so their coverage and reasons stay with the rows.")}</p>
{/if}
<label>
{translate("Report format")}
<select bind:value={selectedFormat}>
{#each output.artifacts as artifact}
<option value={artifact.format}>{artifact.format.toUpperCase()}</option>
{/each}
</select>
</label>
<textarea readonly rows="16" aria-label={translate("Diagnostic report")} value={selected?.content ?? ""}></textarea>
<p role="status">{actionStatus}</p>
<div class="diagnostic-controls">
<button onclick={(event) => void copyReport(event)}>{translate("Copy report")}</button>
<button onclick={saveReport}>{translate("Save report")}</button>
<button onclick={() => setResult("close")}>{translate("Close")}</button>
</div>
{:else if phase === "failed"}
<p role="status">{translate("The inspection could not finish. Close this dialogue and try again after database startup.")}</p>
<button onclick={() => setResult("close")}>{translate("Close")}</button>
{/if}
</div>
<style>
.database-diagnostic { display: flex; flex-direction: column; gap: var(--size-4-2); }
label { display: flex; align-items: center; flex-wrap: wrap; gap: var(--size-4-3); }
.diagnostic-controls { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: var(--size-4-2); }
button, select { min-height: 44px; }
textarea { width: 100%; resize: vertical; font-family: var(--font-monospace); font-size: var(--font-ui-small); }
</style>
@@ -0,0 +1,9 @@
import type { DiagnosticNameMode } from "@vrtmrz/livesync-commonlib/diagnostics";
import type { DatabaseDiagnosticKind, DatabaseDiagnosticOutput } from "./databaseDiagnostics";
/** The dialogue can run or cancel an inspection without receiving the service container. */
export interface DatabaseDiagnosticDialogOptions {
kind: DatabaseDiagnosticKind;
run(nameMode: DiagnosticNameMode, onProgress: (processed: number) => void): Promise<DatabaseDiagnosticOutput>;
cancel(): void;
}
+163
View File
@@ -0,0 +1,163 @@
import {
analyseDatabaseIntegrity,
analyseDatabaseUsage,
createCouchDBDiagnosticSource,
createDiagnosticNameMapper,
createLocalDiagnosticSource,
formatDiagnosticJSON,
formatIntegrityMarkdown,
formatUsageTSV,
projectIntegrityReport,
projectUsageReport,
type DiagnosticContent,
type DiagnosticNameMode,
type DiagnosticStatus,
} from "@vrtmrz/livesync-commonlib/diagnostics";
import { REMOTE_COUCHDB, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
import type { NecessaryServicesInterfaces } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule";
import commonlibPackage from "@vrtmrz/livesync-commonlib/package.json";
export type DatabaseDiagnosticKind = "usage" | "integrity";
export type DatabaseDiagnosticHost = NecessaryServicesInterfaces<
"setting" | "database" | "path" | "API",
"storageAccess"
>;
export interface DatabaseDiagnosticArtifact {
format: "json" | "markdown" | "tsv";
filename: string;
content: string;
}
export interface DatabaseDiagnosticOutput {
status: DiagnosticStatus;
nameMode: DiagnosticNameMode;
artifacts: DatabaseDiagnosticArtifact[];
}
export async function readDiagnosticStorage(
storage: Pick<StorageAccess, "statHidden" | "readHiddenFileBinary">,
path: string
): Promise<DiagnosticContent> {
if (/^(?:[/\\]|[a-zA-Z]:)/.test(path) || path.includes("\0") || path.split(/[/\\]/).includes("..")) {
return { status: "unreadable", reason: "content-unreadable" };
}
try {
const before = await storage.statHidden(path);
if (!before) return { status: "missing" };
if (before.type !== "file") return { status: "unreadable", reason: "content-unreadable" };
const bytes = new Uint8Array(await storage.readHiddenFileBinary(path));
const after = await storage.statHidden(path);
if (
!after ||
after.type !== "file" ||
before.mtime !== after.mtime ||
before.ctime !== after.ctime ||
before.size !== after.size ||
bytes.byteLength !== after.size
) {
return { status: "unreadable", reason: "storage-changed" };
}
return { status: "available", bytes };
} catch {
return { status: "unreadable", reason: "content-unreadable" };
}
}
/** Run only after the host has collected this run's name-display choice. */
export async function runDatabaseDiagnostic(
host: DatabaseDiagnosticHost,
kind: DatabaseDiagnosticKind,
nameMode: DiagnosticNameMode,
options: { signal?: AbortSignal; onProgress?(processed: number): void } = {}
): Promise<DatabaseDiagnosticOutput> {
const configuration = JSON.stringify(host.services.setting.currentSettings());
const settings = JSON.parse(configuration) as ObsidianLiveSyncSettings;
const database = host.services.database.localDatabase.localDatabase;
const isConfigurationCurrent = () =>
JSON.stringify(host.services.setting.currentSettings()) === configuration &&
host.services.database.localDatabase.localDatabase === database;
const mapper = createDiagnosticNameMapper(nameMode);
const versions = {
commonlib: commonlibPackage.version,
plugin: host.services.API.getPluginVersion(),
application: host.services.API.getAppVersion(),
};
const local = createLocalDiagnosticSource(database);
if (kind === "usage") {
const result = await analyseDatabaseUsage({
source: local,
signal: options.signal,
onProgress: (progress) => options.onProgress?.(progress.revisionsProcessed),
});
if (!isConfigurationCurrent()) {
result.errors.push({ code: "configuration_changed" });
if (result.status === "complete") result.status = "partial";
}
const report = projectUsageReport(result, mapper, versions);
const artifacts: DatabaseDiagnosticArtifact[] = [
{
format: "json",
filename: `database-usage-${result.status}.json`,
content: formatDiagnosticJSON(report),
},
];
// A standalone legacy TSV has no coverage fields. Partial runs retain
// their rows in JSON together with the status and reasons.
if (result.status === "complete") {
artifacts.unshift({ format: "tsv", filename: "database-usage.tsv", content: formatUsageTSV(report) });
}
return { status: result.status, nameMode, artifacts };
}
const result = await analyseDatabaseIntegrity({
local,
remote:
settings.remoteType === REMOTE_COUCHDB
? () =>
Promise.resolve(
createCouchDBDiagnosticSource({
settings,
signal: options.signal,
fetch: (request, requestOptions) =>
settings.useRequestAPI
? host.services.API.nativeFetch(request, requestOptions)
: host.services.API.webCompatFetch(request, requestOptions),
})
)
: undefined,
remoteUnavailableReason: settings.remoteType === REMOTE_COUCHDB ? "source-unavailable" : "unsupported-remote",
path: {
getPath: (entry) => host.services.path.getPath(entry),
path2id: (path, prefix) => host.services.path.path2idWithSettings(path, settings, prefix),
},
settings: {
encrypt: settings.encrypt,
usePathObfuscation: settings.usePathObfuscation,
handleFilenameCaseSensitive: settings.handleFilenameCaseSensitive,
idDerivationVersion: settings.idDerivationVersion ?? 0,
pathNormalizationVersion: settings.pathNormalizationVersion ?? 0,
},
readStorage: (path) => readDiagnosticStorage(host.serviceModules.storageAccess, path),
isConfigurationCurrent,
signal: options.signal,
onProgress: (progress) => options.onProgress?.(progress.processed),
});
const report = projectIntegrityReport(result, mapper, versions);
return {
status: result.status,
nameMode,
artifacts: [
{
format: "markdown",
filename: `database-integrity-${result.status}.md`,
content: formatIntegrityMarkdown(report),
},
{
format: "json",
filename: `database-integrity-${result.status}.json`,
content: formatDiagnosticJSON(report),
},
],
};
}
@@ -0,0 +1,213 @@
import { describe, expect, it, vi } from "vitest";
import { DEFAULT_SETTINGS, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { readDiagnosticStorage, runDatabaseDiagnostic, type DatabaseDiagnosticHost } from "./databaseDiagnostics";
function fixture() {
const path = "Private/日本語.md";
const body = "private file contents";
const bytes = new TextEncoder().encode(body);
const settings = {
...DEFAULT_SETTINGS,
remoteType: REMOTE_P2P,
readChunksOnline: true,
passphrase: "private encryption passphrase",
couchDB_PASSWORD: "private credential",
couchDB_URI: "https://private-server.invalid",
};
const documents = {
"f:private-actual-id": {
_id: "f:private-actual-id",
_rev: "1-private-revision",
_revs_info: [{ rev: "1-private-revision", status: "available" }],
type: "plain",
path,
ctime: 1,
mtime: 1,
size: bytes.byteLength,
children: ["h:private-chunk"],
},
"h:private-chunk": {
_id: "h:private-chunk",
_rev: "1-private-chunk-revision",
_revs_info: [{ rev: "1-private-chunk-revision", status: "available" }],
type: "leaf",
data: body,
},
};
const get = vi.fn(async (id: string) => {
const doc = documents[id as keyof typeof documents];
if (!doc) throw { status: 404 };
return structuredClone(doc);
});
const info = vi.fn(async () => ({ update_seq: "1-stable" }));
const put = vi.fn(() => {
throw new Error("Diagnostic attempted a write");
});
const applyPartial = vi.fn();
const fetch = vi.fn();
const storage = {
statHidden: vi.fn(async () => ({ type: "file" as const, ctime: 1, mtime: 1, size: bytes.byteLength })),
readHiddenFileBinary: vi.fn(async () => bytes.buffer),
};
const host = {
services: {
setting: { currentSettings: () => settings, applyPartial },
database: {
localDatabase: {
localDatabase: {
info,
get,
put,
allDocs: vi.fn(async () => ({
rows: Object.keys(documents).map((id) => ({ id, value: { rev: "1-current" } })),
})),
},
},
},
path: {
getPath: (doc: { path: string }) => doc.path,
path2idWithSettings: vi.fn(async (rawPath: string) => `f:expected:${rawPath}`),
},
API: {
webCompatFetch: fetch,
nativeFetch: fetch,
getPluginVersion: () => "1.0.38",
getAppVersion: () => "1.12.7",
},
},
serviceModules: { storageAccess: storage },
} as unknown as DatabaseDiagnosticHost;
return { host, settings, path, body, bytes, get, info, put, applyPartial, fetch, storage };
}
describe("database diagnostic host composition", () => {
it("keeps on-demand fetching enabled while analysing only existing local revisions", async () => {
const data = fixture();
const before = JSON.stringify(data.settings);
const result = await runDatabaseDiagnostic(data.host, "usage", "pseudonym");
expect(result.status).toBe("complete");
expect(result.artifacts.map((artifact) => artifact.format)).toEqual(["tsv", "json"]);
expect(data.settings.readChunksOnline).toBe(true);
expect(JSON.stringify(data.settings)).toBe(before);
expect(data.applyPartial).not.toHaveBeenCalled();
expect(data.fetch).not.toHaveBeenCalled();
expect(data.put).not.toHaveBeenCalled();
expect(data.get).toHaveBeenCalledWith("f:private-actual-id", { rev: "1-private-revision" });
const report = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content);
expect(report.versions).toMatchObject({ plugin: "1.0.38", application: "1.12.7" });
expect(report.versions.commonlib).toMatch(/^\d+\.\d+\.\d+/);
for (const artifact of result.artifacts) {
for (const secret of [
data.path,
data.body,
data.settings.passphrase,
"private-actual-id",
"private-revision",
"private-server.invalid",
]) {
expect(artifact.content).not.toContain(secret);
}
}
});
it("keeps local findings when the remote type is unsupported and projects both formats consistently", async () => {
const data = fixture();
const result = await runDatabaseDiagnostic(data.host, "integrity", "pseudonym");
expect(result.status).toBe("partial");
expect(data.fetch).not.toHaveBeenCalled();
expect(data.put).not.toHaveBeenCalled();
expect(data.storage.readHiddenFileBinary).toHaveBeenCalledWith(data.path);
const json = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content);
expect(json.errors).toContainEqual({ source: "remote", reason: "unsupported-remote" });
expect(json.observations[0]).toMatchObject({
identity: "document-id-mismatch",
content: { status: "available" },
});
expect(json.observations[0].comparisons).toContainEqual({
against: json.observations[0].vault.reference,
relation: "equal",
});
for (const artifact of result.artifacts) {
expect(artifact.content).toContain(json.observations[0].path);
expect(artifact.content).not.toContain(data.path);
expect(artifact.content).not.toContain(data.body);
expect(artifact.content).not.toContain(data.settings.passphrase);
}
});
it("includes names only when requested and never includes credentials or file contents", async () => {
const data = fixture();
const result = await runDatabaseDiagnostic(data.host, "usage", "original");
for (const artifact of result.artifacts) {
expect(artifact.content).toContain(data.path);
expect(artifact.content).toContain("f:private-actual-id");
expect(artifact.content).not.toContain(data.body);
expect(artifact.content).not.toContain(data.settings.couchDB_PASSWORD);
expect(artifact.content).not.toContain(data.settings.couchDB_URI);
}
});
it("retains cancellation and configuration changes in JSON instead of presenting a complete TSV", async () => {
const cancelled = fixture();
const controller = new AbortController();
controller.abort();
const stopped = await runDatabaseDiagnostic(cancelled.host, "usage", "pseudonym", {
signal: controller.signal,
});
expect(stopped.status).toBe("cancelled");
expect(stopped.artifacts.map((artifact) => artifact.format)).toEqual(["json"]);
expect(cancelled.get).not.toHaveBeenCalled();
const changed = fixture();
changed.info.mockImplementationOnce(async () => ({ update_seq: "1-stable" }));
changed.info.mockImplementationOnce(async () => {
changed.settings.couchDB_URI = "https://changed.invalid";
return { update_seq: "1-stable" };
});
const output = await runDatabaseDiagnostic(changed.host, "usage", "original");
expect(output.status).toBe("partial");
expect(output.artifacts.map((artifact) => artifact.format)).toEqual(["json"]);
expect(JSON.parse(output.artifacts[0].content).errors).toContainEqual({ code: "configuration_changed" });
});
});
describe("diagnostic Vault reads", () => {
it("distinguishes an absent file, a read error, and a file changing during its read", async () => {
const data = fixture();
data.storage.statHidden.mockResolvedValueOnce(null as never);
expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ status: "missing" });
data.storage.readHiddenFileBinary.mockRejectedValueOnce(new Error("private filename and content"));
expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({
status: "unreadable",
reason: "content-unreadable",
});
data.storage.statHidden.mockResolvedValueOnce({
type: "file",
ctime: 1,
mtime: 1,
size: data.bytes.byteLength,
});
data.storage.statHidden.mockResolvedValueOnce({
type: "file",
ctime: 1,
mtime: 2,
size: data.bytes.byteLength,
});
expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({
status: "unreadable",
reason: "storage-changed",
});
});
it("rejects paths outside the Vault before touching storage", async () => {
const data = fixture();
for (const path of ["../private", "/private", "C:\\private", "x/../private", "x\0y"]) {
expect(await readDiagnosticStorage(data.storage, path)).toEqual({
status: "unreadable",
reason: "content-unreadable",
});
}
expect(data.storage.statHidden).not.toHaveBeenCalled();
expect(data.storage.readHiddenFileBinary).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,76 @@
import {
createServiceFeature,
type NecessaryServices,
} from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule";
import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events";
import { $msg } from "@/common/translation";
import DatabaseDiagnosticDialog from "@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte";
import { runDatabaseDiagnostic, type DatabaseDiagnosticKind } from "./databaseDiagnostics";
import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes";
type DatabaseDiagnosticsFeatureHost = NecessaryServices<
"setting" | "database" | "path" | "API" | "UI" | "appLifecycle",
"storageAccess"
>;
export const useDatabaseDiagnosticsFeature = createServiceFeature((host: DatabaseDiagnosticsFeatureHost) => {
const { services } = host;
let active: AbortController | undefined;
const open = async (kind: DatabaseDiagnosticKind): Promise<void> => {
if (active) return;
const controller = new AbortController();
active = controller;
try {
await services.UI.dialogManager.open<"close", DatabaseDiagnosticDialogOptions>(DatabaseDiagnosticDialog, {
kind,
run: async (nameMode, onProgress) => {
if (!services.database.isDatabaseReady()) throw new Error("local-database-unavailable");
return await runDatabaseDiagnostic(host, kind, nameMode, {
signal: controller.signal,
onProgress,
});
},
cancel: () => controller.abort(),
});
} catch {
// Closing the host during unload can reject the dialogue result.
// Diagnostic errors are presented inside the dialogue without raw logs.
} finally {
controller.abort();
if (active === controller) active = undefined;
}
};
services.appLifecycle.onInitialise.addHandler(() => {
services.API.addCommand({
id: "analyse-database",
name: "Analyse Database Usage (advanced)",
icon: "database-search",
checkCallback: (checking) => {
if (!services.setting.currentSettings().useAdvancedMode || !services.database.isDatabaseReady())
return false;
if (!checking) void open("usage");
return true;
},
});
services.API.addCommand({
id: "inspect-database-integrity",
name: $msg("Inspect database integrity"),
icon: "database-search",
checkCallback: (checking) => {
if (!services.database.isDatabaseReady()) return false;
if (!checking) void open("integrity");
return true;
},
});
return Promise.resolve(true);
});
const stopUsage = services.context.events.onEvent(EVENT_ANALYSE_DB_USAGE, () => void open("usage"));
const stopIntegrity = services.context.events.onEvent(EVENT_INSPECT_DB_INTEGRITY, () => void open("integrity"));
services.appLifecycle.onUnload.addHandler(() => {
active?.abort();
stopUsage();
stopIntegrity();
return Promise.resolve(true);
});
return { open };
});
@@ -0,0 +1,122 @@
import { describe, expect, it, vi } from "vitest";
import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes";
vi.mock("@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte", () => ({ default: {} }));
vi.mock("./databaseDiagnostics", () => ({
runDatabaseDiagnostic: vi.fn(async () => ({ status: "complete", nameMode: "pseudonym", artifacts: [] })),
}));
vi.mock("@/common/translation", () => ({ $msg: (value: string) => value }));
import { runDatabaseDiagnostic } from "./databaseDiagnostics";
import { useDatabaseDiagnosticsFeature } from "./useDatabaseDiagnostics";
import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events";
function fixture() {
const initialise: Array<() => Promise<boolean>> = [];
const unload: Array<() => Promise<boolean>> = [];
const commands: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = [];
const listeners = new Map<string, () => void>();
const unsubscribe = vi.fn((event: string) => listeners.delete(event));
const settings = { useAdvancedMode: false };
let ready = true;
let close!: () => void;
const openDialog = vi.fn(
(_component: unknown, _options: DatabaseDiagnosticDialogOptions) =>
new Promise<void>((resolve) => {
close = resolve;
})
);
const host = {
services: {
setting: { currentSettings: () => settings },
database: { isDatabaseReady: () => ready },
path: {},
API: { addCommand: (command: (typeof commands)[number]) => commands.push(command) },
UI: { dialogManager: { open: openDialog } },
appLifecycle: {
onInitialise: { addHandler: (handler: (typeof initialise)[number]) => initialise.push(handler) },
onUnload: { addHandler: (handler: (typeof unload)[number]) => unload.push(handler) },
},
context: {
events: {
onEvent: (event: string, handler: () => void) => {
listeners.set(event, handler);
return () => unsubscribe(event);
},
},
},
},
serviceModules: { storageAccess: {} },
} as unknown as Parameters<typeof useDatabaseDiagnosticsFeature>[0];
const view = useDatabaseDiagnosticsFeature(host);
return {
host,
view,
initialise,
unload,
commands,
listeners,
settings,
openDialog,
unsubscribe,
close: () => close(),
setReady: (value: boolean) => {
ready = value;
},
};
}
describe("database diagnostics serviceFeature", () => {
it("retains the existing Advanced-mode Usage command gate in its new owner", async () => {
const data = fixture();
for (const initialise of data.initialise) await initialise();
const usage = data.commands.find((command) => command.id === "analyse-database")!;
const integrity = data.commands.find((command) => command.id === "inspect-database-integrity")!;
expect(usage.checkCallback(true)).toBe(false);
expect(integrity.checkCallback(true)).toBe(true);
data.settings.useAdvancedMode = true;
expect(usage.checkCallback(true)).toBe(true);
data.setReady(false);
expect(usage.checkCallback(true)).toBe(false);
expect(integrity.checkCallback(true)).toBe(false);
expect(data.openDialog).not.toHaveBeenCalled();
});
it("uses the existing dialogue manager and performs no analysis until the run choice is submitted", async () => {
vi.mocked(runDatabaseDiagnostic).mockClear();
const data = fixture();
const opened = data.view.open("usage");
expect(data.openDialog).toHaveBeenCalledTimes(1);
expect(runDatabaseDiagnostic).not.toHaveBeenCalled();
const options = data.openDialog.mock.calls[0][1];
expect(Object.keys(options).sort()).toEqual(["cancel", "kind", "run"]);
await options.run("original", () => {});
expect(runDatabaseDiagnostic).toHaveBeenCalledWith(
data.host,
"usage",
"original",
expect.objectContaining({ signal: expect.any(AbortSignal) })
);
const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!;
await data.view.open("integrity");
expect(data.openDialog).toHaveBeenCalledTimes(1);
data.close();
await opened;
expect(signal.aborted).toBe(true);
});
it("wires both Hatch actions and releases its listeners and active request on unload", async () => {
const data = fixture();
expect(data.listeners.has(EVENT_ANALYSE_DB_USAGE)).toBe(true);
expect(data.listeners.has(EVENT_INSPECT_DB_INTEGRITY)).toBe(true);
const opened = data.view.open("integrity");
await data.openDialog.mock.calls[0][1].run("pseudonym", () => {});
const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!;
for (const unload of data.unload) await unload();
expect(signal.aborted).toBe(true);
expect(data.listeners.size).toBe(0);
expect(data.unsubscribe).toHaveBeenCalledTimes(2);
data.close();
await opened;
});
});
+3
View File
@@ -74,6 +74,7 @@ After changing plug-in source, use the focused wrapper rather than invoking a sc
```bash ```bash
npm run test:e2e:obsidian:focused -- settings-ui npm run test:e2e:obsidian:focused -- settings-ui
npm run test:e2e:obsidian:focused -- database-diagnostics
npm run test:e2e:obsidian:focused -- configuration-encryption npm run test:e2e:obsidian:focused -- configuration-encryption
npm run test:e2e:obsidian:focused -- configuration-encryption-sync npm run test:e2e:obsidian:focused -- configuration-encryption-sync
npm run test:e2e:obsidian:focused -- two-vault-sync npm run test:e2e:obsidian:focused -- two-vault-sync
@@ -85,6 +86,8 @@ npm run test:e2e:obsidian:focused -- couchdb-http-cache
The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite.
`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including the choice before inspection, original names and pseudonyms, retained local findings when the remote is unavailable, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice dialogue in mobile emulation. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. Commonlib's separate CouchDB integration test verifies encrypted reads and missing Security Seed handling against a real server.
`folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems.
`stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application.
@@ -0,0 +1,237 @@
import assert from "node:assert/strict";
import type { Page } from "playwright";
import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc";
import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts";
import { evalObsidianJson } from "../runner/cli.ts";
import {
createE2eObsidianDeviceLocalState,
waitForLiveSyncCoreReady,
waitForLocalDatabaseEntry,
} from "../runner/liveSyncWorkflow.ts";
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
import { captureObsidianElement, captureObsidianPage, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts";
import { assertMobileDialogueLayout, setObsidianMobileTestMode } from "../runner/mobileUi.ts";
import { createTemporaryVault } from "../runner/vault.ts";
const fixturePath = "Private/diagnostic-秘密.md";
const fixtureContent = "Diagnostic fixture content which must never appear in a report.\n";
const shadowId = "diagnostic-shadow-document";
const timeout = 15000;
type DiagnosticPageGlobal = typeof globalThis & {
app: { commands: { executeCommandById(id: string): boolean } };
diagnosticClipboard?: string;
diagnosticDownload?: { name: string; content: string };
};
async function openCommand(page: Page, kind: "usage" | "integrity") {
const id = kind === "usage" ? "analyse-database" : "inspect-database-integrity";
const opened = await page.evaluate(
(command) => (globalThis as DiagnosticPageGlobal).app.commands.executeCommandById(command),
`obsidian-livesync:${id}`
);
assert.equal(opened, true);
const dialogue = page.locator(".database-diagnostic");
await dialogue.locator("select").selectOption("pseudonym");
assert.equal(await dialogue.getAttribute("data-diagnostic-phase"), "choice");
assert.equal(await dialogue.locator("textarea").count(), 0);
return dialogue;
}
async function readDatabaseState(cli: string, session: ObsidianLiveSyncSession) {
return await evalObsidianJson<{ sequence: unknown; settings: string; names: string[] }>(
cli,
"(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;return JSON.stringify({sequence:(await core.localDatabase.localDatabase.info()).update_seq,settings:JSON.stringify(core.services.setting.currentSettings()),names:app.vault.getFiles().map(f=>f.path).sort()});})()",
session.cliEnv
);
}
async function main(): Promise<void> {
const binary = requireObsidianBinary();
const cli = requireObsidianCli();
const vault = await createTemporaryVault("obsidian-livesync-diagnostics-");
let session: ObsidianLiveSyncSession | undefined;
try {
session = await startObsidianLiveSyncSession({
binary,
cliBinary: cli,
vault,
pluginData: {
displayLanguage: "def",
doctorProcessedVersion: DoctorRegulation.version,
isConfigured: true,
useAdvancedMode: true,
readChunksOnline: true,
liveSync: false,
remoteType: "",
couchDB_URI: "",
couchDB_DBNAME: "",
couchDB_USER: "",
couchDB_PASSWORD: "",
remoteConfigurations: {},
activeConfigurationId: "",
notifyThresholdOfRemoteStorageSize: -1,
periodicReplication: false,
syncAfterMerge: false,
syncOnEditorSave: false,
syncOnFileOpen: false,
syncOnSave: false,
syncOnStart: false,
useEden: false,
},
localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
});
await waitForLiveSyncCoreReady(cli, session.cliEnv);
await evalObsidianJson(
cli,
`(async()=>{await app.vault.createFolder('Private');await app.vault.create(${JSON.stringify(fixturePath)},${JSON.stringify(fixtureContent)});return JSON.stringify(true);})()`,
session.cliEnv
);
await waitForLocalDatabaseEntry(cli, session.cliEnv, fixturePath);
await evalObsidianJson(
cli,
`(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;await core.services.fileProcessing.commitPendingFileEvents();const db=core.localDatabase.localDatabase;const id=await core.services.path.path2id(${JSON.stringify(fixturePath)});const original=await db.get(id);const shadow={...original,_id:${JSON.stringify(shadowId)}};delete shadow._rev;await db.put(shadow);return JSON.stringify(true);})()`,
session.cliEnv
);
const before = await readDatabaseState(cli, session);
let usageJSON = "";
await withObsidianPage(session.remoteDebuggingPort, async (page) => {
const choice = await openCommand(page, "usage");
await choice.getByRole("button", { name: "Cancel", exact: true }).click();
await choice.waitFor({ state: "hidden", timeout });
const dialogue = await openCommand(page, "usage");
assert.equal(await dialogue.locator("select").inputValue(), "pseudonym");
await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click();
await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout });
const tsv = await dialogue.locator("textarea").inputValue();
assert.equal(tsv.split("\n")[0].split("\t").length, 9);
assert.ok(!tsv.includes(fixturePath));
assert.ok(!tsv.includes(shadowId));
assert.ok(!tsv.includes(fixtureContent.trim()));
await dialogue.locator("select").selectOption("json");
usageJSON = await dialogue.locator("textarea").inputValue();
const report = JSON.parse(usageJSON);
assert.equal(report.status, "complete");
assert.ok(report.versions.commonlib);
assert.ok(report.rows.some((row: { path: string }) => row.path.startsWith("file-")));
// Capture exports inside the isolated renderer without changing the
// host clipboard or leaving downloads in a user's download directory.
await page.evaluate(`(() => {
Object.defineProperty(navigator.clipboard, "writeText", {
configurable: true,
value: async (text) => { globalThis.diagnosticClipboard = text; },
});
document.addEventListener("click", (event) => {
const link = event.target;
if (!(link instanceof HTMLAnchorElement) || !link.download || !link.href.startsWith("blob:")) return;
event.preventDefault();
void fetch(link.href).then((response) => response.text()).then((content) => {
globalThis.diagnosticDownload = { name: link.download, content };
});
}, true);
})()`);
await dialogue.getByRole("button", { name: "Copy report", exact: true }).click();
assert.equal(
await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard),
usageJSON
);
await dialogue.getByRole("button", { name: "Save report", exact: true }).click();
await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, {
timeout,
});
assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), {
name: "database-usage-complete.json",
content: usageJSON,
});
await dialogue.getByRole("button", { name: "Close", exact: true }).click();
await dialogue.waitFor({ state: "hidden", timeout });
const original = await openCommand(page, "usage");
await original.locator("select").selectOption("original");
await original.getByRole("button", { name: "Begin inspection", exact: true }).click();
await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout });
const rawTSV = await original.locator("textarea").inputValue();
assert.ok(rawTSV.includes(fixturePath));
assert.ok(rawTSV.includes(shadowId));
assert.ok(!rawTSV.includes(fixtureContent.trim()));
await original.getByRole("button", { name: "Close", exact: true }).click();
await original.waitFor({ state: "hidden", timeout });
const settingsNavigator = await openLiveSyncSettings(page, timeout);
const hatch = await settingsNavigator.openPage("Hatch");
const setting = hatch
.locator(".setting-item")
.filter({ has: page.getByText("Inspect database integrity", { exact: true }) });
await setting.getByRole("button", { name: "Inspect", exact: true }).click();
const integrity = page.locator(".database-diagnostic");
assert.equal(await integrity.locator("select").inputValue(), "pseudonym");
await integrity.getByRole("button", { name: "Begin inspection", exact: true }).click();
await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout });
await integrity.locator("select").selectOption("json");
const reportText = await integrity.locator("textarea").inputValue();
const integrityReport = JSON.parse(reportText);
assert.equal(integrityReport.status, "partial");
assert.ok(
integrityReport.errors.some((error: { reason: string }) => error.reason === "source-unavailable")
);
assert.ok(
integrityReport.observations.some(
(observation: { identity: string }) => observation.identity === "document-id-mismatch"
)
);
assert.ok(
integrityReport.observations.some((observation: { comparisons: { relation: string }[] }) =>
observation.comparisons.some((comparison) => comparison.relation === "equal")
)
);
for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()])
assert.ok(!reportText.includes(privateValue));
});
const desktopScreenshot = await captureObsidianElement(
session.remoteDebuggingPort,
"database-diagnostics-desktop.png",
(page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") })
);
await withObsidianPage(session.remoteDebuggingPort, async (page) => {
await page.locator(".database-diagnostic").getByRole("button", { name: "Close", exact: true }).click();
await page.keyboard.press("Escape");
});
const after = await readDatabaseState(cli, session);
assert.deepEqual(after, before, "Diagnostics must leave settings, revisions, and Vault files unchanged");
await setObsidianMobileTestMode(session.remoteDebuggingPort, true, timeout);
await waitForLiveSyncCoreReady(cli, session.cliEnv);
await withObsidianPage(session.remoteDebuggingPort, async (page) => {
await openCommand(page, "integrity");
});
const mobileScreenshot = await captureObsidianElement(
session.remoteDebuggingPort,
"database-diagnostics-mobile.png",
(page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") })
);
await withObsidianPage(session.remoteDebuggingPort, async (page) => {
const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") });
await assertMobileDialogueLayout(page, modal, "Database diagnostic choice");
await modal.getByRole("button", { name: "Cancel", exact: true }).click();
});
console.log(JSON.stringify({ status: "passed", desktopScreenshot, mobileScreenshot, readOnly: true }));
} catch (error) {
if (session) {
await captureObsidianPage(
session.remoteDebuggingPort,
"database-diagnostics-failure.png",
async () => {}
).catch(() => {});
}
throw error;
} finally {
if (session) await session.app.stop();
await vault.dispose();
}
}
main().catch((error: unknown) => {
console.error(error instanceof Error ? error.stack : error);
process.exitCode = 1;
});
+1
View File
@@ -8,6 +8,7 @@ const focusedScenarios = new Set([
"smoke", "smoke",
"onboarding-invitation", "onboarding-invitation",
"dialog-mounts", "dialog-mounts",
"database-diagnostics",
"conflict-dialog-policy", "conflict-dialog-policy",
"revision-repair", "revision-repair",
"document-history-nav", "document-history-nav",
+5
View File
@@ -12,6 +12,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
## Unreleased ## Unreleased
### Diagnostics
- We can inspect normal-file Metadata and current conflicts in the local database and CouchDB together, with byte comparisons for suspect groups. Partial results retain the available evidence when a source is unavailable. The inspection leaves databases and settings unchanged.
- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons.
## 1.0.38 ## 1.0.38
17th October, 2026 17th October, 2026