From 8aeb7c60eff7742d0cb4ac1f2cce3103e2e78733 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 19 Oct 2026 12:49:02 +0000 Subject: [PATCH] Add read-only database diagnostics --- docs/troubleshooting.md | 6 + package-lock.json | 8 +- package.json | 3 +- src/common/events.ts | 2 + .../messages/LiveSyncProvisionalMessages.ts | 41 +++ .../CmdLocalDatabaseMainte.ts | 228 +---------------- .../CmdLocalDatabaseMainte.unit.spec.ts | 9 +- src/main.ts | 2 + .../features/SettingDialogue/PaneHatch.ts | 9 + .../dialogues/DatabaseDiagnosticDialog.svelte | 148 +++++++++++ .../databaseDiagnosticDialogTypes.ts | 9 + src/serviceFeatures/databaseDiagnostics.ts | 163 ++++++++++++ .../databaseDiagnostics.unit.spec.ts | 213 ++++++++++++++++ src/serviceFeatures/useDatabaseDiagnostics.ts | 76 ++++++ .../useDatabaseDiagnostics.unit.spec.ts | 122 +++++++++ test/e2e-obsidian/README.md | 3 + .../scripts/database-diagnostics.ts | 237 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + updates.md | 5 + 19 files changed, 1049 insertions(+), 236 deletions(-) create mode 100644 src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte create mode 100644 src/serviceFeatures/databaseDiagnosticDialogTypes.ts create mode 100644 src/serviceFeatures/databaseDiagnostics.ts create mode 100644 src/serviceFeatures/databaseDiagnostics.unit.spec.ts create mode 100644 src/serviceFeatures/useDatabaseDiagnostics.ts create mode 100644 src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts create mode 100644 test/e2e-obsidian/scripts/database-diagnostics.ts diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b4a2decc..5bdfe39d 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -163,6 +163,12 @@ Enable Obsidian's `Detect all file extensions`, then check LiveSync selectors, i ## Collect a report +For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. The inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete. + +The result dialogue provides Markdown and JSON reports with the same name policy. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. They are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled. + +**Analyse database usage** offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads. + Follow [How to report an issue](to_issue_reporting.md) for the complete reporting checklist, including Obsidian debug information and the privacy review required before sharing evidence. Run `Generate full report for opening the issue with debug info` to copy the current settings summary and recent verbose log lines. Remove credentials, remote URLs, Vault names, file contents, and other private information before sharing it. diff --git a/package-lock.json b/package-lock.json index 583f5928..fcfe4617 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.42", + "@vrtmrz/livesync-commonlib": "0.1.43", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4389,9 +4389,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.42", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.42.tgz", - "integrity": "sha512-LyWl0Mkrj9UsQXV0nOUaYJybcN3wLkpAAxCeHMZYOEizaUTQ7ezN+FgzN+OQxDRA8UXWnoatxh719NWzOwBpBw==", + "version": "0.1.43", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.43.tgz", + "integrity": "sha512-WM2k545hT0ctLv64WdKZ0PkHwyw9Fnlf9xssH7Uv9C0HEMm8CDtj6dfJsPOHs0+Z41Kfco1U/y6R5f5+xWZ00g==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 4f33396b..3103193f 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts", "test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts", "test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts", + "test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts", "test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts", "test:e2e:obsidian:revision-repair": "tsx test/e2e-obsidian/scripts/revision-repair.ts", "test:e2e:obsidian:document-history-nav": "tsx test/e2e-obsidian/scripts/document-history-nav.ts", @@ -191,7 +192,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.42", + "@vrtmrz/livesync-commonlib": "0.1.43", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/src/common/events.ts b/src/common/events.ts index 911cc2ca..d358f997 100644 --- a/src/common/events.ts +++ b/src/common/events.ts @@ -19,6 +19,7 @@ export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor"; export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete"; export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage"; +export const EVENT_INSPECT_DB_INTEGRITY = "inspect-db-integrity"; export const EVENT_REQUEST_PERFORM_GC_V3 = "request-perform-gc-v3"; // export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size"; // export const EVENT_FILE_CHANGED = "file-changed"; @@ -37,6 +38,7 @@ declare global { [EVENT_REQUEST_RUN_DOCTOR]: string; [EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined; [EVENT_ANALYSE_DB_USAGE]: undefined; + [EVENT_INSPECT_DB_INTEGRITY]: undefined; [EVENT_REQUEST_PERFORM_GC_V3]: undefined; } } diff --git a/src/common/messages/LiveSyncProvisionalMessages.ts b/src/common/messages/LiveSyncProvisionalMessages.ts index e4e60160..073cbac7 100644 --- a/src/common/messages/LiveSyncProvisionalMessages.ts +++ b/src/common/messages/LiveSyncProvisionalMessages.ts @@ -251,6 +251,47 @@ export const liveSyncProvisionalEnglishMessages = { "An ID source is required to enable this option.": "An ID source is required to enable this option.", "The ID source or recovery code is invalid. Check it and try again.": "The ID source or recovery code is invalid. Check it and try again.", + "Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.": + "Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.", + Inspect: "Inspect", + "Inspect database integrity": "Inspect database integrity", + "Inspection completed. Review the findings and coverage before sharing.": + "Inspection completed. Review the findings and coverage before sharing.", + "Inspection partially completed. Some information could not be checked; review the reported reasons.": + "Inspection partially completed. Some information could not be checked; review the reported reasons.", + "Inspection failed. The report records the available evidence and reasons.": + "Inspection failed. The report records the available evidence and reasons.", + "Inspection cancelled. The report contains only the evidence collected so far.": + "Inspection cancelled. The report contains only the evidence collected so far.", + "Report copied.": "Report copied.", + "Copy failed. Select the report text and copy it manually.": + "Copy failed. Select the report text and copy it manually.", + "Download requested. You can also copy the report if downloads are unavailable on this device.": + "Download requested. You can also copy the report if downloads are unavailable on this device.", + "Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.": + "Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.", + "Read available local revisions and report their Chunk references. Settings are left unchanged.": + "Read available local revisions and report their Chunk references. Settings are left unchanged.", + "Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.": + "Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.", + "Names in this report": "Names in this report", + "Pseudonyms (recommended)": "Pseudonyms (recommended)", + "Original names": "Original names", + "Cancelling inspection…": "Cancelling inspection…", + "Items inspected": "Items inspected", + "Cancel inspection": "Cancel inspection", + "This report uses temporary pseudonyms. Counts and relationships remain visible.": + "This report uses temporary pseudonyms. Counts and relationships remain visible.", + "This report contains original paths and identifiers. Review them before sharing.": + "This report contains original paths and identifiers. Review them before sharing.", + "Partial Usage results are available as JSON so their coverage and reasons stay with the rows.": + "Partial Usage results are available as JSON so their coverage and reasons stay with the rows.", + "Report format": "Report format", + "Diagnostic report": "Diagnostic report", + "Copy report": "Copy report", + "Save report": "Save report", + "The inspection could not finish. Close this dialogue and try again after database startup.": + "The inspection could not finish. Close this dialogue and try again after database startup.", } as const; export type LiveSyncProvisionalMessageKey = keyof typeof liveSyncProvisionalEnglishMessages; diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts index 85b15d90..955813f3 100644 --- a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts @@ -8,14 +8,13 @@ import { type DocumentID, type EntryDoc, type EntryLeaf, - type FilePathWithPrefix, type MetaEntry, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { getNoFromRev } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; import { LiveSyncCommands } from "@/features/LiveSyncCommands"; import { serialized } from "octagonal-wheels/concurrency/lock_v2"; import { arrayToChunkedArray } from "octagonal-wheels/collection"; -import { EVENT_ANALYSE_DB_USAGE, EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events"; +import { EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events"; import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; import type { ReplicatorInstance } from "@vrtmrz/livesync-commonlib/replication"; import { delay } from "@vrtmrz/livesync-commonlib/compat/common/utils"; @@ -61,18 +60,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands { } onload(): void | Promise { // NO OP. - this.services.API.addCommand({ - id: "analyse-database", - name: "Analyse Database Usage (advanced)", - icon: "database-search", - checkCallback: (checking) => { - if (!this.settings.useAdvancedMode || !this._isDatabaseReady()) return false; - if (!checking) { - void this.analyseDatabase(); - } - return true; - }, - }); this.services.API.addCommand({ id: "gc-v3", name: "Garbage Collection V3 (advanced, beta)", @@ -88,7 +75,6 @@ export class LocalDatabaseMaintenance extends LiveSyncCommands { return true; }, }); - eventHub.onEvent(EVENT_ANALYSE_DB_USAGE, () => this.analyseDatabase()); eventHub.onEvent(EVENT_REQUEST_PERFORM_GC_V3, () => this.gcv3()); } async allChunks(includeDeleted: boolean = false) { @@ -550,218 +536,6 @@ Success: ${successCount}, Errored: ${errored}`; await kvDB.set(DB_KEY_CHUNK_SET, chunkSet); } - // Analyse the database and report chunk usage. - async analyseDatabase() { - if (!(await this.ensureAvailable("Analyse Database Usage"))) return; - const db = this.localDatabase.localDatabase; - // Map of chunk ID to its info - type ChunkInfo = { - id: DocumentID; - refCount: number; - length: number; - }; - const chunkMap = new Map>(); - // Map of document ID to its info - type DocumentInfo = { - id: DocumentID; - rev: Rev; - chunks: Set; - uniqueChunks: Set; - sharedChunks: Set; - path: FilePathWithPrefix; - }; - const docMap = new Map>(); - const info = await db.info(); - // Total number of revisions to process (approximate) - const maxSeq = Number.parseInt(`${info.update_seq ?? 0}`, 10); - let processed = 0; - let read = 0; - let errored = 0; - // Fetch Tasks - const ft = [] as ReturnType[]; - // Fetch a specific revision of a document and make note of its chunks, or add chunk info. - const fetchRevision = async (id: DocumentID, rev: Rev, seq: string | number) => { - try { - processed++; - const doc = await db.get(id, { rev: rev }); - if (doc) { - if ("children" in doc) { - const id = doc._id; - const rev = doc._rev; - const children = (doc.children || []) as DocumentID[]; - const set = docMap.get(id) || new Set(); - set.add({ - id, - rev, - chunks: new Set(children), - uniqueChunks: new Set(), - sharedChunks: new Set(), - path: doc.path, - }); - docMap.set(id, set); - } else if (doc.type === EntryTypes.CHUNK) { - const id = doc._id; - if (chunkMap.has(id)) { - return; - } - if (doc._deleted) { - // Deleted chunk, skip (possibly resurrected later) - return; - } - const length = doc.data.length; - const set = chunkMap.get(id) || new Set(); - set.add({ id, length, refCount: 0 }); - chunkMap.set(id, set); - } - read++; - } else { - this._log(`Analysing Database: not found: ${id} / ${rev}`); - errored++; - } - } catch (error) { - this._log(`Error fetching document ${id} / ${rev}: $`, LOG_LEVEL_NOTICE); - this._log(error, LOG_LEVEL_VERBOSE); - errored++; - } - if (processed % 100 == 0) { - this._log(`Analysing database: ${read} (${errored}) / ${maxSeq} `, LOG_LEVEL_NOTICE, "db-analyse"); - } - }; - - // Enumerate all documents and their revisions. - const IDs = this.localDatabase.findEntryNames("", "", {}); - for await (const id of IDs) { - const revList = await this.localDatabase.getRaw(id as DocumentID, { - revs: true, - revs_info: true, - conflicts: true, - }); - const revInfos = revList._revs_info || []; - for (const revInfo of revInfos) { - // All available revisions should be processed. - // If the revision is not available, it means the revision is already tombstoned. - if (revInfo.status == "available") { - // Schedule fetch task - ft.push(fetchRevision(id as DocumentID, revInfo.rev, 0)); - } - } - } - // Wait for all fetch tasks to complete. - await Promise.all(ft); - // Reference count marking and unique/shared chunk classification. - for (const [, docRevs] of docMap) { - for (const docRev of docRevs) { - for (const chunkId of docRev.chunks) { - const chunkInfos = chunkMap.get(chunkId); - if (chunkInfos) { - for (const chunkInfo of chunkInfos) { - if (chunkInfo.refCount === 0) { - docRev.uniqueChunks.add(chunkId); - } else { - docRev.sharedChunks.add(chunkId); - } - chunkInfo.refCount++; - } - } - } - } - } - // Prepare results - const result = []; - // Calculate total size of chunks in the given set. - const getTotalSize = (ids: Set) => { - return [...ids].reduce((acc, chunkId) => { - const chunkInfos = chunkMap.get(chunkId); - if (chunkInfos) { - for (const chunkInfo of chunkInfos) { - acc += chunkInfo.length; - } - } - return acc; - }, 0); - }; - - // Compile results for each document revision - for (const doc of docMap.values()) { - for (const rev of doc) { - const title = `${rev.path} (${rev.rev})`; - const id = rev.id; - const revStr = `${getNoFromRev(rev.rev)}`; - const revHash = rev.rev.split("-")[1].substring(0, 6); - const path = rev.path; - const uniqueChunkCount = rev.uniqueChunks.size; - const sharedChunkCount = rev.sharedChunks.size; - const uniqueChunkSize = getTotalSize(rev.uniqueChunks); - const sharedChunkSize = getTotalSize(rev.sharedChunks); - result.push({ - title, - path, - rev: revStr, - revHash, - id, - uniqueChunkCount: uniqueChunkCount, - sharedChunkCount, - uniqueChunkSize: uniqueChunkSize, - sharedChunkSize: sharedChunkSize, - }); - } - } - - const titleMap = { - title: "Title", - id: "Document ID", - path: "Path", - rev: "Revision No", - revHash: "Revision Hash", - uniqueChunkCount: "Unique Chunk Count", - sharedChunkCount: "Shared Chunk Count", - uniqueChunkSize: "Unique Chunk Size", - sharedChunkSize: "Shared Chunk Size", - } as const; - // Enumerate orphan chunks (not referenced by any document) - const orphanChunks = [...chunkMap.entries()].filter(([chunkId, infos]) => { - const totalRefCount = [...infos].reduce((acc, info) => acc + info.refCount, 0); - return totalRefCount === 0; - }); - const orphanChunkSize = orphanChunks.reduce((acc, [chunkId, infos]) => { - for (const info of infos) { - acc += info.length; - } - return acc; - }, 0); - result.push({ - title: "__orphan", - id: "__orphan", - path: "__orphan", - rev: "1", - revHash: "xxxxx", - uniqueChunkCount: orphanChunks.length, - sharedChunkCount: 0, - uniqueChunkSize: orphanChunkSize, - sharedChunkSize: 0, - } as const); - - const csvSrc = result.map((e) => { - return [ - `${e.title.replace(/"/g, '""')}"`, - `${e.id}`, - `${e.path}`, - `${e.rev}`, - `${e.revHash}`, - `${e.uniqueChunkCount}`, - `${e.sharedChunkCount}`, - `${e.uniqueChunkSize}`, - `${e.sharedChunkSize}`, - ].join("\t"); - }); - // Add title row - csvSrc.unshift(Object.values(titleMap).join("\t")); - const csv = csvSrc.join("\n"); - - // Prompt to copy to clipboard - await this.services.UI.promptCopyToClipboard("Database Analysis data (TSV):", csv); - } - async compactDatabase() { const replicator = this.core.replicator; if (!canCompactCouchDBRemote(replicator)) return; diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts index 0d337484..89796f4f 100644 --- a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts @@ -42,6 +42,7 @@ import { } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { LocalDatabaseMaintenance } from "./CmdLocalDatabaseMainte"; import { ensureLocalDatabaseMaintenancePrerequisites } from "./maintenancePrerequisites"; +import { eventHub, EVENT_REQUEST_PERFORM_GC_V3 } from "@/common/events"; function createPrerequisites(settingsOverride: Partial = {}) { const askSelectStringDialogue = vi.fn( @@ -63,7 +64,7 @@ function createPrerequisites(settingsOverride: Partial } describe("LocalDatabaseMaintenance prerequisites", () => { - it("shows database analysis in Advanced mode and Garbage Collection only in applicable Edge Case mode", () => { + it("shows Garbage Collection only in applicable Edge Case mode", () => { const commands: Array<{ id: string; checkCallback?: (checking: boolean) => boolean | void; @@ -90,15 +91,15 @@ describe("LocalDatabaseMaintenance prerequisites", () => { _isDatabaseReady: vi.fn(() => true), }); + vi.mocked(eventHub.onEvent).mockClear(); maintenance.onload(); + expect(eventHub.onEvent).toHaveBeenCalledTimes(1); + expect(eventHub.onEvent).toHaveBeenCalledWith(EVENT_REQUEST_PERFORM_GC_V3, expect.any(Function)); - const analyse = commands.find(({ id }) => id === "analyse-database"); const garbageCollect = commands.find(({ id }) => id === "gc-v3"); - expect(analyse?.checkCallback?.(true)).toBe(false); expect(garbageCollect?.checkCallback?.(true)).toBe(false); settings.useAdvancedMode = true; - expect(analyse?.checkCallback?.(true)).toBe(true); expect(garbageCollect?.checkCallback?.(true)).toBe(false); settings.useEdgeCaseMode = true; diff --git a/src/main.ts b/src/main.ts index dd58dcc1..4dcec641 100644 --- a/src/main.ts +++ b/src/main.ts @@ -51,6 +51,7 @@ import { useInteractiveConflictResolutionFeature } from "./serviceFeatures/inter import { ConflictResolveModal } from "./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts"; import { useObsidianReplicationRibbonFeature } from "./serviceFeatures/obsidianReplicationRibbon.ts"; import { useStartupLifecycleFeature } from "./serviceFeatures/startupLifecycle"; +import { useDatabaseDiagnosticsFeature } from "./serviceFeatures/useDatabaseDiagnostics"; export type LiveSyncCore = LiveSyncBaseCore; export default class ObsidianLiveSyncPlugin extends Plugin { core: LiveSyncCore; @@ -195,6 +196,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin { useP2PReplicatorCommands(core, replicator); useP2PReplicatorUI(core, core, replicator, createInteractiveP2PReplication(replicator)); useObsidianReplicationRibbonFeature(core); + useDatabaseDiagnosticsFeature(core); useRemoteConfiguration(core); useSetupProtocolFeature(core, setupManager); diff --git a/src/modules/features/SettingDialogue/PaneHatch.ts b/src/modules/features/SettingDialogue/PaneHatch.ts index d0653ee5..711556ff 100644 --- a/src/modules/features/SettingDialogue/PaneHatch.ts +++ b/src/modules/features/SettingDialogue/PaneHatch.ts @@ -16,6 +16,7 @@ import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; import { EVENT_ANALYSE_DB_USAGE, + EVENT_INSPECT_DB_INTEGRITY, EVENT_REQUEST_CHECK_REMOTE_SIZE, EVENT_REQUEST_RUN_DOCTOR, EVENT_REQUEST_RUN_FIX_INCOMPLETE, @@ -113,6 +114,14 @@ export function paneHatch(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, await chooseAndCopyFileDatabaseInfo(this.core); }) ); + new Setting(paneEl) + .setName($msg("Inspect database integrity")) + .setDesc( + $msg("Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.") + ) + .addButton((button) => + button.setButtonText($msg("Inspect")).onClick(() => eventHub.emitEvent(EVENT_INSPECT_DB_INTEGRITY)) + ); new Setting(paneEl) .setName($msg("Analyse database usage")) .setDesc( diff --git a/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte new file mode 100644 index 00000000..5403d1c9 --- /dev/null +++ b/src/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte @@ -0,0 +1,148 @@ + + + +
+ {#if phase === "choice"} +

{translate("Choose how names appear in this run. Pseudonyms replace paths and identifiers consistently. Original names include private filenames. File contents and credentials are omitted in either mode.")}

+

+ {options?.kind === "usage" + ? translate("Read available local revisions and report their Chunk references. Settings are left unchanged.") + : translate("Read normal-file Metadata, current conflicts, and content for suspect groups from this device and CouchDB. Other remote types and older revision history are outside this inspection. Results do not authorise deletion.")} +

+ +
+ + +
+ {:else if phase === "running"} +

+ {cancelling ? translate("Cancelling inspection…") : `${translate("Items inspected")}: ${processed}`} +

+ + {:else if phase === "result" && output} +

{statusText}

+

+ {output.nameMode === "pseudonym" + ? translate("This report uses temporary pseudonyms. Counts and relationships remain visible.") + : translate("This report contains original paths and identifiers. Review them before sharing.")} +

+ {#if options?.kind === "usage" && output.status !== "complete"} +

{translate("Partial Usage results are available as JSON so their coverage and reasons stay with the rows.")}

+ {/if} + + +

{actionStatus}

+
+ + + +
+ {:else if phase === "failed"} +

{translate("The inspection could not finish. Close this dialogue and try again after database startup.")}

+ + {/if} +
+ + diff --git a/src/serviceFeatures/databaseDiagnosticDialogTypes.ts b/src/serviceFeatures/databaseDiagnosticDialogTypes.ts new file mode 100644 index 00000000..ea142bd6 --- /dev/null +++ b/src/serviceFeatures/databaseDiagnosticDialogTypes.ts @@ -0,0 +1,9 @@ +import type { DiagnosticNameMode } from "@vrtmrz/livesync-commonlib/diagnostics"; +import type { DatabaseDiagnosticKind, DatabaseDiagnosticOutput } from "./databaseDiagnostics"; + +/** The dialogue can run or cancel an inspection without receiving the service container. */ +export interface DatabaseDiagnosticDialogOptions { + kind: DatabaseDiagnosticKind; + run(nameMode: DiagnosticNameMode, onProgress: (processed: number) => void): Promise; + cancel(): void; +} diff --git a/src/serviceFeatures/databaseDiagnostics.ts b/src/serviceFeatures/databaseDiagnostics.ts new file mode 100644 index 00000000..779a8aef --- /dev/null +++ b/src/serviceFeatures/databaseDiagnostics.ts @@ -0,0 +1,163 @@ +import { + analyseDatabaseIntegrity, + analyseDatabaseUsage, + createCouchDBDiagnosticSource, + createDiagnosticNameMapper, + createLocalDiagnosticSource, + formatDiagnosticJSON, + formatIntegrityMarkdown, + formatUsageTSV, + projectIntegrityReport, + projectUsageReport, + type DiagnosticContent, + type DiagnosticNameMode, + type DiagnosticStatus, +} from "@vrtmrz/livesync-commonlib/diagnostics"; +import { REMOTE_COUCHDB, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; +import type { NecessaryServicesInterfaces } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import commonlibPackage from "@vrtmrz/livesync-commonlib/package.json"; + +export type DatabaseDiagnosticKind = "usage" | "integrity"; +export type DatabaseDiagnosticHost = NecessaryServicesInterfaces< + "setting" | "database" | "path" | "API", + "storageAccess" +>; + +export interface DatabaseDiagnosticArtifact { + format: "json" | "markdown" | "tsv"; + filename: string; + content: string; +} + +export interface DatabaseDiagnosticOutput { + status: DiagnosticStatus; + nameMode: DiagnosticNameMode; + artifacts: DatabaseDiagnosticArtifact[]; +} + +export async function readDiagnosticStorage( + storage: Pick, + path: string +): Promise { + if (/^(?:[/\\]|[a-zA-Z]:)/.test(path) || path.includes("\0") || path.split(/[/\\]/).includes("..")) { + return { status: "unreadable", reason: "content-unreadable" }; + } + try { + const before = await storage.statHidden(path); + if (!before) return { status: "missing" }; + if (before.type !== "file") return { status: "unreadable", reason: "content-unreadable" }; + const bytes = new Uint8Array(await storage.readHiddenFileBinary(path)); + const after = await storage.statHidden(path); + if ( + !after || + after.type !== "file" || + before.mtime !== after.mtime || + before.ctime !== after.ctime || + before.size !== after.size || + bytes.byteLength !== after.size + ) { + return { status: "unreadable", reason: "storage-changed" }; + } + return { status: "available", bytes }; + } catch { + return { status: "unreadable", reason: "content-unreadable" }; + } +} + +/** Run only after the host has collected this run's name-display choice. */ +export async function runDatabaseDiagnostic( + host: DatabaseDiagnosticHost, + kind: DatabaseDiagnosticKind, + nameMode: DiagnosticNameMode, + options: { signal?: AbortSignal; onProgress?(processed: number): void } = {} +): Promise { + const configuration = JSON.stringify(host.services.setting.currentSettings()); + const settings = JSON.parse(configuration) as ObsidianLiveSyncSettings; + const database = host.services.database.localDatabase.localDatabase; + const isConfigurationCurrent = () => + JSON.stringify(host.services.setting.currentSettings()) === configuration && + host.services.database.localDatabase.localDatabase === database; + const mapper = createDiagnosticNameMapper(nameMode); + const versions = { + commonlib: commonlibPackage.version, + plugin: host.services.API.getPluginVersion(), + application: host.services.API.getAppVersion(), + }; + const local = createLocalDiagnosticSource(database); + if (kind === "usage") { + const result = await analyseDatabaseUsage({ + source: local, + signal: options.signal, + onProgress: (progress) => options.onProgress?.(progress.revisionsProcessed), + }); + if (!isConfigurationCurrent()) { + result.errors.push({ code: "configuration_changed" }); + if (result.status === "complete") result.status = "partial"; + } + const report = projectUsageReport(result, mapper, versions); + const artifacts: DatabaseDiagnosticArtifact[] = [ + { + format: "json", + filename: `database-usage-${result.status}.json`, + content: formatDiagnosticJSON(report), + }, + ]; + // A standalone legacy TSV has no coverage fields. Partial runs retain + // their rows in JSON together with the status and reasons. + if (result.status === "complete") { + artifacts.unshift({ format: "tsv", filename: "database-usage.tsv", content: formatUsageTSV(report) }); + } + return { status: result.status, nameMode, artifacts }; + } + const result = await analyseDatabaseIntegrity({ + local, + remote: + settings.remoteType === REMOTE_COUCHDB + ? () => + Promise.resolve( + createCouchDBDiagnosticSource({ + settings, + signal: options.signal, + fetch: (request, requestOptions) => + settings.useRequestAPI + ? host.services.API.nativeFetch(request, requestOptions) + : host.services.API.webCompatFetch(request, requestOptions), + }) + ) + : undefined, + remoteUnavailableReason: settings.remoteType === REMOTE_COUCHDB ? "source-unavailable" : "unsupported-remote", + path: { + getPath: (entry) => host.services.path.getPath(entry), + path2id: (path, prefix) => host.services.path.path2idWithSettings(path, settings, prefix), + }, + settings: { + encrypt: settings.encrypt, + usePathObfuscation: settings.usePathObfuscation, + handleFilenameCaseSensitive: settings.handleFilenameCaseSensitive, + idDerivationVersion: settings.idDerivationVersion ?? 0, + pathNormalizationVersion: settings.pathNormalizationVersion ?? 0, + }, + readStorage: (path) => readDiagnosticStorage(host.serviceModules.storageAccess, path), + isConfigurationCurrent, + signal: options.signal, + onProgress: (progress) => options.onProgress?.(progress.processed), + }); + const report = projectIntegrityReport(result, mapper, versions); + return { + status: result.status, + nameMode, + artifacts: [ + { + format: "markdown", + filename: `database-integrity-${result.status}.md`, + content: formatIntegrityMarkdown(report), + }, + { + format: "json", + filename: `database-integrity-${result.status}.json`, + content: formatDiagnosticJSON(report), + }, + ], + }; +} diff --git a/src/serviceFeatures/databaseDiagnostics.unit.spec.ts b/src/serviceFeatures/databaseDiagnostics.unit.spec.ts new file mode 100644 index 00000000..b572832d --- /dev/null +++ b/src/serviceFeatures/databaseDiagnostics.unit.spec.ts @@ -0,0 +1,213 @@ +import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_SETTINGS, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { readDiagnosticStorage, runDatabaseDiagnostic, type DatabaseDiagnosticHost } from "./databaseDiagnostics"; + +function fixture() { + const path = "Private/日本語.md"; + const body = "private file contents"; + const bytes = new TextEncoder().encode(body); + const settings = { + ...DEFAULT_SETTINGS, + remoteType: REMOTE_P2P, + readChunksOnline: true, + passphrase: "private encryption passphrase", + couchDB_PASSWORD: "private credential", + couchDB_URI: "https://private-server.invalid", + }; + const documents = { + "f:private-actual-id": { + _id: "f:private-actual-id", + _rev: "1-private-revision", + _revs_info: [{ rev: "1-private-revision", status: "available" }], + type: "plain", + path, + ctime: 1, + mtime: 1, + size: bytes.byteLength, + children: ["h:private-chunk"], + }, + "h:private-chunk": { + _id: "h:private-chunk", + _rev: "1-private-chunk-revision", + _revs_info: [{ rev: "1-private-chunk-revision", status: "available" }], + type: "leaf", + data: body, + }, + }; + const get = vi.fn(async (id: string) => { + const doc = documents[id as keyof typeof documents]; + if (!doc) throw { status: 404 }; + return structuredClone(doc); + }); + const info = vi.fn(async () => ({ update_seq: "1-stable" })); + const put = vi.fn(() => { + throw new Error("Diagnostic attempted a write"); + }); + const applyPartial = vi.fn(); + const fetch = vi.fn(); + const storage = { + statHidden: vi.fn(async () => ({ type: "file" as const, ctime: 1, mtime: 1, size: bytes.byteLength })), + readHiddenFileBinary: vi.fn(async () => bytes.buffer), + }; + const host = { + services: { + setting: { currentSettings: () => settings, applyPartial }, + database: { + localDatabase: { + localDatabase: { + info, + get, + put, + allDocs: vi.fn(async () => ({ + rows: Object.keys(documents).map((id) => ({ id, value: { rev: "1-current" } })), + })), + }, + }, + }, + path: { + getPath: (doc: { path: string }) => doc.path, + path2idWithSettings: vi.fn(async (rawPath: string) => `f:expected:${rawPath}`), + }, + API: { + webCompatFetch: fetch, + nativeFetch: fetch, + getPluginVersion: () => "1.0.38", + getAppVersion: () => "1.12.7", + }, + }, + serviceModules: { storageAccess: storage }, + } as unknown as DatabaseDiagnosticHost; + return { host, settings, path, body, bytes, get, info, put, applyPartial, fetch, storage }; +} + +describe("database diagnostic host composition", () => { + it("keeps on-demand fetching enabled while analysing only existing local revisions", async () => { + const data = fixture(); + const before = JSON.stringify(data.settings); + const result = await runDatabaseDiagnostic(data.host, "usage", "pseudonym"); + expect(result.status).toBe("complete"); + expect(result.artifacts.map((artifact) => artifact.format)).toEqual(["tsv", "json"]); + expect(data.settings.readChunksOnline).toBe(true); + expect(JSON.stringify(data.settings)).toBe(before); + expect(data.applyPartial).not.toHaveBeenCalled(); + expect(data.fetch).not.toHaveBeenCalled(); + expect(data.put).not.toHaveBeenCalled(); + expect(data.get).toHaveBeenCalledWith("f:private-actual-id", { rev: "1-private-revision" }); + const report = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content); + expect(report.versions).toMatchObject({ plugin: "1.0.38", application: "1.12.7" }); + expect(report.versions.commonlib).toMatch(/^\d+\.\d+\.\d+/); + for (const artifact of result.artifacts) { + for (const secret of [ + data.path, + data.body, + data.settings.passphrase, + "private-actual-id", + "private-revision", + "private-server.invalid", + ]) { + expect(artifact.content).not.toContain(secret); + } + } + }); + + it("keeps local findings when the remote type is unsupported and projects both formats consistently", async () => { + const data = fixture(); + const result = await runDatabaseDiagnostic(data.host, "integrity", "pseudonym"); + expect(result.status).toBe("partial"); + expect(data.fetch).not.toHaveBeenCalled(); + expect(data.put).not.toHaveBeenCalled(); + expect(data.storage.readHiddenFileBinary).toHaveBeenCalledWith(data.path); + const json = JSON.parse(result.artifacts.find((artifact) => artifact.format === "json")!.content); + expect(json.errors).toContainEqual({ source: "remote", reason: "unsupported-remote" }); + expect(json.observations[0]).toMatchObject({ + identity: "document-id-mismatch", + content: { status: "available" }, + }); + expect(json.observations[0].comparisons).toContainEqual({ + against: json.observations[0].vault.reference, + relation: "equal", + }); + for (const artifact of result.artifacts) { + expect(artifact.content).toContain(json.observations[0].path); + expect(artifact.content).not.toContain(data.path); + expect(artifact.content).not.toContain(data.body); + expect(artifact.content).not.toContain(data.settings.passphrase); + } + }); + + it("includes names only when requested and never includes credentials or file contents", async () => { + const data = fixture(); + const result = await runDatabaseDiagnostic(data.host, "usage", "original"); + for (const artifact of result.artifacts) { + expect(artifact.content).toContain(data.path); + expect(artifact.content).toContain("f:private-actual-id"); + expect(artifact.content).not.toContain(data.body); + expect(artifact.content).not.toContain(data.settings.couchDB_PASSWORD); + expect(artifact.content).not.toContain(data.settings.couchDB_URI); + } + }); + + it("retains cancellation and configuration changes in JSON instead of presenting a complete TSV", async () => { + const cancelled = fixture(); + const controller = new AbortController(); + controller.abort(); + const stopped = await runDatabaseDiagnostic(cancelled.host, "usage", "pseudonym", { + signal: controller.signal, + }); + expect(stopped.status).toBe("cancelled"); + expect(stopped.artifacts.map((artifact) => artifact.format)).toEqual(["json"]); + expect(cancelled.get).not.toHaveBeenCalled(); + + const changed = fixture(); + changed.info.mockImplementationOnce(async () => ({ update_seq: "1-stable" })); + changed.info.mockImplementationOnce(async () => { + changed.settings.couchDB_URI = "https://changed.invalid"; + return { update_seq: "1-stable" }; + }); + const output = await runDatabaseDiagnostic(changed.host, "usage", "original"); + expect(output.status).toBe("partial"); + expect(output.artifacts.map((artifact) => artifact.format)).toEqual(["json"]); + expect(JSON.parse(output.artifacts[0].content).errors).toContainEqual({ code: "configuration_changed" }); + }); +}); + +describe("diagnostic Vault reads", () => { + it("distinguishes an absent file, a read error, and a file changing during its read", async () => { + const data = fixture(); + data.storage.statHidden.mockResolvedValueOnce(null as never); + expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ status: "missing" }); + data.storage.readHiddenFileBinary.mockRejectedValueOnce(new Error("private filename and content")); + expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ + status: "unreadable", + reason: "content-unreadable", + }); + data.storage.statHidden.mockResolvedValueOnce({ + type: "file", + ctime: 1, + mtime: 1, + size: data.bytes.byteLength, + }); + data.storage.statHidden.mockResolvedValueOnce({ + type: "file", + ctime: 1, + mtime: 2, + size: data.bytes.byteLength, + }); + expect(await readDiagnosticStorage(data.storage, data.path)).toEqual({ + status: "unreadable", + reason: "storage-changed", + }); + }); + + it("rejects paths outside the Vault before touching storage", async () => { + const data = fixture(); + for (const path of ["../private", "/private", "C:\\private", "x/../private", "x\0y"]) { + expect(await readDiagnosticStorage(data.storage, path)).toEqual({ + status: "unreadable", + reason: "content-unreadable", + }); + } + expect(data.storage.statHidden).not.toHaveBeenCalled(); + expect(data.storage.readHiddenFileBinary).not.toHaveBeenCalled(); + }); +}); diff --git a/src/serviceFeatures/useDatabaseDiagnostics.ts b/src/serviceFeatures/useDatabaseDiagnostics.ts new file mode 100644 index 00000000..71021b1c --- /dev/null +++ b/src/serviceFeatures/useDatabaseDiagnostics.ts @@ -0,0 +1,76 @@ +import { + createServiceFeature, + type NecessaryServices, +} from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events"; +import { $msg } from "@/common/translation"; +import DatabaseDiagnosticDialog from "@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte"; +import { runDatabaseDiagnostic, type DatabaseDiagnosticKind } from "./databaseDiagnostics"; +import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes"; + +type DatabaseDiagnosticsFeatureHost = NecessaryServices< + "setting" | "database" | "path" | "API" | "UI" | "appLifecycle", + "storageAccess" +>; + +export const useDatabaseDiagnosticsFeature = createServiceFeature((host: DatabaseDiagnosticsFeatureHost) => { + const { services } = host; + let active: AbortController | undefined; + const open = async (kind: DatabaseDiagnosticKind): Promise => { + if (active) return; + const controller = new AbortController(); + active = controller; + try { + await services.UI.dialogManager.open<"close", DatabaseDiagnosticDialogOptions>(DatabaseDiagnosticDialog, { + kind, + run: async (nameMode, onProgress) => { + if (!services.database.isDatabaseReady()) throw new Error("local-database-unavailable"); + return await runDatabaseDiagnostic(host, kind, nameMode, { + signal: controller.signal, + onProgress, + }); + }, + cancel: () => controller.abort(), + }); + } catch { + // Closing the host during unload can reject the dialogue result. + // Diagnostic errors are presented inside the dialogue without raw logs. + } finally { + controller.abort(); + if (active === controller) active = undefined; + } + }; + services.appLifecycle.onInitialise.addHandler(() => { + services.API.addCommand({ + id: "analyse-database", + name: "Analyse Database Usage (advanced)", + icon: "database-search", + checkCallback: (checking) => { + if (!services.setting.currentSettings().useAdvancedMode || !services.database.isDatabaseReady()) + return false; + if (!checking) void open("usage"); + return true; + }, + }); + services.API.addCommand({ + id: "inspect-database-integrity", + name: $msg("Inspect database integrity"), + icon: "database-search", + checkCallback: (checking) => { + if (!services.database.isDatabaseReady()) return false; + if (!checking) void open("integrity"); + return true; + }, + }); + return Promise.resolve(true); + }); + const stopUsage = services.context.events.onEvent(EVENT_ANALYSE_DB_USAGE, () => void open("usage")); + const stopIntegrity = services.context.events.onEvent(EVENT_INSPECT_DB_INTEGRITY, () => void open("integrity")); + services.appLifecycle.onUnload.addHandler(() => { + active?.abort(); + stopUsage(); + stopIntegrity(); + return Promise.resolve(true); + }); + return { open }; +}); diff --git a/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts new file mode 100644 index 00000000..0e9669e0 --- /dev/null +++ b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from "vitest"; +import type { DatabaseDiagnosticDialogOptions } from "./databaseDiagnosticDialogTypes"; + +vi.mock("@/modules/services/LiveSyncUI/dialogues/DatabaseDiagnosticDialog.svelte", () => ({ default: {} })); +vi.mock("./databaseDiagnostics", () => ({ + runDatabaseDiagnostic: vi.fn(async () => ({ status: "complete", nameMode: "pseudonym", artifacts: [] })), +})); +vi.mock("@/common/translation", () => ({ $msg: (value: string) => value })); + +import { runDatabaseDiagnostic } from "./databaseDiagnostics"; +import { useDatabaseDiagnosticsFeature } from "./useDatabaseDiagnostics"; +import { EVENT_ANALYSE_DB_USAGE, EVENT_INSPECT_DB_INTEGRITY } from "@/common/events"; + +function fixture() { + const initialise: Array<() => Promise> = []; + const unload: Array<() => Promise> = []; + const commands: Array<{ id: string; checkCallback: (checking: boolean) => boolean }> = []; + const listeners = new Map void>(); + const unsubscribe = vi.fn((event: string) => listeners.delete(event)); + const settings = { useAdvancedMode: false }; + let ready = true; + let close!: () => void; + const openDialog = vi.fn( + (_component: unknown, _options: DatabaseDiagnosticDialogOptions) => + new Promise((resolve) => { + close = resolve; + }) + ); + const host = { + services: { + setting: { currentSettings: () => settings }, + database: { isDatabaseReady: () => ready }, + path: {}, + API: { addCommand: (command: (typeof commands)[number]) => commands.push(command) }, + UI: { dialogManager: { open: openDialog } }, + appLifecycle: { + onInitialise: { addHandler: (handler: (typeof initialise)[number]) => initialise.push(handler) }, + onUnload: { addHandler: (handler: (typeof unload)[number]) => unload.push(handler) }, + }, + context: { + events: { + onEvent: (event: string, handler: () => void) => { + listeners.set(event, handler); + return () => unsubscribe(event); + }, + }, + }, + }, + serviceModules: { storageAccess: {} }, + } as unknown as Parameters[0]; + const view = useDatabaseDiagnosticsFeature(host); + return { + host, + view, + initialise, + unload, + commands, + listeners, + settings, + openDialog, + unsubscribe, + close: () => close(), + setReady: (value: boolean) => { + ready = value; + }, + }; +} + +describe("database diagnostics serviceFeature", () => { + it("retains the existing Advanced-mode Usage command gate in its new owner", async () => { + const data = fixture(); + for (const initialise of data.initialise) await initialise(); + const usage = data.commands.find((command) => command.id === "analyse-database")!; + const integrity = data.commands.find((command) => command.id === "inspect-database-integrity")!; + expect(usage.checkCallback(true)).toBe(false); + expect(integrity.checkCallback(true)).toBe(true); + data.settings.useAdvancedMode = true; + expect(usage.checkCallback(true)).toBe(true); + data.setReady(false); + expect(usage.checkCallback(true)).toBe(false); + expect(integrity.checkCallback(true)).toBe(false); + expect(data.openDialog).not.toHaveBeenCalled(); + }); + + it("uses the existing dialogue manager and performs no analysis until the run choice is submitted", async () => { + vi.mocked(runDatabaseDiagnostic).mockClear(); + const data = fixture(); + const opened = data.view.open("usage"); + expect(data.openDialog).toHaveBeenCalledTimes(1); + expect(runDatabaseDiagnostic).not.toHaveBeenCalled(); + const options = data.openDialog.mock.calls[0][1]; + expect(Object.keys(options).sort()).toEqual(["cancel", "kind", "run"]); + await options.run("original", () => {}); + expect(runDatabaseDiagnostic).toHaveBeenCalledWith( + data.host, + "usage", + "original", + expect.objectContaining({ signal: expect.any(AbortSignal) }) + ); + const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!; + await data.view.open("integrity"); + expect(data.openDialog).toHaveBeenCalledTimes(1); + data.close(); + await opened; + expect(signal.aborted).toBe(true); + }); + + it("wires both Hatch actions and releases its listeners and active request on unload", async () => { + const data = fixture(); + expect(data.listeners.has(EVENT_ANALYSE_DB_USAGE)).toBe(true); + expect(data.listeners.has(EVENT_INSPECT_DB_INTEGRITY)).toBe(true); + const opened = data.view.open("integrity"); + await data.openDialog.mock.calls[0][1].run("pseudonym", () => {}); + const signal = vi.mocked(runDatabaseDiagnostic).mock.calls.slice(-1)[0][3]!.signal!; + for (const unload of data.unload) await unload(); + expect(signal.aborted).toBe(true); + expect(data.listeners.size).toBe(0); + expect(data.unsubscribe).toHaveBeenCalledTimes(2); + data.close(); + await opened; + }); +}); diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index aa3afb23..ecef9569 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -74,6 +74,7 @@ After changing plug-in source, use the focused wrapper rather than invoking a sc ```bash npm run test:e2e:obsidian:focused -- settings-ui +npm run test:e2e:obsidian:focused -- database-diagnostics npm run test:e2e:obsidian:focused -- configuration-encryption npm run test:e2e:obsidian:focused -- configuration-encryption-sync npm run test:e2e:obsidian:focused -- two-vault-sync @@ -85,6 +86,8 @@ npm run test:e2e:obsidian:focused -- couchdb-http-cache The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. +`database-diagnostics` needs no remote service. It checks the Usage command and Hatch integrity action with an isolated Vault, including the choice before inspection, original names and pseudonyms, retained local findings when the remote is unavailable, and unchanged settings, local revisions, and Vault files. It captures desktop results and the choice dialogue in mobile emulation. Copy and download payloads are captured inside the isolated renderer, so the scenario leaves the host clipboard and download directory unchanged. Commonlib's separate CouchDB integration test verifies encrypted reads and missing Security Seed handling against a real server. + `folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file whose recorded revision has been pruned receives the newer database content using persisted content evidence without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. A third file begins with no provenance while its bytes still match the current database revision; start-up must record that revision without creating a new one, and a later incoming revision must reflect without a conflict. A fourth file has legacy revision-only provenance; start-up must verify its exact content and add a hash without creating a database revision or changing storage. The database advances and pending snapshot are controlled fixtures; start-up processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. diff --git a/test/e2e-obsidian/scripts/database-diagnostics.ts b/test/e2e-obsidian/scripts/database-diagnostics.ts new file mode 100644 index 00000000..d6e59714 --- /dev/null +++ b/test/e2e-obsidian/scripts/database-diagnostics.ts @@ -0,0 +1,237 @@ +import assert from "node:assert/strict"; +import type { Page } from "playwright"; +import { DoctorRegulation } from "@vrtmrz/livesync-commonlib/compat/common/configForDoc"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { captureObsidianElement, captureObsidianPage, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts"; +import { assertMobileDialogueLayout, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; + +const fixturePath = "Private/diagnostic-秘密.md"; +const fixtureContent = "Diagnostic fixture content which must never appear in a report.\n"; +const shadowId = "diagnostic-shadow-document"; +const timeout = 15000; + +type DiagnosticPageGlobal = typeof globalThis & { + app: { commands: { executeCommandById(id: string): boolean } }; + diagnosticClipboard?: string; + diagnosticDownload?: { name: string; content: string }; +}; + +async function openCommand(page: Page, kind: "usage" | "integrity") { + const id = kind === "usage" ? "analyse-database" : "inspect-database-integrity"; + const opened = await page.evaluate( + (command) => (globalThis as DiagnosticPageGlobal).app.commands.executeCommandById(command), + `obsidian-livesync:${id}` + ); + assert.equal(opened, true); + const dialogue = page.locator(".database-diagnostic"); + await dialogue.locator("select").selectOption("pseudonym"); + assert.equal(await dialogue.getAttribute("data-diagnostic-phase"), "choice"); + assert.equal(await dialogue.locator("textarea").count(), 0); + return dialogue; +} + +async function readDatabaseState(cli: string, session: ObsidianLiveSyncSession) { + return await evalObsidianJson<{ sequence: unknown; settings: string; names: string[] }>( + cli, + "(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;return JSON.stringify({sequence:(await core.localDatabase.localDatabase.info()).update_seq,settings:JSON.stringify(core.services.setting.currentSettings()),names:app.vault.getFiles().map(f=>f.path).sort()});})()", + session.cliEnv + ); +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = requireObsidianCli(); + const vault = await createTemporaryVault("obsidian-livesync-diagnostics-"); + let session: ObsidianLiveSyncSession | undefined; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli, + vault, + pluginData: { + displayLanguage: "def", + doctorProcessedVersion: DoctorRegulation.version, + isConfigured: true, + useAdvancedMode: true, + readChunksOnline: true, + liveSync: false, + remoteType: "", + couchDB_URI: "", + couchDB_DBNAME: "", + couchDB_USER: "", + couchDB_PASSWORD: "", + remoteConfigurations: {}, + activeConfigurationId: "", + notifyThresholdOfRemoteStorageSize: -1, + periodicReplication: false, + syncAfterMerge: false, + syncOnEditorSave: false, + syncOnFileOpen: false, + syncOnSave: false, + syncOnStart: false, + useEden: false, + }, + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + }); + await waitForLiveSyncCoreReady(cli, session.cliEnv); + await evalObsidianJson( + cli, + `(async()=>{await app.vault.createFolder('Private');await app.vault.create(${JSON.stringify(fixturePath)},${JSON.stringify(fixtureContent)});return JSON.stringify(true);})()`, + session.cliEnv + ); + await waitForLocalDatabaseEntry(cli, session.cliEnv, fixturePath); + await evalObsidianJson( + cli, + `(async()=>{const core=app.plugins.plugins['obsidian-livesync'].core;await core.services.fileProcessing.commitPendingFileEvents();const db=core.localDatabase.localDatabase;const id=await core.services.path.path2id(${JSON.stringify(fixturePath)});const original=await db.get(id);const shadow={...original,_id:${JSON.stringify(shadowId)}};delete shadow._rev;await db.put(shadow);return JSON.stringify(true);})()`, + session.cliEnv + ); + const before = await readDatabaseState(cli, session); + let usageJSON = ""; + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const choice = await openCommand(page, "usage"); + await choice.getByRole("button", { name: "Cancel", exact: true }).click(); + await choice.waitFor({ state: "hidden", timeout }); + const dialogue = await openCommand(page, "usage"); + assert.equal(await dialogue.locator("select").inputValue(), "pseudonym"); + await dialogue.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + const tsv = await dialogue.locator("textarea").inputValue(); + assert.equal(tsv.split("\n")[0].split("\t").length, 9); + assert.ok(!tsv.includes(fixturePath)); + assert.ok(!tsv.includes(shadowId)); + assert.ok(!tsv.includes(fixtureContent.trim())); + await dialogue.locator("select").selectOption("json"); + usageJSON = await dialogue.locator("textarea").inputValue(); + const report = JSON.parse(usageJSON); + assert.equal(report.status, "complete"); + assert.ok(report.versions.commonlib); + assert.ok(report.rows.some((row: { path: string }) => row.path.startsWith("file-"))); + + // Capture exports inside the isolated renderer without changing the + // host clipboard or leaving downloads in a user's download directory. + await page.evaluate(`(() => { + Object.defineProperty(navigator.clipboard, "writeText", { + configurable: true, + value: async (text) => { globalThis.diagnosticClipboard = text; }, + }); + document.addEventListener("click", (event) => { + const link = event.target; + if (!(link instanceof HTMLAnchorElement) || !link.download || !link.href.startsWith("blob:")) return; + event.preventDefault(); + void fetch(link.href).then((response) => response.text()).then((content) => { + globalThis.diagnosticDownload = { name: link.download, content }; + }); + }, true); + })()`); + await dialogue.getByRole("button", { name: "Copy report", exact: true }).click(); + assert.equal( + await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticClipboard), + usageJSON + ); + await dialogue.getByRole("button", { name: "Save report", exact: true }).click(); + await page.waitForFunction(() => !!(globalThis as DiagnosticPageGlobal).diagnosticDownload, undefined, { + timeout, + }); + assert.deepEqual(await page.evaluate(() => (globalThis as DiagnosticPageGlobal).diagnosticDownload), { + name: "database-usage-complete.json", + content: usageJSON, + }); + await dialogue.getByRole("button", { name: "Close", exact: true }).click(); + await dialogue.waitFor({ state: "hidden", timeout }); + + const original = await openCommand(page, "usage"); + await original.locator("select").selectOption("original"); + await original.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + const rawTSV = await original.locator("textarea").inputValue(); + assert.ok(rawTSV.includes(fixturePath)); + assert.ok(rawTSV.includes(shadowId)); + assert.ok(!rawTSV.includes(fixtureContent.trim())); + await original.getByRole("button", { name: "Close", exact: true }).click(); + await original.waitFor({ state: "hidden", timeout }); + + const settingsNavigator = await openLiveSyncSettings(page, timeout); + const hatch = await settingsNavigator.openPage("Hatch"); + const setting = hatch + .locator(".setting-item") + .filter({ has: page.getByText("Inspect database integrity", { exact: true }) }); + await setting.getByRole("button", { name: "Inspect", exact: true }).click(); + const integrity = page.locator(".database-diagnostic"); + assert.equal(await integrity.locator("select").inputValue(), "pseudonym"); + await integrity.getByRole("button", { name: "Begin inspection", exact: true }).click(); + await page.locator('[data-diagnostic-phase="result"]').waitFor({ timeout }); + await integrity.locator("select").selectOption("json"); + const reportText = await integrity.locator("textarea").inputValue(); + const integrityReport = JSON.parse(reportText); + assert.equal(integrityReport.status, "partial"); + assert.ok( + integrityReport.errors.some((error: { reason: string }) => error.reason === "source-unavailable") + ); + assert.ok( + integrityReport.observations.some( + (observation: { identity: string }) => observation.identity === "document-id-mismatch" + ) + ); + assert.ok( + integrityReport.observations.some((observation: { comparisons: { relation: string }[] }) => + observation.comparisons.some((comparison) => comparison.relation === "equal") + ) + ); + for (const privateValue of [fixturePath, shadowId, fixtureContent.trim()]) + assert.ok(!reportText.includes(privateValue)); + }); + const desktopScreenshot = await captureObsidianElement( + session.remoteDebuggingPort, + "database-diagnostics-desktop.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.locator(".database-diagnostic").getByRole("button", { name: "Close", exact: true }).click(); + await page.keyboard.press("Escape"); + }); + const after = await readDatabaseState(cli, session); + assert.deepEqual(after, before, "Diagnostics must leave settings, revisions, and Vault files unchanged"); + + await setObsidianMobileTestMode(session.remoteDebuggingPort, true, timeout); + await waitForLiveSyncCoreReady(cli, session.cliEnv); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await openCommand(page, "integrity"); + }); + const mobileScreenshot = await captureObsidianElement( + session.remoteDebuggingPort, + "database-diagnostics-mobile.png", + (page) => page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }) + ); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const modal = page.locator(".modal-container").filter({ has: page.locator(".database-diagnostic") }); + await assertMobileDialogueLayout(page, modal, "Database diagnostic choice"); + await modal.getByRole("button", { name: "Cancel", exact: true }).click(); + }); + console.log(JSON.stringify({ status: "passed", desktopScreenshot, mobileScreenshot, readOnly: true })); + } catch (error) { + if (session) { + await captureObsidianPage( + session.remoteDebuggingPort, + "database-diagnostics-failure.png", + async () => {} + ).catch(() => {}); + } + throw error; + } finally { + if (session) await session.app.stop(); + await vault.dispose(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index b0199b89..7ce9514e 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -8,6 +8,7 @@ const focusedScenarios = new Set([ "smoke", "onboarding-invitation", "dialog-mounts", + "database-diagnostics", "conflict-dialog-policy", "revision-repair", "document-history-nav", diff --git a/updates.md b/updates.md index c8d1371e..7b8c9756 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Diagnostics + +- We can inspect normal-file Metadata and current conflicts in the local database and CouchDB together, with byte comparisons for suspect groups. Partial results retain the available evidence when a source is unavailable. The inspection leaves databases and settings unchanged. +- Database Usage analysis and integrity inspection now offer temporary pseudonyms or original names before each run. Reports omit file contents and credentials in both modes. Complete Usage runs retain the existing TSV columns; partial runs use JSON to retain their status and reasons. + ## 1.0.38 17th October, 2026