From aad0e56f09771761bcdfb8a9eb7748d11d1e8b7f Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 22 Jul 2026 06:21:54 +0000 Subject: [PATCH] Preserve legacy settings across same-profile upgrades --- ...elease_notes_and_database_compatibility.md | 16 +- package.json | 1 + src/common/databaseCompatibility.ts | 10 - src/common/databaseCompatibility.unit.spec.ts | 10 +- src/serviceFeatures/compatibilityReview.ts | 42 +- .../compatibilityReview.unit.spec.ts | 90 +- .../compatibilityReviewMarkdown.ts | 17 +- .../compatibilityReviewObsidian.ts | 11 +- .../compatibilityReviewObsidian.unit.spec.ts | 33 +- test/e2e-obsidian/runner/couchdb.ts | 38 + test/e2e-obsidian/runner/liveSyncWorkflow.ts | 33 +- test/e2e-obsidian/runner/objectStorage.ts | 17 + .../runner/releaseArtifact.test.ts | 76 ++ test/e2e-obsidian/runner/releaseArtifact.ts | 128 +++ test/e2e-obsidian/runner/session.test.ts | 55 ++ test/e2e-obsidian/runner/session.ts | 3 +- .../runner/twoVaultSyncLifecycle.test.ts | 1 - .../runner/upgradeContinuity.test.ts | 54 ++ test/e2e-obsidian/runner/upgradeContinuity.ts | 199 +++++ test/e2e-obsidian/runner/upgradeWorkflow.ts | 837 ++++++++++++++++++ test/e2e-obsidian/scripts/run-focused.ts | 1 + .../scripts/upgrade-from-stable.ts | 753 ++++++++++++++++ 22 files changed, 2258 insertions(+), 167 deletions(-) create mode 100644 test/e2e-obsidian/runner/releaseArtifact.test.ts create mode 100644 test/e2e-obsidian/runner/releaseArtifact.ts create mode 100644 test/e2e-obsidian/runner/session.test.ts create mode 100644 test/e2e-obsidian/runner/upgradeContinuity.test.ts create mode 100644 test/e2e-obsidian/runner/upgradeContinuity.ts create mode 100644 test/e2e-obsidian/runner/upgradeWorkflow.ts create mode 100644 test/e2e-obsidian/scripts/upgrade-from-stable.ts diff --git a/docs/adr/2026_07_release_notes_and_database_compatibility.md b/docs/adr/2026_07_release_notes_and_database_compatibility.md index d133244d..15b21c1e 100644 --- a/docs/adr/2026_07_release_notes_and_database_compatibility.md +++ b/docs/adr/2026_07_release_notes_and_database_compatibility.md @@ -35,13 +35,18 @@ Commonlib's `settingVersion` describes the stored settings shape, while `DEFAULT The current new-Vault base selects a 50 MB maximum synchronised file size, Rabin–Karp chunk splitting, Plug-in Sync V2, case-insensitive file-name handling, and E2EE V2. It does not enable synchronisation, encryption, or a remote connection without user action. Chunk revisions are always content-derived; `doNotUseFixedRevisionForChunks` remains only as deprecated compatibility input and is not a recommendation or review setting. -An existing settings document without an explicit `handleFilenameCaseSensitive` choice keeps that value unresolved and enters compatibility review. The running host can explicitly retain legacy case-sensitive handling. Adopting cross-platform case-insensitive handling remains paused until the person has checked case-only path conflicts and rebuilt the local database. +Relative to the conservative existing-setting fallbacks, only Plug-in Sync V2 and the explicit case-insensitive value differ for a new Vault. The 50 MB limit, Rabin–Karp splitter, and E2EE V2 already match the legacy fallback values. Data Compression, Eden, V1 dynamic iteration, the legacy IndexedDB adapter, Hidden File Sync, and automatic synchronisation remain disabled when absent from an existing settings document. + +An existing settings document without an explicit `handleFilenameCaseSensitive` choice is normalised to `false` and saved. This preserves the effective case-insensitive branch used by earlier releases when the value was absent, and does not require a review or rebuild. An explicit `true` or `false` choice remains unchanged. + +Keep configured-state inference separate from new-Vault initialisation. If an existing legacy document has no `isConfigured` value, repeat the pre-1.0 comparison with the conservative defaults: a default-equivalent document remains unconfigured, while a non-default stored value is evidence that it was configured. Persist that inferred boolean so a migration cannot turn an unconfigured Vault into an irreversible configured state merely because its settings document was non-empty. ### Database compatibility - Continue to use the internal database version `VER` for changes which require explicit compatibility review. Changing the plug-in SemVer alone does not increment `VER`. - Store the last acknowledged internal database version through Commonlib's device-local small-configuration contract under `database-compatibility-version`. Copy the legacy raw local-storage value into that contract once, then remove the legacy key after the copy has completed. -- Initialise the marker to the current `VER` only when Commonlib identifies a genuinely new Vault with no pending review. An existing Vault with a missing or invalid marker requires review instead of being silently accepted. +- Initialise the marker to the current `VER` only when Commonlib identifies a genuinely new Vault with no pending review. A configured existing Vault with a missing or invalid marker requires review instead of being silently accepted. +- Defer database compatibility evaluation for an existing unconfigured Vault. It cannot replicate, so do not persist a misleading pause or acknowledge its missing marker while onboarding is still pending. Keep the marker absent so a later configured start evaluates the same state before ordinary synchronisation. - Treat a missing marker on a configured Vault as an ambiguous device transition. Copying or restoring a Vault, or opening it with a new Obsidian profile, can preserve settings and database files without preserving device-local storage. Do not infer acknowledgement from an empty local database: a recovery operation, partial copy, or remote-first setup can also produce that state. Explain these cases and require an explicit decision in the compatibility dialogue. - Derive one structured pause from the acknowledged database version, Commonlib's settings-migration state, and any persisted legacy review message. Persist the generic `versionUpFlash` message without changing any automatic synchronisation setting, because Commonlib already treats that field as a replication gate. - Treat non-empty `versionUpFlash` as a runtime replication gate. Standard and one-shot replication must stop before remote work begins. @@ -66,7 +71,7 @@ An existing settings document without an explicit `handleFilenameCaseSensitive` ### Flag-file recovery order -- Evaluate and persist the compatibility gate after settings load, before Obsidian layout-ready recovery begins. This blocks ordinary and one-shot replication even while the review dialogue has not yet opened. +- For a configured Vault, evaluate and persist the compatibility gate after settings load, before Obsidian layout-ready recovery begins. This blocks ordinary and one-shot replication even while the review dialogue has not yet opened. An existing unconfigured Vault follows the deferred rule above instead. - Preserve the existing ordered flag-file recovery handlers: SCRAM at priority 5, fetch-all at priority 10, and rebuild-all at priority 20. These files express an explicit recovery instruction and may invoke their focused storage or rebuild service while ordinary replication remains gated. - Present the compatibility review at priority 30, after any selected recovery operation. A recovery handler which cancels start-up, keeps SCRAM active, or schedules a restart returns `false`, so the current process does not open a competing compatibility dialogue. If recovery completes and start-up continues, the dialogue opens before normal synchronisation is allowed to resume. - Keep database preparation independent of an unanswered compatibility dialogue, because the compatibility gate already blocks replication. Before Config Doctor begins its interactive checks, await the active initial review so that the two update dialogues cannot overlap. @@ -78,6 +83,7 @@ An existing settings document without an explicit `handleFilenameCaseSensitive` - SemVer pre-releases such as `1.0.0-rc.0` no longer require a special numeric encoding inside plug-in settings. - An internal compatibility change remains fail-closed for replication, but it no longer destroys the person's synchronisation preferences. - A new installation has no previous internal-version marker and therefore does not show an upgrade review. Its initial settings and onboarding remain responsible for keeping replication disabled until configuration is complete. +- An existing unconfigured installation also remains on onboarding without a compatibility warning. Unlike a genuinely new Vault, it does not receive an acknowledgement marker; activation leaves the compatibility decision for its next configured start. - A copied or restored configured Vault can show a one-time compatibility review on its new device or profile. This is intentional even when its local database appears empty, because emptiness does not prove how the Vault was produced. - A genuinely new Vault receives current recommendations without applying them as fallbacks to an existing configuration. It remains inert until onboarding is accepted. - Accepted new-device and existing-device setup cannot enable ordinary processing before the selected Rebuild or Fetch has been reserved. @@ -88,8 +94,8 @@ An existing settings document without an explicit `handleFilenameCaseSensitive` ## Verification - Unit tests verify new-Vault initialisation, upgrades, missing and invalid markers, downgrades, future settings schemas, legacy marker migration, acknowledgement ordering, and save-failure recovery while retaining automatic synchronisation choices. -- Commonlib package tests verify conservative stored-setting completion, independently mutable new-Vault settings, unresolved file-name case policy, future-schema protection, and the focused settings entry from a clean consumer. -- Host unit tests verify new-Vault factory use, conservative import paths, the unconfigured start-up gate, flag-before-settings ordering, rollback when the flag cannot be reserved, ordinary configured edits, and the explicit file-name case decision. +- Commonlib package tests verify conservative stored-setting completion, independently mutable new-Vault settings, legacy file-name case normalisation, future-schema protection, and the focused settings entry from a clean consumer. +- Host unit tests verify new-Vault factory use, conservative import paths, the unconfigured start-up gate, deferred compatibility evaluation and later re-evaluation, flag-before-settings ordering, rollback when the flag cannot be reserved, ordinary configured edits, and compatibility acknowledgement persistence. - Unit tests verify that a pending review is honoured by the packaged Commonlib replication service before remote activity begins. - Unit and Compose tests verify that ordinary P2P replication observes the policy, explicit P2P rebuild uses the setup bypass, and replacement leaves host actions on the current replicator. - A real-Obsidian settings test verifies the dedicated summary and details dialogues, captures representative screenshots, confirms that the acknowledged internal version advances only after explicit resume, and confirms that the Change Log contains no acknowledgement control. diff --git a/package.json b/package.json index 49e7b720..4a54e7fb 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts", "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", "test:e2e:obsidian:setting-markdown-export": "tsx test/e2e-obsidian/scripts/setting-markdown-export.ts", + "test:e2e:obsidian:upgrade-from-stable": "tsx test/e2e-obsidian/scripts/upgrade-from-stable.ts", "test:e2e:obsidian:local-suite": "tsx test/e2e-obsidian/scripts/local-suite.ts", "test:e2e:obsidian:local-suite:services": "tsx test/e2e-obsidian/scripts/local-suite.ts --manage-services", "test:docker-p2p:start": "docker compose -f test/fixtures/p2p-relay/compose.yml up -d", diff --git a/src/common/databaseCompatibility.ts b/src/common/databaseCompatibility.ts index 0cca8570..2b0a3c47 100644 --- a/src/common/databaseCompatibility.ts +++ b/src/common/databaseCompatibility.ts @@ -55,16 +55,6 @@ export interface CompatibilityEvaluationInput { legacyReviewMessage: string; } -const FILENAME_CASE_SENSITIVITY_UNRESOLVED = "filename-case-sensitivity-unresolved"; - -export function requiresFilenameCaseSensitivityDecision(pause: CompatibilityPause): boolean { - return pause.reasons.some( - (reason) => - reason.source === "settings-schema" && - reason.reviewReasons.some(({ code }) => code === FILENAME_CASE_SENSITIVITY_UNRESOLVED) - ); -} - function databaseVersionReason( acknowledgedVersion: string | null, currentVersion: number, diff --git a/src/common/databaseCompatibility.unit.spec.ts b/src/common/databaseCompatibility.unit.spec.ts index a988dd3c..a94e6b5b 100644 --- a/src/common/databaseCompatibility.unit.spec.ts +++ b/src/common/databaseCompatibility.unit.spec.ts @@ -109,11 +109,11 @@ describe("database compatibility evaluation", () => { }); }); - it("retains an unresolved filename-case decision in the host compatibility reason", () => { + it("retains a settings migration review in the host compatibility reason", () => { const reviewReasons = [ { - code: "filename-case-sensitivity-unresolved", - fromVersion: 10, + code: "legacy-update-review-pending", + fromVersion: 9, toVersion: 10, }, ]; @@ -121,7 +121,7 @@ describe("database compatibility evaluation", () => { acknowledgedVersion: "12", currentVersion: 12, migrationState: migrationState({ - sourceVersion: 10, + sourceVersion: 9, targetVersion: 10, requiresSyncReview: true, reviewReasons, @@ -131,7 +131,7 @@ describe("database compatibility evaluation", () => { expect(result.pause?.reasons).toContainEqual({ source: "settings-schema", - sourceVersion: 10, + sourceVersion: 9, currentVersion: 10, isFromFutureSchema: false, resumable: true, diff --git a/src/serviceFeatures/compatibilityReview.ts b/src/serviceFeatures/compatibilityReview.ts index d4670c2e..8fc85f3e 100644 --- a/src/serviceFeatures/compatibilityReview.ts +++ b/src/serviceFeatures/compatibilityReview.ts @@ -6,16 +6,10 @@ import { DATABASE_COMPATIBILITY_VERSION_KEY, evaluateCompatibilityPause, legacyDatabaseCompatibilityVersionKey, - requiresFilenameCaseSensitivityDecision, type CompatibilityPause, } from "@/common/databaseCompatibility.ts"; -export type CompatibilityReviewSummaryAction = - | "details" - | "resume" - | "use-case-sensitive" - | "keep-paused" - | false; +export type CompatibilityReviewSummaryAction = "details" | "resume" | "keep-paused" | false; export type CompatibilityReviewDetailsAction = "back" | false; // Explicit flag-file recovery runs at priorities 5, 10, and 20. Present the @@ -68,11 +62,25 @@ export class CompatibilityReviewController { if (this.disposed) return true; const setting = this.core.services.setting; const settings = setting.currentSettings(); + const migrationState = setting.getSettingsMigrationState(); + + // An existing unconfigured Vault cannot replicate, so a database + // compatibility pause would only compete with onboarding and persist + // a misleading sync warning. Do not acknowledge the missing marker: + // activation on a later start must evaluate the same state again. + // Genuinely new Vaults still initialise their marker below. + if (settings.isConfigured !== true && migrationState?.isNewVault !== true) { + this.pause = undefined; + this.ui.clearReminder(); + this._initialised = true; + return true; + } + const acknowledgedVersion = this.readAcknowledgedVersion(); const evaluation = evaluateCompatibilityPause({ acknowledgedVersion, currentVersion: this.currentVersion, - migrationState: setting.getSettingsMigrationState(), + migrationState, legacyReviewMessage: settings.versionUpFlash, }); @@ -95,26 +103,16 @@ export class CompatibilityReviewController { return true; } - private async acknowledge(options: { useCaseSensitiveFilenames?: boolean } = {}): Promise { + private async acknowledge(): Promise { if (!this.pause?.resumable) return; const setting = this.core.services.setting; const settings = setting.currentSettings(); const previousMessage = settings.versionUpFlash; - const hadFilenameCaseDecision = typeof settings.handleFilenameCaseSensitive === "boolean"; - const previousFilenameCaseDecision = settings.handleFilenameCaseSensitive; - if (options.useCaseSensitiveFilenames) { - settings.handleFilenameCaseSensitive = true; - } settings.versionUpFlash = ""; try { await setting.saveSettingData(); } catch (error) { settings.versionUpFlash = previousMessage || COMPATIBILITY_PAUSE_SETTING_MESSAGE; - if (hadFilenameCaseDecision) { - settings.handleFilenameCaseSensitive = previousFilenameCaseDecision; - } else { - delete (settings as Partial).handleFilenameCaseSensitive; - } throw error; } @@ -135,15 +133,9 @@ export class CompatibilityReviewController { break; } if (action === "resume" && this.pause.resumable) { - if (requiresFilenameCaseSensitivityDecision(this.pause)) break; await this.acknowledge(); return; } - if (action === "use-case-sensitive" && this.pause.resumable) { - if (!requiresFilenameCaseSensitivityDecision(this.pause)) break; - await this.acknowledge({ useCaseSensitiveFilenames: true }); - return; - } break; } if (this.pause && !this.disposed) { diff --git a/src/serviceFeatures/compatibilityReview.unit.spec.ts b/src/serviceFeatures/compatibilityReview.unit.spec.ts index 63b43d9b..3155477f 100644 --- a/src/serviceFeatures/compatibilityReview.unit.spec.ts +++ b/src/serviceFeatures/compatibilityReview.unit.spec.ts @@ -28,6 +28,7 @@ function createFixture( marker?: string | null; legacyMarker?: string | null; versionUpFlash?: string; + isConfigured?: boolean; migration?: Record; } = {} ) { @@ -39,10 +40,10 @@ function createFixture( if (options.legacyMarker !== undefined && options.legacyMarker !== null) { local.set(legacyKey, options.legacyMarker); } - const settings: { - versionUpFlash: string; - handleFilenameCaseSensitive?: boolean; - } = { versionUpFlash: options.versionUpFlash ?? "" }; + const settings = { + versionUpFlash: options.versionUpFlash ?? "", + isConfigured: options.isConfigured ?? true, + }; const saveSettingData = vi.fn().mockResolvedValue(undefined); const applySettings = vi.fn().mockResolvedValue(true); const setting = { @@ -77,7 +78,7 @@ describe("compatibility review controller", () => { }); it("initialises the acknowledged version for a new Vault without showing a pause", async () => { - const fixture = createFixture({ marker: null, migration: { isNewVault: true } }); + const fixture = createFixture({ marker: null, isConfigured: false, migration: { isNewVault: true } }); expect(fixture.controller.initialised).toBe(false); @@ -89,6 +90,30 @@ describe("compatibility review controller", () => { expect(fixture.saveSettingData).not.toHaveBeenCalled(); }); + it("defers a missing database marker while the Vault remains unconfigured", async () => { + const fixture = createFixture({ marker: null, isConfigured: false }); + + await expect(fixture.controller.initialise()).resolves.toBe(true); + + expect(fixture.controller.pendingPause).toBeUndefined(); + expect(fixture.settings.versionUpFlash).toBe(""); + expect(fixture.local.has(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe(false); + expect(fixture.saveSettingData).not.toHaveBeenCalled(); + + fixture.settings.isConfigured = true; + await expect(fixture.controller.initialise()).resolves.toBe(true); + + expect(fixture.controller.pendingPause?.reasons).toContainEqual({ + source: "database-version", + state: "missing", + currentVersion: 12, + resumable: true, + }); + expect(fixture.settings.versionUpFlash).toBe(COMPATIBILITY_PAUSE_SETTING_MESSAGE); + expect(fixture.local.has(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe(false); + expect(fixture.saveSettingData).toHaveBeenCalledOnce(); + }); + it("preserves preferences and advances the marker only after an upgrade review is resumed", async () => { const fixture = createFixture({ marker: "11" }); vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); @@ -109,61 +134,6 @@ describe("compatibility review controller", () => { expect(fixture.ui.clearReminder).toHaveBeenCalled(); }); - it("requires an explicit legacy-compatible filename-case decision before resuming", async () => { - const fixture = createFixture({ - marker: "12", - migration: { - sourceVersion: 10, - targetVersion: 10, - requiresSyncReview: true, - reviewReasons: [ - { - code: "filename-case-sensitivity-unresolved", - fromVersion: 10, - toVersion: 10, - }, - ], - }, - }); - vi.mocked(fixture.ui.showSummary).mockResolvedValue("use-case-sensitive" as never); - - await fixture.controller.initialise(); - await fixture.controller.openReview(); - - expect(fixture.settings.handleFilenameCaseSensitive).toBe(true); - expect(fixture.local.get(DATABASE_COMPATIBILITY_VERSION_KEY)).toBe("12"); - expect(fixture.saveSettingData).toHaveBeenCalledTimes(2); - expect(fixture.applySettings).toHaveBeenCalledOnce(); - expect(fixture.controller.pendingPause).toBeUndefined(); - }); - - it("does not let a generic resume action bypass an unresolved filename-case decision", async () => { - const fixture = createFixture({ - marker: "12", - migration: { - sourceVersion: 10, - targetVersion: 10, - requiresSyncReview: true, - reviewReasons: [ - { - code: "filename-case-sensitivity-unresolved", - fromVersion: 10, - toVersion: 10, - }, - ], - }, - }); - vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); - - await fixture.controller.initialise(); - await fixture.controller.openReview(); - - expect(fixture.settings.handleFilenameCaseSensitive).toBeUndefined(); - expect(fixture.applySettings).not.toHaveBeenCalled(); - expect(fixture.controller.pendingPause).toBeDefined(); - expect(fixture.ui.showReminder).toHaveBeenCalledOnce(); - }); - it("does not allow a downgrade pause to be resumed", async () => { const fixture = createFixture({ marker: "13" }); vi.mocked(fixture.ui.showSummary).mockResolvedValue("resume"); diff --git a/src/serviceFeatures/compatibilityReviewMarkdown.ts b/src/serviceFeatures/compatibilityReviewMarkdown.ts index e73cb544..c4730b3b 100644 --- a/src/serviceFeatures/compatibilityReviewMarkdown.ts +++ b/src/serviceFeatures/compatibilityReviewMarkdown.ts @@ -1,15 +1,9 @@ -import { - requiresFilenameCaseSensitivityDecision, - type CompatibilityPause, - type CompatibilityPauseReason, -} from "@/common/databaseCompatibility.ts"; +import type { CompatibilityPause, CompatibilityPauseReason } from "@/common/databaseCompatibility.ts"; export function compatibilityReviewSummaryMarkdown(pause: CompatibilityPause): string { const action = !pause.resumable ? "This installation cannot safely acknowledge the detected state. Update Self-hosted LiveSync before attempting to synchronise again." - : requiresFilenameCaseSensitivityDecision(pause) - ? "Before resuming, review the missing file-name case policy and explicitly keep the legacy-compatible behaviour, or leave synchronisation paused while you plan a database rebuild." - : "Before resuming, review the compatibility details and update Self-hosted LiveSync on every device which uses this remote database."; + : "Before resuming, review the compatibility details and update Self-hosted LiveSync on every device which uses this remote database."; return `Remote synchronisation is paused on this device because its compatibility state requires attention. ${action} @@ -34,9 +28,6 @@ function reasonMarkdown(reason: CompatibilityPauseReason): string { if (reason.isFromFutureSchema) { return `- The saved settings use schema **${reason.sourceVersion}**, which is newer than schema **${reason.currentVersion}** supported by this installation.`; } - if (reason.reviewReasons.some(({ code }) => code === "filename-case-sensitivity-unresolved")) { - return "- This existing Vault has no saved file-name case policy. Self-hosted LiveSync will not infer a cross-platform policy while synchronisation is paused."; - } return `- The settings were migrated from schema **${reason.sourceVersion}** to **${reason.currentVersion}** and require review before synchronisation resumes.`; } const escapedMessage = reason.message.replace(/[\\`*_{}[\]()<>#+.!|-]/gu, "\\$&"); @@ -46,9 +37,7 @@ function reasonMarkdown(reason: CompatibilityPauseReason): string { export function compatibilityReviewDetailsMarkdown(pause: CompatibilityPause): string { const resolution = !pause.resumable ? "Install a compatible current version of Self-hosted LiveSync. This pause cannot be dismissed by the current installation." - : requiresFilenameCaseSensitivityDecision(pause) - ? "Choosing 'Keep case-sensitive handling and resume' records an explicit decision; case-sensitive handling preserves the earlier behaviour. To adopt cross-platform case-insensitive handling, keep synchronisation paused and use the compatibility setting workflow after checking for paths which differ only by letter case; case-insensitive handling requires a database rebuild." - : "After all devices have been updated, return to the compatibility review summary and explicitly resume synchronisation. The current internal version will only then be recorded as acknowledged."; + : "After all devices have been updated, return to the compatibility review summary and explicitly resume synchronisation. The current internal version will only then be recorded as acknowledged."; return `## Why synchronisation is paused ${pause.reasons.map(reasonMarkdown).join("\n")} diff --git a/src/serviceFeatures/compatibilityReviewObsidian.ts b/src/serviceFeatures/compatibilityReviewObsidian.ts index 2a33e47c..623b111d 100644 --- a/src/serviceFeatures/compatibilityReviewObsidian.ts +++ b/src/serviceFeatures/compatibilityReviewObsidian.ts @@ -1,9 +1,6 @@ import { Notice } from "@/deps.ts"; import type { Confirm } from "@vrtmrz/livesync-commonlib/compat/interfaces/Confirm"; -import { - requiresFilenameCaseSensitivityDecision, - type CompatibilityPause, -} from "@/common/databaseCompatibility.ts"; +import type { CompatibilityPause } from "@/common/databaseCompatibility.ts"; import type { CompatibilityReviewDetailsAction, CompatibilityReviewSummaryAction, @@ -16,7 +13,6 @@ import { const REVIEW_DETAILS = "Review compatibility details"; const KEEP_PAUSED = "Keep synchronisation paused"; -const USE_CASE_SENSITIVE = "Keep case-sensitive handling and resume"; const RESUME = "Resume synchronisation"; const BACK = "Back to compatibility review"; @@ -28,9 +24,7 @@ export class ObsidianCompatibilityReviewUi implements CompatibilityReviewUi { async showSummary(pause: CompatibilityPause): Promise { const buttons = !pause.resumable ? ([REVIEW_DETAILS, KEEP_PAUSED] as const) - : requiresFilenameCaseSensitivityDecision(pause) - ? ([REVIEW_DETAILS, USE_CASE_SENSITIVE, KEEP_PAUSED] as const) - : ([REVIEW_DETAILS, RESUME, KEEP_PAUSED] as const); + : ([REVIEW_DETAILS, RESUME, KEEP_PAUSED] as const); const result = await this.confirm.confirmWithMessage( "Synchronisation paused for compatibility review", compatibilityReviewSummaryMarkdown(pause), @@ -40,7 +34,6 @@ export class ObsidianCompatibilityReviewUi implements CompatibilityReviewUi { "vertical" ); if (result === REVIEW_DETAILS) return "details"; - if (result === USE_CASE_SENSITIVE) return "use-case-sensitive"; if (result === RESUME) return "resume"; if (result === KEEP_PAUSED) return "keep-paused"; return false; diff --git a/src/serviceFeatures/compatibilityReviewObsidian.unit.spec.ts b/src/serviceFeatures/compatibilityReviewObsidian.unit.spec.ts index c6547052..17a961a0 100644 --- a/src/serviceFeatures/compatibilityReviewObsidian.unit.spec.ts +++ b/src/serviceFeatures/compatibilityReviewObsidian.unit.spec.ts @@ -9,22 +9,15 @@ vi.mock("@/deps.ts", () => ({ }, })); -const unresolvedFilenameCasePause: CompatibilityPause = { +const resumablePause: CompatibilityPause = { resumable: true, reasons: [ { - source: "settings-schema", - sourceVersion: 10, - currentVersion: 10, - isFromFutureSchema: false, + source: "database-version", + state: "upgrade", + acknowledgedVersion: 11, + currentVersion: 12, resumable: true, - reviewReasons: [ - { - code: "filename-case-sensitivity-unresolved", - fromVersion: 10, - toVersion: 10, - }, - ], }, ], }; @@ -49,23 +42,15 @@ describe("Obsidian compatibility review", () => { expect(details).toContain("does not mean that it is safe to resume automatically"); }); - it("explains the safe choices for an unresolved filename-case policy", () => { - const details = compatibilityReviewDetailsMarkdown(unresolvedFilenameCasePause); - expect(details).toContain("file-name case policy"); - expect(details).toContain("case-sensitive handling preserves the earlier behaviour"); - expect(details).toContain("case-insensitive handling requires a database rebuild"); - }); - - it("offers the legacy-compatible case decision instead of a generic resume action", async () => { - const label = "Keep case-sensitive handling and resume"; - const confirmWithMessage = vi.fn().mockResolvedValue(label); + it("offers the generic resume action in a vertical action dialogue", async () => { + const confirmWithMessage = vi.fn().mockResolvedValue("Resume synchronisation"); const ui = new ObsidianCompatibilityReviewUi({ confirmWithMessage } as never); - await expect(ui.showSummary(unresolvedFilenameCasePause)).resolves.toBe("use-case-sensitive"); + await expect(ui.showSummary(resumablePause)).resolves.toBe("resume"); expect(confirmWithMessage).toHaveBeenCalledWith( "Synchronisation paused for compatibility review", expect.any(String), - ["Review compatibility details", label, "Keep synchronisation paused"], + ["Review compatibility details", "Resume synchronisation", "Keep synchronisation paused"], "Keep synchronisation paused", undefined, "vertical" diff --git a/test/e2e-obsidian/runner/couchdb.ts b/test/e2e-obsidian/runner/couchdb.ts index 0f7c1550..11672549 100644 --- a/test/e2e-obsidian/runner/couchdb.ts +++ b/test/e2e-obsidian/runner/couchdb.ts @@ -26,6 +26,22 @@ export type CouchDbAllDocsResponse = { }>; }; +export type CouchDbLocalDocsResponse = { + rows: Array<{ + id: string; + key: string; + value: { rev: string }; + doc?: CouchDbDocument; + }>; +}; + +export type CouchDbDatabaseInfo = { + db_name: string; + doc_count: number; + doc_del_count: number; + update_seq: number | string; +}; + function parseEnvFile(content: string): Record { const entries = content .split(/\r?\n/u) @@ -170,6 +186,28 @@ export async function fetchAllCouchDbDocs(config: CouchDbConfig, dbName: string) return (await response.json()) as CouchDbAllDocsResponse; } +export async function fetchCouchDbLocalDocs(config: CouchDbConfig, dbName: string): Promise { + const response = await fetch(databaseUrl(config, dbName, "/_local_docs?include_docs=true"), { + headers: { authorization: authHeader(config) }, + }); + if (!response.ok) { + throw new Error( + `Failed to read CouchDB local documents from ${dbName}. HTTP ${response.status}: ${await response.text()}` + ); + } + return (await response.json()) as CouchDbLocalDocsResponse; +} + +export async function fetchCouchDbDatabaseInfo(config: CouchDbConfig, dbName: string): Promise { + const response = await fetch(databaseUrl(config, dbName), { + headers: { authorization: authHeader(config) }, + }); + if (!response.ok) { + throw new Error(`Failed to inspect CouchDB ${dbName}. HTTP ${response.status}: ${await response.text()}`); + } + return (await response.json()) as CouchDbDatabaseInfo; +} + export async function waitForCouchDbDocs( config: CouchDbConfig, dbName: string, diff --git a/test/e2e-obsidian/runner/liveSyncWorkflow.ts b/test/e2e-obsidian/runner/liveSyncWorkflow.ts index 2a2e5e2a..7bad3605 100644 --- a/test/e2e-obsidian/runner/liveSyncWorkflow.ts +++ b/test/e2e-obsidian/runner/liveSyncWorkflow.ts @@ -287,19 +287,7 @@ export async function configureObjectStorage( settings: ObjectStorageConfig & { bucketPrefix: string }, overrides: Record = {} ): Promise { - const nextSettings = { - remoteType: "MINIO", - endpoint: settings.endpoint, - accessKey: settings.accessKey, - secretKey: settings.secretKey, - bucket: settings.bucket, - region: settings.region, - forcePathStyle: settings.forcePathStyle, - bucketPrefix: settings.bucketPrefix, - bucketCustomHeaders: "", - ...E2E_PREFERRED_SETTINGS, - ...overrides, - }; + const nextSettings = createE2eObjectStoragePluginData(settings, overrides); return await evalObsidianJson( cliBinary, [ @@ -328,6 +316,25 @@ export async function configureObjectStorage( ); } +export function createE2eObjectStoragePluginData( + settings: ObjectStorageConfig & { bucketPrefix: string }, + overrides: Record = {} +): Record { + return { + remoteType: "MINIO", + endpoint: settings.endpoint, + accessKey: settings.accessKey, + secretKey: settings.secretKey, + bucket: settings.bucket, + region: settings.region, + forcePathStyle: settings.forcePathStyle, + bucketPrefix: settings.bucketPrefix, + bucketCustomHeaders: "", + ...E2E_PREFERRED_SETTINGS, + ...overrides, + }; +} + export async function waitForLiveSyncCoreReady( cliBinary: string, env: NodeJS.ProcessEnv, diff --git a/test/e2e-obsidian/runner/objectStorage.ts b/test/e2e-obsidian/runner/objectStorage.ts index f0ea8455..01fee5cf 100644 --- a/test/e2e-obsidian/runner/objectStorage.ts +++ b/test/e2e-obsidian/runner/objectStorage.ts @@ -1,6 +1,7 @@ import { CreateBucketCommand, DeleteObjectsCommand, + GetObjectCommand, ListObjectsV2Command, S3Client, type _Object, @@ -120,6 +121,22 @@ export async function listObjectStorageObjects(config: ObjectStorageConfig, pref } } +export async function readObjectStorageObject(config: ObjectStorageConfig, key: string): Promise { + const client = createObjectStorageClient(config); + try { + const response = await client.send(new GetObjectCommand({ Bucket: config.bucket, Key: key })); + if (!response.Body) throw new Error(`Object Storage returned an empty body for ${key}.`); + return await response.Body.transformToByteArray(); + } finally { + client.destroy(); + } +} + +export async function readObjectStorageJson(config: ObjectStorageConfig, key: string): Promise { + const bytes = await readObjectStorageObject(config, key); + return JSON.parse(new TextDecoder().decode(bytes)) as T; +} + export async function deleteObjectStoragePrefix(config: ObjectStorageConfig, prefix: string): Promise { const client = createObjectStorageClient(config); try { diff --git a/test/e2e-obsidian/runner/releaseArtifact.test.ts b/test/e2e-obsidian/runner/releaseArtifact.test.ts new file mode 100644 index 00000000..ef76b7ce --- /dev/null +++ b/test/e2e-obsidian/runner/releaseArtifact.test.ts @@ -0,0 +1,76 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + ensurePinnedReleaseArtifact, + type PinnedPluginRelease, +} from "./releaseArtifact.ts"; + +const temporaryDirectories: string[] = []; + +function sha256(content: string): string { + return createHash("sha256").update(content).digest("hex"); +} + +function fixtureRelease(contents: Record<"main.js" | "manifest.json" | "styles.css", string>): PinnedPluginRelease { + return { + pluginId: "fixture-plugin", + version: "1.2.3", + files: (Object.keys(contents) as Array).map((name) => ({ + name, + url: `https://example.invalid/${name}`, + sha256: sha256(contents[name]), + })), + }; +} + +afterEach(async () => { + for (const path of temporaryDirectories.splice(0)) { + await rm(path, { recursive: true, force: true }); + } +}); + +describe("pinned plug-in release artefacts", () => { + it("downloads, verifies, and reuses an immutable release cache", async () => { + const root = await mkdtemp(join(tmpdir(), "livesync-release-artifact-")); + temporaryDirectories.push(root); + const contents = { + "main.js": "console.log('fixture');\n", + "manifest.json": '{"id":"fixture-plugin","version":"1.2.3"}\n', + "styles.css": ".fixture {}\n", + }; + const release = fixtureRelease(contents); + const fetchImplementation = vi.fn(async (input: string | URL | Request) => { + const name = new URL(String(input)).pathname.split("/").pop() as keyof typeof contents; + return new Response(contents[name], { status: 200 }); + }) as unknown as typeof fetch; + + await expect( + ensurePinnedReleaseArtifact(release, { artifactRoot: root, fetchImplementation }) + ).resolves.toBe(root); + await expect(readFile(join(root, "main.js"), "utf8")).resolves.toBe(contents["main.js"]); + expect(fetchImplementation).toHaveBeenCalledTimes(3); + + await ensurePinnedReleaseArtifact(release, { artifactRoot: root, fetchImplementation }); + expect(fetchImplementation).toHaveBeenCalledTimes(3); + }); + + it("rejects a downloaded file before it enters the release cache when its checksum differs", async () => { + const root = await mkdtemp(join(tmpdir(), "livesync-release-artifact-")); + temporaryDirectories.push(root); + const contents = { + "main.js": "expected\n", + "manifest.json": '{"id":"fixture-plugin","version":"1.2.3"}\n', + "styles.css": ".fixture {}\n", + }; + const release = fixtureRelease(contents); + const fetchImplementation = vi.fn(async () => new Response("tampered\n", { status: 200 })) as unknown as typeof fetch; + + await expect( + ensurePinnedReleaseArtifact(release, { artifactRoot: root, fetchImplementation }) + ).rejects.toThrow("checksum mismatch"); + await expect(readFile(join(root, "main.js"))).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); diff --git a/test/e2e-obsidian/runner/releaseArtifact.ts b/test/e2e-obsidian/runner/releaseArtifact.ts new file mode 100644 index 00000000..2c0e62e1 --- /dev/null +++ b/test/e2e-obsidian/runner/releaseArtifact.ts @@ -0,0 +1,128 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +export type PinnedReleaseArtifactFile = { + name: "main.js" | "manifest.json" | "styles.css"; + url: string; + sha256: string; +}; + +export type PinnedPluginRelease = { + pluginId: string; + version: string; + files: readonly PinnedReleaseArtifactFile[]; +}; + +export type EnsurePinnedReleaseArtifactOptions = { + artifactRoot?: string; + fetchImplementation?: typeof fetch; +}; + +export const UPGRADE_SOURCE_RELEASE: PinnedPluginRelease = { + pluginId: "obsidian-livesync", + version: "0.25.83", + files: [ + { + name: "main.js", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/0.25.83/main.js", + sha256: "5e57f990635ab0cf2ff3879f3c6cb91ddfdbc146958d33d1e5d21f1869dff6a4", + }, + { + name: "manifest.json", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/0.25.83/manifest.json", + sha256: "4944f5665c94bcbb58db0e3708ec2bd8ee36118791271c01d085668876dc8ba6", + }, + { + name: "styles.css", + url: "https://github.com/vrtmrz/obsidian-livesync/releases/download/0.25.83/styles.css", + sha256: "37d31798186d7e97ea979e6d2aae8021ea1ac1df2c3b9d2b03dce269959c27f3", + }, + ], +}; + +function digest(content: Uint8Array): string { + return createHash("sha256").update(content).digest("hex"); +} + +function assertDigest(file: PinnedReleaseArtifactFile, content: Uint8Array): void { + const actual = digest(content); + if (actual !== file.sha256) { + throw new Error( + `Release artefact checksum mismatch for ${file.name}. Expected ${file.sha256}, received ${actual}.` + ); + } +} + +async function readCachedFile( + path: string, + file: PinnedReleaseArtifactFile +): Promise | undefined> { + try { + const content = new Uint8Array(await readFile(path)); + assertDigest(file, content); + return content; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +async function downloadVerifiedFile( + root: string, + file: PinnedReleaseArtifactFile, + fetchImplementation: typeof fetch +): Promise> { + const path = join(root, file.name); + const cached = await readCachedFile(path, file); + if (cached) return cached; + + const response = await fetchImplementation(file.url, { redirect: "follow" }); + if (!response.ok) { + throw new Error(`Could not download ${file.url}. HTTP ${response.status}: ${await response.text()}`); + } + const content = new Uint8Array(await response.arrayBuffer()); + assertDigest(file, content); + + const temporaryPath = `${path}.download-${process.pid}-${Date.now()}`; + try { + await writeFile(temporaryPath, content, { flag: "wx" }); + await rename(temporaryPath, path); + } finally { + await rm(temporaryPath, { force: true }); + } + return content; +} + +/** + * Materialise one immutable published plug-in release in the ignored E2E cache. + * + * Existing files are always verified before use. A mismatched cache is left in + * place for inspection and must be removed explicitly by the operator. + */ +export async function ensurePinnedReleaseArtifact( + release: PinnedPluginRelease = UPGRADE_SOURCE_RELEASE, + options: EnsurePinnedReleaseArtifactOptions = {} +): Promise { + const root = resolve( + options.artifactRoot ?? + process.env.E2E_LIVESYNC_SOURCE_ARTIFACT_ROOT?.trim() ?? + join("_testdata", "releases", release.pluginId, release.version) + ); + await mkdir(root, { recursive: true }); + + const fetched = new Map>(); + for (const file of release.files) { + fetched.set(file.name, await downloadVerifiedFile(root, file, options.fetchImplementation ?? fetch)); + } + + const manifestBytes = fetched.get("manifest.json"); + if (!manifestBytes) throw new Error("The pinned release does not define manifest.json."); + const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as { id?: unknown; version?: unknown }; + if (manifest.id !== release.pluginId || manifest.version !== release.version) { + throw new Error( + `Release manifest identity mismatch. Expected ${release.pluginId}@${release.version}, received ${String(manifest.id)}@${String(manifest.version)}.` + ); + } + return root; +} diff --git a/test/e2e-obsidian/runner/session.test.ts b/test/e2e-obsidian/runner/session.test.ts new file mode 100644 index 00000000..c959ffd3 --- /dev/null +++ b/test/e2e-obsidian/runner/session.test.ts @@ -0,0 +1,55 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { startObsidianPluginSession } from "@vrtmrz/obsidian-test-session"; +import { + startObsidianLiveSyncSession, + type StartObsidianLiveSyncSessionOptions, +} from "./session.ts"; + +vi.mock("@vrtmrz/obsidian-test-session", () => ({ + startObsidianPluginSession: vi.fn(async () => ({ + app: {}, + cliEnv: {}, + install: {}, + readiness: {}, + pluginId: "obsidian-livesync", + remoteDebuggingPort: 28052, + })), +})); + +describe("LiveSync real-Obsidian session", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("installs an explicitly selected plug-in artefact while retaining the supplied Vault and profile", async () => { + const vault = { + path: "/tmp/upgrade-vault", + statePath: "/tmp/upgrade-state", + name: "upgrade-vault", + id: "upgrade-vault-id", + homePath: "/tmp/upgrade-state/home", + xdgConfigPath: "/tmp/upgrade-state/xdg-config", + xdgCachePath: "/tmp/upgrade-state/xdg-cache", + xdgDataPath: "/tmp/upgrade-state/xdg-data", + userDataPath: "/tmp/upgrade-state/user-data", + processMarker: "/tmp/upgrade-state", + dispose: vi.fn(async () => undefined), + }; + const options: StartObsidianLiveSyncSessionOptions & { artifactRoot: string } = { + binary: "/Applications/Obsidian", + cliBinary: "obsidian-cli", + vault, + artifactRoot: "/tmp/obsidian-livesync-0.25.83", + }; + + await startObsidianLiveSyncSession(options); + + expect(startObsidianPluginSession).toHaveBeenCalledWith( + expect.objectContaining({ + artifactRoot: options.artifactRoot, + pluginId: "obsidian-livesync", + vault, + }) + ); + }); +}); diff --git a/test/e2e-obsidian/runner/session.ts b/test/e2e-obsidian/runner/session.ts index f6a45948..8f617587 100644 --- a/test/e2e-obsidian/runner/session.ts +++ b/test/e2e-obsidian/runner/session.ts @@ -7,6 +7,7 @@ export type StartObsidianLiveSyncSessionOptions = { binary: string; cliBinary: string; vault: TemporaryVault; + artifactRoot?: string; startupGraceMs?: number; pluginData?: Record; localStorageEntries?: Readonly>; @@ -21,7 +22,7 @@ export async function startObsidianLiveSyncSession( cliBinary: options.cliBinary, vault: options.vault, pluginId: "obsidian-livesync", - artifactRoot: process.cwd(), + artifactRoot: options.artifactRoot ?? process.cwd(), startupGraceMs: options.startupGraceMs, pluginData: options.pluginData, localStorageEntries: options.localStorageEntries, diff --git a/test/e2e-obsidian/runner/twoVaultSyncLifecycle.test.ts b/test/e2e-obsidian/runner/twoVaultSyncLifecycle.test.ts index b255114e..7ee47ffd 100644 --- a/test/e2e-obsidian/runner/twoVaultSyncLifecycle.test.ts +++ b/test/e2e-obsidian/runner/twoVaultSyncLifecycle.test.ts @@ -39,7 +39,6 @@ vi.mock("./liveSyncWorkflow.ts", () => ({ assertE2eCompatibilityReviewPending: vi.fn(async () => undefined), configureCouchDb: vi.fn(async () => undefined), createE2eCouchDbPluginData: vi.fn(() => ({})), - createE2eObsidianDeviceLocalState: vi.fn(() => ({})), prepareRemote: vi.fn(async () => undefined), pushLocalChanges: vi.fn(async () => { throw new Error("simulated Obsidian CLI timeout"); diff --git a/test/e2e-obsidian/runner/upgradeContinuity.test.ts b/test/e2e-obsidian/runner/upgradeContinuity.test.ts new file mode 100644 index 00000000..505cecc8 --- /dev/null +++ b/test/e2e-obsidian/runner/upgradeContinuity.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { + assertCouchDbCheckpointContinuity, + assertJournalCheckpointLoaded, + assertNoJournalReplay, + type JournalCheckpointSnapshot, +} from "./upgradeContinuity.ts"; + +const journalCheckpoint: JournalCheckpointSnapshot = { + remoteKey: "remote-a", + lastLocalSeq: 42, + journalEpoch: "2:salt", + knownIDs: ["known-a"], + sentIDs: ["sent-a"], + receivedFiles: ["100-docs.jsonl.gz"], + sentFiles: ["101-docs.jsonl.gz"], +}; + +describe("upgrade synchronisation continuity assertions", () => { + it("rejects a fresh CouchDB checkpoint lineage even when final documents could still converge", () => { + expect(() => + assertCouchDbCheckpointContinuity( + [{ id: "_local/original", lastSequence: 42 }], + [{ id: "_local/replacement", lastSequence: 42 }] + ) + ).toThrow("checkpoint identity changed"); + }); + + it("rejects an Object Storage checkpoint which was reset to its initial state", () => { + expect(() => + assertJournalCheckpointLoaded(journalCheckpoint, { + remoteKey: journalCheckpoint.remoteKey, + lastLocalSeq: 0, + journalEpoch: "", + knownIDs: [], + sentIDs: [], + receivedFiles: [], + sentFiles: [], + }) + ).toThrow(/lastLocalSeq regressed|history was lost/u); + }); + + it("rejects hidden Object Storage replay during an otherwise unchanged sync", () => { + expect(() => + assertNoJournalReplay( + journalCheckpoint, + journalCheckpoint, + [{ key: "101-docs.jsonl.gz", size: 10, etag: "etag" }], + [{ key: "101-docs.jsonl.gz", size: 10, etag: "etag" }], + { downloadedJournalKeys: ["101-docs.jsonl.gz"], uploadedJournalKeys: [] } + ) + ).toThrow("downloaded previously processed journals"); + }); +}); diff --git a/test/e2e-obsidian/runner/upgradeContinuity.ts b/test/e2e-obsidian/runner/upgradeContinuity.ts new file mode 100644 index 00000000..b5426265 --- /dev/null +++ b/test/e2e-obsidian/runner/upgradeContinuity.ts @@ -0,0 +1,199 @@ +export type CouchDbCheckpointSnapshot = { + id: string; + lastSequence: unknown; +}; + +export type CouchDbDocumentRevision = { + id: string; + revision: string; + deleted: boolean; +}; + +export type JournalCheckpointSnapshot = { + remoteKey: string; + lastLocalSeq: number | string; + journalEpoch: string; + knownIDs: readonly string[]; + sentIDs: readonly string[]; + receivedFiles: readonly string[]; + sentFiles: readonly string[]; +}; + +export type JournalIoObservation = { + downloadedJournalKeys: readonly string[]; + uploadedJournalKeys: readonly string[]; +}; + +export type RemoteObjectSnapshot = { + key: string; + size: number; + etag: string; +}; + +export type MilestoneIdentity = { + created: unknown; + locked: boolean; + acceptedNodes: readonly string[]; +}; + +function sorted(values: readonly string[]): string[] { + return [...values].sort((left, right) => left.localeCompare(right)); +} + +function assertEqualStrings(actual: readonly string[], expected: readonly string[], message: string): void { + const actualSorted = sorted(actual); + const expectedSorted = sorted(expected); + if (JSON.stringify(actualSorted) !== JSON.stringify(expectedSorted)) { + throw new Error(`${message}\nExpected: ${JSON.stringify(expectedSorted)}\nActual: ${JSON.stringify(actualSorted)}`); + } +} + +function assertSubset(previous: readonly string[], current: readonly string[], message: string): void { + const currentSet = new Set(current); + const missing = previous.filter((value) => !currentSet.has(value)); + if (missing.length > 0) throw new Error(`${message}: ${missing.join(", ")}`); +} + +function sequenceNumber(sequence: unknown): number | undefined { + if (typeof sequence === "number" && Number.isFinite(sequence)) return sequence; + if (typeof sequence !== "string") return undefined; + const match = /^(\d+)/u.exec(sequence); + return match ? Number(match[1]) : undefined; +} + +function assertSequenceDidNotRegress(before: unknown, after: unknown, label: string): void { + const beforeNumber = sequenceNumber(before); + const afterNumber = sequenceNumber(after); + if (beforeNumber !== undefined && afterNumber !== undefined) { + if (afterNumber < beforeNumber) { + throw new Error(`${label} regressed from ${String(before)} to ${String(after)}.`); + } + return; + } + if (before !== after) { + throw new Error(`${label} changed from an opaque sequence ${String(before)} to ${String(after)}.`); + } +} + +export function assertCouchDbCheckpointContinuity( + before: readonly CouchDbCheckpointSnapshot[], + after: readonly CouchDbCheckpointSnapshot[] +): void { + if (before.length === 0) throw new Error("The stable release did not create a CouchDB replication checkpoint."); + assertEqualStrings( + after.map(({ id }) => id), + before.map(({ id }) => id), + "The CouchDB replication checkpoint identity changed during the upgrade." + ); + const afterById = new Map(after.map((checkpoint) => [checkpoint.id, checkpoint])); + for (const checkpoint of before) { + assertSequenceDidNotRegress( + checkpoint.lastSequence, + afterById.get(checkpoint.id)?.lastSequence, + `CouchDB checkpoint ${checkpoint.id}` + ); + } +} + +export function assertSomeCouchDbCheckpointAdvanced( + before: readonly CouchDbCheckpointSnapshot[], + after: readonly CouchDbCheckpointSnapshot[] +): void { + assertCouchDbCheckpointContinuity(before, after); + const afterById = new Map(after.map((checkpoint) => [checkpoint.id, checkpoint])); + const advanced = before.some((checkpoint) => { + const previous = sequenceNumber(checkpoint.lastSequence); + const current = sequenceNumber(afterById.get(checkpoint.id)?.lastSequence); + return previous !== undefined && current !== undefined && current > previous; + }); + if (!advanced) throw new Error("No CouchDB replication checkpoint advanced after the post-upgrade change."); +} + +export function assertCouchDbDocumentsUnchanged( + before: readonly CouchDbDocumentRevision[], + after: readonly CouchDbDocumentRevision[] +): void { + const serialise = (documents: readonly CouchDbDocumentRevision[]) => + [...documents].sort((left, right) => left.id.localeCompare(right.id)); + if (JSON.stringify(serialise(before)) !== JSON.stringify(serialise(after))) { + throw new Error("A no-op post-upgrade CouchDB synchronisation changed ordinary remote documents."); + } +} + +export function assertJournalCheckpointLoaded( + before: JournalCheckpointSnapshot, + after: JournalCheckpointSnapshot +): void { + if (sequenceNumber(before.lastLocalSeq) === 0) { + throw new Error("The stable release did not advance the Object Storage local checkpoint."); + } + if (after.remoteKey !== before.remoteKey) { + throw new Error(`The Object Storage checkpoint key changed from ${before.remoteKey} to ${after.remoteKey}.`); + } + assertSequenceDidNotRegress(before.lastLocalSeq, after.lastLocalSeq, "Object Storage lastLocalSeq"); + assertSubset(before.knownIDs, after.knownIDs, "Object Storage known revision history was lost"); + assertSubset(before.sentIDs, after.sentIDs, "Object Storage sent revision history was lost"); + assertSubset(before.receivedFiles, after.receivedFiles, "Object Storage received journal history was lost"); + assertSubset(before.sentFiles, after.sentFiles, "Object Storage sent journal history was lost"); + if (before.journalEpoch && after.journalEpoch !== before.journalEpoch) { + throw new Error( + `The Object Storage journal epoch changed from ${before.journalEpoch} to ${after.journalEpoch}.` + ); + } +} + +export function assertNoJournalReplay( + beforeCheckpoint: JournalCheckpointSnapshot, + afterCheckpoint: JournalCheckpointSnapshot, + beforeObjects: readonly RemoteObjectSnapshot[], + afterObjects: readonly RemoteObjectSnapshot[], + observation: JournalIoObservation +): void { + assertJournalCheckpointLoaded(beforeCheckpoint, afterCheckpoint); + assertEqualStrings( + afterObjects.map(({ key }) => key), + beforeObjects.map(({ key }) => key), + "A no-op post-upgrade Object Storage synchronisation changed the journal object set." + ); + if (observation.downloadedJournalKeys.length > 0) { + throw new Error( + `The no-op synchronisation downloaded previously processed journals: ${observation.downloadedJournalKeys.join(", ")}` + ); + } + if (observation.uploadedJournalKeys.length > 0) { + throw new Error( + `The no-op synchronisation uploaded replay journals: ${observation.uploadedJournalKeys.join(", ")}` + ); + } +} + +export function assertJournalCheckpointAdvanced( + before: JournalCheckpointSnapshot, + after: JournalCheckpointSnapshot, + observation: JournalIoObservation +): void { + assertJournalCheckpointLoaded(before, after); + const beforeSequence = sequenceNumber(before.lastLocalSeq); + const afterSequence = sequenceNumber(after.lastLocalSeq); + if (beforeSequence === undefined || afterSequence === undefined || afterSequence <= beforeSequence) { + throw new Error( + `The Object Storage checkpoint did not advance after the post-upgrade change (${String(before.lastLocalSeq)} -> ${String(after.lastLocalSeq)}).` + ); + } + if (observation.uploadedJournalKeys.length === 0) { + throw new Error("The post-upgrade Object Storage change did not create a new journal."); + } +} + +export function assertMilestoneContinuity(before: MilestoneIdentity, after: MilestoneIdentity): void { + if (before.created === undefined || before.created === null) { + throw new Error("The stable release milestone does not expose a remote generation identity."); + } + if (after.created !== before.created) { + throw new Error(`The remote milestone generation changed from ${String(before.created)} to ${String(after.created)}.`); + } + if (after.locked !== before.locked) { + throw new Error(`The remote milestone lock changed from ${String(before.locked)} to ${String(after.locked)}.`); + } + assertSubset(before.acceptedNodes, after.acceptedNodes, "The remote milestone lost an accepted device"); +} diff --git a/test/e2e-obsidian/runner/upgradeWorkflow.ts b/test/e2e-obsidian/runner/upgradeWorkflow.ts new file mode 100644 index 00000000..7d4f35be --- /dev/null +++ b/test/e2e-obsidian/runner/upgradeWorkflow.ts @@ -0,0 +1,837 @@ +import { mkdir, readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { evalObsidianJson } from "./cli.ts"; +import type { CouchDbConfig } from "./couchdb.ts"; +import type { ObjectStorageConfig } from "./objectStorage.ts"; +import { withObsidianPage } from "./ui.ts"; +import type { + CouchDbCheckpointSnapshot, + JournalCheckpointSnapshot, + JournalIoObservation, +} from "./upgradeContinuity.ts"; +import { waitForLocalDatabaseEntry } from "./liveSyncWorkflow.ts"; +import type { TemporaryVault } from "./vault.ts"; + +export const STABLE_RELEASE_VERSION = "0.25.83"; + +export type UpgradeTransportConfiguration = + | { + kind: "couchdb"; + config: CouchDbConfig; + databaseName: string; + } + | { + kind: "object-storage"; + config: ObjectStorageConfig; + bucketPrefix: string; + }; + +export type UpgradeScenarioPaths = { + original: string; + renamed: string; + deleted: string; + postUpgrade: string; + returnFromVerifier: string; +}; + +export type RuntimeUpgradeState = { + pluginVersion: string; + vaultName: string; + localDatabaseName: string; + localDatabaseUpdateSequence: number | string; + localDatabaseDocumentCount: number; + nodeId: string; + legacyCompatibilityMarker: string | null; + compatibilityMarker: string; + compatibilityStorageEntries: Record; + migrationState?: { + sourceVersion: number; + targetVersion: number; + isNewVault: boolean; + isFromFutureSchema: boolean; + changed: boolean; + requiresSyncReview: boolean; + reviewReasons: Array<{ code: string; fromVersion: number; toVersion: number }>; + }; + settings: { + isConfigured: boolean | undefined; + settingVersion: number; + versionUpFlash: string; + liveSync: boolean; + syncOnStart: boolean; + syncOnSave: boolean; + syncOnEditorSave: boolean; + syncOnFileOpen: boolean; + syncAfterMerge: boolean; + periodicReplication: boolean; + encrypt: boolean; + usePathObfuscation: boolean; + syncInternalFiles: boolean; + customChunkSize: number; + usePluginSyncV2: boolean; + enableCompression: boolean; + useEden: boolean; + filenameCaseType: string; + handleFilenameCaseSensitive?: boolean; + doNotUseFixedRevisionForChunks: boolean; + chunkSplitterVersion: string; + E2EEAlgorithm: string; + additionalSuffixOfDatabaseName: string; + remoteType: string; + couchDB_DBNAME: string; + endpoint: string; + bucket: string; + bucketPrefix: string; + activeConfigurationId: string; + remoteConfigurationIds: string[]; + doctorProcessedVersion: string; + }; +}; + +export type RuntimeSettingsUpgradeState = Pick & { + compatibilityMarker: string; +}; + +export type CouchDbReplicationObservation = { + succeeded: boolean; + sentDocuments: number; + arrivedDocuments: number; +}; + +export type JournalReplicationObservation = JournalIoObservation & { + succeeded: boolean; +}; + +const firstContent = "# Stable release history\n\nCreated before the 1.0 upgrade.\n"; +const editedContent = "# Stable release history\n\nEdited and renamed before the 1.0 upgrade.\n"; +const deletedContent = "# Deleted before upgrade\n\nThis note must not be resurrected.\n"; +const postUpgradeContent = "# Post-upgrade delta\n\nCreated by the upgraded 1.0 device.\n"; +const returnContent = "# Return journey\n\nCreated by a fresh 1.0 verifier device.\n"; + +function assertEqual(actual: unknown, expected: unknown, message: string): void { + if (actual !== expected) { + throw new Error(`${message}\nExpected: ${String(expected)}\nActual: ${String(actual)}`); + } +} + +function assertStringArraysEqual(actual: readonly string[], expected: readonly string[], message: string): void { + const actualSorted = [...actual].sort(); + const expectedSorted = [...expected].sort(); + if (JSON.stringify(actualSorted) !== JSON.stringify(expectedSorted)) { + throw new Error( + `${message}\nExpected: ${JSON.stringify(expectedSorted)}\nActual: ${JSON.stringify(actualSorted)}` + ); + } +} + +export function createUpgradeScenarioPaths(label: string): UpgradeScenarioPaths { + const root = `E2E/upgrade-from-${STABLE_RELEASE_VERSION}/${label}`; + return { + original: `${root}/rename-source.md`, + renamed: `${root}/renamed.md`, + deleted: `${root}/deleted.md`, + postUpgrade: `${root}/post-upgrade.md`, + returnFromVerifier: `${root}/return-from-verifier.md`, + }; +} + +function remoteSettings(configuration: UpgradeTransportConfiguration): Record { + if (configuration.kind === "couchdb") { + return { + remoteType: "", + couchDB_URI: configuration.config.uri, + couchDB_USER: configuration.config.username, + couchDB_PASSWORD: configuration.config.password, + couchDB_DBNAME: configuration.databaseName, + isConfigured: true, + }; + } + return { + remoteType: "MINIO", + endpoint: configuration.config.endpoint, + accessKey: configuration.config.accessKey, + secretKey: configuration.config.secretKey, + bucket: configuration.config.bucket, + region: configuration.config.region, + forcePathStyle: configuration.config.forcePathStyle, + bucketPrefix: configuration.bucketPrefix, + bucketCustomHeaders: "", + isConfigured: true, + }; +} + +export async function configureStableRelease( + cliBinary: string, + environment: NodeJS.ProcessEnv, + configuration: UpgradeTransportConfiguration +): Promise { + const partial = remoteSettings(configuration); + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + `const partial=${JSON.stringify(partial)};`, + "await core.services.setting.applyExternalSettings(partial,true);", + "await core.services.control.applySettings();", + "return JSON.stringify({ok:true});", + "})()", + ].join(""), + environment + ); +} + +export async function prepareStableRemote(cliBinary: string, environment: NodeJS.ProcessEnv): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const settings=core.services.setting.currentSettings();", + "const replicator=core.services.replicator.getActiveReplicator();", + "await replicator.tryCreateRemoteDatabase(settings);", + "await replicator.markRemoteResolved(settings);", + "return JSON.stringify({ok:true});", + "})()", + ].join(""), + environment + ); +} + +export async function waitForPersistentNodeIdentity( + cliBinary: string, + environment: NodeJS.ProcessEnv, + timeoutMs = Number(process.env.E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS ?? 15000) +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const deadline=Date.now()+${JSON.stringify(timeoutMs)};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const database=core.localDatabase.localDatabase;", + "const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));", + "let persistent='';let active='';", + "while(Date.now()null);", + "persistent=typeof nodeInfo?.nodeid==='string'?nodeInfo.nodeid:'';", + "active=core.services.replicator.getActiveReplicator()?.nodeid??'';", + "if(persistent!==''&&active===persistent) return JSON.stringify(persistent);", + "await sleep(100);", + "}", + "throw new Error(`Timed out waiting for persistent node identity: persistent=${persistent}, active=${active}`);", + "})()", + ].join(""), + environment + ); +} + +export async function readRuntimeUpgradeState( + cliBinary: string, + environment: NodeJS.ProcessEnv +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const plugin=app.plugins.plugins['obsidian-livesync'];", + "const core=plugin.core;", + "const setting=core.services.setting;", + "const settings=setting.currentSettings();", + "const vaultName=core.services.vault.getVaultName();", + "const replicator=core.services.replicator.getActiveReplicator();", + "const databaseInfo=await core.localDatabase.localDatabase.info();", + "const nodeInfo=await core.localDatabase.localDatabase.get('_local/obsydian_livesync_nodeinfo').catch(()=>null);", + "const migrationState=setting.getSettingsMigrationState?.();", + "return JSON.stringify({", + "pluginVersion:app.plugins.manifests['obsidian-livesync']?.version??'unknown',", + "vaultName,", + "localDatabaseName:databaseInfo.db_name,", + "localDatabaseUpdateSequence:databaseInfo.update_seq,", + "localDatabaseDocumentCount:databaseInfo.doc_count,", + "nodeId:nodeInfo?.nodeid??replicator?.nodeid??'',", + "legacyCompatibilityMarker:localStorage.getItem(`obsidian-live-sync-ver${vaultName}`),", + "compatibilityMarker:setting.getSmallConfig('database-compatibility-version')??'',", + "compatibilityStorageEntries:Object.fromEntries(Array.from({length:localStorage.length},(_,index)=>localStorage.key(index))", + ".filter((key)=>key!==null)", + ".filter(key=>key.startsWith('obsidian-live-sync-ver')||key.endsWith('-database-compatibility-version'))", + ".map(key=>[key,localStorage.getItem(key)??''])),", + "migrationState,", + "settings:{", + "isConfigured:settings.isConfigured,settingVersion:settings.settingVersion,", + "versionUpFlash:settings.versionUpFlash,", + "liveSync:settings.liveSync,syncOnStart:settings.syncOnStart,syncOnSave:settings.syncOnSave,", + "syncOnEditorSave:settings.syncOnEditorSave,syncOnFileOpen:settings.syncOnFileOpen,", + "syncAfterMerge:settings.syncAfterMerge,periodicReplication:settings.periodicReplication,", + "encrypt:settings.encrypt,usePathObfuscation:settings.usePathObfuscation,", + "syncInternalFiles:settings.syncInternalFiles,customChunkSize:settings.customChunkSize,", + "usePluginSyncV2:settings.usePluginSyncV2,enableCompression:settings.enableCompression,", + "useEden:settings.useEden,filenameCaseType:typeof settings.handleFilenameCaseSensitive,", + "handleFilenameCaseSensitive:settings.handleFilenameCaseSensitive,", + "doNotUseFixedRevisionForChunks:settings.doNotUseFixedRevisionForChunks,", + "chunkSplitterVersion:settings.chunkSplitterVersion,E2EEAlgorithm:settings.E2EEAlgorithm,", + "additionalSuffixOfDatabaseName:settings.additionalSuffixOfDatabaseName??'',", + "remoteType:settings.remoteType,couchDB_DBNAME:settings.couchDB_DBNAME??'',", + "endpoint:settings.endpoint??'',bucket:settings.bucket??'',bucketPrefix:settings.bucketPrefix??'',", + "activeConfigurationId:settings.activeConfigurationId??'',", + "remoteConfigurationIds:Object.keys(settings.remoteConfigurations??{}),", + "doctorProcessedVersion:settings.doctorProcessedVersion??'',", + "}", + "});", + "})()", + ].join(""), + environment + ); +} + +export async function readRuntimeSettingsUpgradeState( + cliBinary: string, + environment: NodeJS.ProcessEnv +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const plugin=app.plugins.plugins['obsidian-livesync'];", + "const setting=plugin.core.services.setting;", + "const settings=setting.currentSettings();", + "const migrationState=setting.getSettingsMigrationState?.();", + "return JSON.stringify({", + "pluginVersion:app.plugins.manifests['obsidian-livesync']?.version??'unknown',", + "compatibilityMarker:setting.getSmallConfig?.('database-compatibility-version')??'',", + "migrationState,", + "settings:{", + "isConfigured:settings.isConfigured,settingVersion:settings.settingVersion,", + "versionUpFlash:settings.versionUpFlash,", + "liveSync:settings.liveSync,syncOnStart:settings.syncOnStart,syncOnSave:settings.syncOnSave,", + "syncOnEditorSave:settings.syncOnEditorSave,syncOnFileOpen:settings.syncOnFileOpen,", + "syncAfterMerge:settings.syncAfterMerge,periodicReplication:settings.periodicReplication,", + "encrypt:settings.encrypt,usePathObfuscation:settings.usePathObfuscation,", + "syncInternalFiles:settings.syncInternalFiles,customChunkSize:settings.customChunkSize,", + "usePluginSyncV2:settings.usePluginSyncV2,enableCompression:settings.enableCompression,", + "useEden:settings.useEden,filenameCaseType:typeof settings.handleFilenameCaseSensitive,", + "handleFilenameCaseSensitive:settings.handleFilenameCaseSensitive,", + "doNotUseFixedRevisionForChunks:settings.doNotUseFixedRevisionForChunks,", + "chunkSplitterVersion:settings.chunkSplitterVersion,E2EEAlgorithm:settings.E2EEAlgorithm,", + "additionalSuffixOfDatabaseName:settings.additionalSuffixOfDatabaseName??'',", + "remoteType:settings.remoteType,couchDB_DBNAME:settings.couchDB_DBNAME??'',", + "endpoint:settings.endpoint??'',bucket:settings.bucket??'',bucketPrefix:settings.bucketPrefix??'',", + "activeConfigurationId:settings.activeConfigurationId??'',", + "remoteConfigurationIds:Object.keys(settings.remoteConfigurations??{}),", + "doctorProcessedVersion:settings.doctorProcessedVersion??'',", + "}", + "});", + "})()", + ].join(""), + environment + ); +} + +export function assertStableReleaseDefaults(state: RuntimeSettingsUpgradeState, configured: boolean): void { + assertEqual( + state.pluginVersion, + STABLE_RELEASE_VERSION, + "The source session did not load the pinned stable release." + ); + assertEqual(state.settings.isConfigured, configured, "The stable release configuration lifecycle was unexpected."); + assertEqual(state.settings.settingVersion, 10, "The stable release settings schema was not version 10."); + assertEqual(state.settings.liveSync, false, "The stable release LiveSync default changed."); + assertEqual(state.settings.syncOnStart, false, "The stable release sync-on-start default changed."); + assertEqual(state.settings.syncOnSave, false, "The stable release sync-on-save default changed."); + assertEqual(state.settings.syncOnEditorSave, false, "The stable release editor-save default changed."); + assertEqual(state.settings.syncOnFileOpen, false, "The stable release file-open default changed."); + assertEqual(state.settings.syncAfterMerge, false, "The stable release post-merge default changed."); + assertEqual(state.settings.periodicReplication, false, "The stable release periodic default changed."); + assertEqual(state.settings.encrypt, false, "The stable release encryption default changed."); + assertEqual(state.settings.usePathObfuscation, false, "The stable release path-obfuscation default changed."); + assertEqual(state.settings.syncInternalFiles, false, "The stable release Hidden File default changed."); + assertEqual(state.settings.customChunkSize, 0, "The stable release custom chunk default changed."); + assertEqual(state.settings.usePluginSyncV2, false, "The stable release Customisation Sync V2 default changed."); + assertEqual(state.settings.enableCompression, false, "The stable release compression default changed."); + assertEqual(state.settings.useEden, false, "The stable release Eden default changed."); + assertEqual( + state.settings.filenameCaseType, + "undefined", + "The stable release filename-case decision was preselected." + ); + assertEqual( + state.settings.doNotUseFixedRevisionForChunks, + true, + "The stable release fixed-revision compatibility value changed." + ); + assertEqual(state.settings.chunkSplitterVersion, "v3-rabin-karp", "The stable release chunk splitter changed."); + assertEqual(state.settings.E2EEAlgorithm, "v2", "The stable release E2EE algorithm changed."); + assertEqual( + state.settings.remoteConfigurationIds.length, + configured ? 1 : 0, + "The stable release remote-profile count was unexpected." + ); +} + +export function assertUnconfiguredUpgradeReady( + stable: RuntimeSettingsUpgradeState, + upgraded: RuntimeSettingsUpgradeState, + targetVersion: string +): void { + assertStableReleaseDefaults(stable, false); + assertEqual(upgraded.pluginVersion, targetVersion, "The unconfigured Vault did not load the target artefact."); + assertEqual(upgraded.settings.isConfigured, false, "The upgrade changed an unconfigured Vault to configured."); + assertEqual( + upgraded.settings.usePluginSyncV2, + stable.settings.usePluginSyncV2, + "The upgrade applied a new-Vault recommendation to a non-empty legacy store." + ); + assertEqual( + upgraded.settings.handleFilenameCaseSensitive, + false, + "The unconfigured legacy Vault did not retain case-insensitive handling." + ); + assertEqual(upgraded.settings.versionUpFlash, "", "The unconfigured Vault was paused for compatibility review."); + assertEqual( + upgraded.compatibilityMarker, + "", + "The unconfigured Vault acknowledged database compatibility before activation." + ); + if (!upgraded.migrationState) throw new Error("The unconfigured settings migration state was not available."); + assertEqual(upgraded.migrationState.isNewVault, false, "The non-empty legacy store was treated as a new store."); + // The real-session helper deliberately reloads an already enabled plug-in. + // The first target load performs and persists the migration; the observed + // post-reload state can therefore report changed=false. The workflow reads + // data.json after stopping the session to prove the persisted values. + assertEqual( + upgraded.migrationState.requiresSyncReview, + false, + "The unconfigured legacy settings unexpectedly required compatibility review." + ); + assertEqual(upgraded.migrationState.reviewReasons.length, 0, "The unconfigured migration emitted a review reason."); +} + +export function assertUnconfiguredUpgradeRestarted(state: RuntimeSettingsUpgradeState, targetVersion: string): void { + assertEqual(state.pluginVersion, targetVersion, "The unconfigured restart did not load the target artefact."); + assertEqual(state.settings.isConfigured, false, "The unconfigured state was not persisted across restart."); + assertEqual(state.settings.usePluginSyncV2, false, "Restart applied a new-Vault recommendation."); + assertEqual(state.settings.handleFilenameCaseSensitive, false, "Restart lost the case-insensitive policy."); + assertEqual( + state.compatibilityMarker, + "", + "Restart acknowledged database compatibility while the Vault remained unconfigured." + ); + if (!state.migrationState) throw new Error("The restarted settings migration state was not available."); + assertEqual(state.migrationState.changed, false, "The settings migration was not idempotent after restart."); + assertEqual(state.migrationState.requiresSyncReview, false, "Restart introduced a compatibility review."); +} + +export function assertStableRemoteSelection( + state: RuntimeUpgradeState, + configuration: UpgradeTransportConfiguration +): void { + assertEqual(state.settings.isConfigured, true, "The stable release was not marked as configured."); + if (!state.settings.activeConfigurationId) throw new Error("The stable release did not select its remote profile."); + if (!state.settings.remoteConfigurationIds.includes(state.settings.activeConfigurationId)) { + throw new Error("The stable release active remote profile was not persisted."); + } + if (configuration.kind === "couchdb") { + assertEqual(state.settings.remoteType, "", "The stable release did not select CouchDB."); + assertEqual( + state.settings.couchDB_DBNAME, + configuration.databaseName, + "The stable release CouchDB database changed." + ); + } else { + assertEqual(state.settings.remoteType, "MINIO", "The stable release did not select Object Storage."); + assertEqual(state.settings.endpoint, configuration.config.endpoint, "The Object Storage endpoint changed."); + assertEqual(state.settings.bucket, configuration.config.bucket, "The Object Storage bucket changed."); + assertEqual(state.settings.bucketPrefix, configuration.bucketPrefix, "The Object Storage prefix changed."); + } +} + +export function assertUpgradeCompatibilityReady( + stable: RuntimeUpgradeState, + upgraded: RuntimeUpgradeState, + targetVersion: string, + configuration: UpgradeTransportConfiguration +): void { + assertEqual(upgraded.pluginVersion, targetVersion, "The upgraded session did not load the target artefact."); + assertEqual( + upgraded.localDatabaseName, + stable.localDatabaseName, + "The upgrade opened a different local synchronisation database." + ); + if (upgraded.localDatabaseDocumentCount < stable.localDatabaseDocumentCount) { + throw new Error("The upgrade lost local synchronisation documents before its first sync."); + } + if (stable.nodeId.length === 0) { + throw new Error("The stable release did not persist a device node identity."); + } + assertEqual(upgraded.nodeId, stable.nodeId, "The upgrade changed the persistent device node identity."); + assertEqual( + upgraded.settings.additionalSuffixOfDatabaseName, + stable.settings.additionalSuffixOfDatabaseName, + "The upgrade changed the local database suffix." + ); + assertStringArraysEqual( + upgraded.settings.remoteConfigurationIds, + stable.settings.remoteConfigurationIds, + "The upgrade changed the stored remote-profile identities." + ); + assertEqual( + upgraded.settings.activeConfigurationId, + stable.settings.activeConfigurationId, + "The upgrade changed the active remote profile." + ); + assertStableRemoteSelection(upgraded, configuration); + for (const key of [ + "liveSync", + "syncOnStart", + "syncOnSave", + "syncOnEditorSave", + "syncOnFileOpen", + "syncAfterMerge", + "periodicReplication", + "customChunkSize", + "usePluginSyncV2", + "enableCompression", + "useEden", + "doNotUseFixedRevisionForChunks", + ] as const) { + assertEqual(upgraded.settings[key], stable.settings[key], `The upgrade rewrote the stored ${key} preference.`); + } + if (!upgraded.migrationState) throw new Error("The 1.0 settings migration state was not available."); + // The session helper reloads the enabled target after its first load has + // persisted the normalised case value. Runtime settings below and the + // later restart prove that persisted result without depending on whether + // this observation came from the first or second load. + assertEqual( + upgraded.migrationState.requiresSyncReview, + false, + "The legacy case-insensitive setting unexpectedly required compatibility review." + ); + assertEqual(upgraded.migrationState.reviewReasons.length, 0, "The settings migration emitted a spurious review."); + assertEqual(stable.legacyCompatibilityMarker, "12", "The stable release did not persist its legacy marker."); + assertEqual(upgraded.legacyCompatibilityMarker, null, "The upgrade did not retire the legacy marker."); + assertEqual( + upgraded.compatibilityMarker, + "12", + [ + "The upgrade did not migrate the legacy compatibility marker.", + `Vault: ${upgraded.vaultName}`, + `Database suffix: ${upgraded.settings.additionalSuffixOfDatabaseName}`, + `Device-local entries: ${JSON.stringify(upgraded.compatibilityStorageEntries)}`, + ].join("\n") + ); + assertEqual(upgraded.settings.versionUpFlash, "", "Synchronisation was unexpectedly paused after migration."); + assertEqual( + upgraded.settings.handleFilenameCaseSensitive, + false, + "The missing legacy filename-case value did not preserve case-insensitive handling." + ); +} + +export function assertUpgradeRemainsReady(state: RuntimeUpgradeState, targetVersion: string): void { + assertEqual(state.pluginVersion, targetVersion, "The upgraded session changed target artefact."); + assertEqual(state.settings.versionUpFlash, "", "A compatibility pause reappeared."); + assertEqual(state.compatibilityMarker, "12", "The compatibility acknowledgement was not persisted."); + assertEqual( + state.settings.handleFilenameCaseSensitive, + false, + "The migrated legacy case-insensitive policy was not persisted." + ); +} + +export async function dismissConfigDoctorIfShown(port: number): Promise { + const timeoutMs = Number(process.env.E2E_OBSIDIAN_UI_TIMEOUT_MS ?? 10000); + return await withObsidianPage(port, async (page) => { + const doctor = page.locator(".modal-container").filter({ + has: page.locator(".modal-title").filter({ hasText: "Self-hosted LiveSync Config Doctor" }), + }); + const visible = await doctor + .waitFor({ state: "visible", timeout: Math.min(timeoutMs, 5000) }) + .then(() => true) + .catch(() => false); + if (!visible) return false; + await doctor.getByRole("button", { name: /No, and do not ask again/u }).click(); + await doctor.waitFor({ state: "hidden", timeout: timeoutMs }); + return true; + }); +} + +async function writeNote( + cliBinary: string, + environment: NodeJS.ProcessEnv, + path: string, + content: string +): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(path)};`, + `const content=${JSON.stringify(content)};`, + "const folder=path.split('/').slice(0,-1).join('/');", + "if(folder&&!(await app.vault.adapter.exists(folder))) await app.vault.createFolder(folder);", + "const existing=app.vault.getAbstractFileByPath(path);", + "if(existing) await app.vault.modify(existing,content); else await app.vault.create(path,content);", + "return JSON.stringify({ok:true});", + "})()", + ].join(""), + environment + ); +} + +async function renameNote( + cliBinary: string, + environment: NodeJS.ProcessEnv, + fromPath: string, + toPath: string +): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const fromPath=${JSON.stringify(fromPath)};`, + `const toPath=${JSON.stringify(toPath)};`, + "const folder=toPath.split('/').slice(0,-1).join('/');", + "if(folder&&!(await app.vault.adapter.exists(folder))) await app.vault.createFolder(folder);", + "const existing=app.vault.getAbstractFileByPath(fromPath);", + "if(!existing) throw new Error(`Could not find note to rename: ${fromPath}`);", + "await app.vault.rename(existing,toPath);", + "return JSON.stringify({ok:true});", + "})()", + ].join(""), + environment + ); +} + +async function deleteNote(cliBinary: string, environment: NodeJS.ProcessEnv, path: string): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(path)};`, + "const existing=app.vault.getAbstractFileByPath(path);", + "if(!existing) throw new Error(`Could not find note to delete: ${path}`);", + "await app.vault.delete(existing);", + "return JSON.stringify({ok:true});", + "})()", + ].join(""), + environment + ); +} + +async function waitForChangedRevision( + cliBinary: string, + environment: NodeJS.ProcessEnv, + path: string, + previousRevision: string +): Promise { + const timeoutMs = Number(process.env.E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS ?? 15000); + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(path)};`, + `const previousRevision=${JSON.stringify(previousRevision)};`, + `const deadline=Date.now()+${JSON.stringify(timeoutMs)};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));", + "while(Date.now()false);", + "if(entry&&entry._rev&&entry._rev!==previousRevision) return JSON.stringify({rev:entry._rev});", + "await sleep(250);", + "}", + "throw new Error(`Timed out waiting for a changed local revision: ${path}`);", + "})()", + ].join(""), + environment + ); +} + +export async function runStableFileHistory( + cliBinary: string, + environment: NodeJS.ProcessEnv, + paths: UpgradeScenarioPaths, + synchronise: () => Promise +): Promise { + await writeNote(cliBinary, environment, paths.original, firstContent); + await writeNote(cliBinary, environment, paths.deleted, deletedContent); + const originalEntry = await waitForLocalDatabaseEntry(cliBinary, environment, paths.original); + await waitForLocalDatabaseEntry(cliBinary, environment, paths.deleted); + await synchronise(); + + await writeNote(cliBinary, environment, paths.original, editedContent); + await waitForChangedRevision(cliBinary, environment, paths.original, originalEntry.rev); + await synchronise(); + + await renameNote(cliBinary, environment, paths.original, paths.renamed); + await waitForLocalDatabaseEntry(cliBinary, environment, paths.renamed); + await synchronise(); + + await deleteNote(cliBinary, environment, paths.deleted); + await synchronise(); +} + +export async function createPostUpgradeDelta( + cliBinary: string, + environment: NodeJS.ProcessEnv, + paths: UpgradeScenarioPaths +): Promise { + await writeNote(cliBinary, environment, paths.postUpgrade, postUpgradeContent); + await waitForLocalDatabaseEntry(cliBinary, environment, paths.postUpgrade); +} + +export async function createVerifierReturnDelta( + cliBinary: string, + environment: NodeJS.ProcessEnv, + paths: UpgradeScenarioPaths +): Promise { + await writeNote(cliBinary, environment, paths.returnFromVerifier, returnContent); + await waitForLocalDatabaseEntry(cliBinary, environment, paths.returnFromVerifier); +} + +async function pathExists(vault: TemporaryVault, path: string): Promise { + try { + await readFile(join(vault.path, path)); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +async function waitForPathContent(vault: TemporaryVault, path: string, content: string): Promise { + const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ?? 30000); + let lastContent = ""; + while (Date.now() < deadline) { + try { + lastContent = await readFile(join(vault.path, path), "utf8"); + if (lastContent === content) return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`Timed out waiting for ${path}. Last content:\n${lastContent}`); +} + +export async function verifyPreUpgradeHistory(vault: TemporaryVault, paths: UpgradeScenarioPaths): Promise { + await waitForPathContent(vault, paths.renamed, editedContent); + if (await pathExists(vault, paths.original)) throw new Error(`Renamed source was resurrected: ${paths.original}`); + if (await pathExists(vault, paths.deleted)) throw new Error(`Deleted note was resurrected: ${paths.deleted}`); +} + +export async function verifyPostUpgradeHistory(vault: TemporaryVault, paths: UpgradeScenarioPaths): Promise { + await verifyPreUpgradeHistory(vault, paths); + await waitForPathContent(vault, paths.postUpgrade, postUpgradeContent); +} + +export async function verifyReturnDelta(vault: TemporaryVault, paths: UpgradeScenarioPaths): Promise { + await waitForPathContent(vault, paths.returnFromVerifier, returnContent); +} + +export async function ensureScenarioDirectory(vault: TemporaryVault, paths: UpgradeScenarioPaths): Promise { + await mkdir(dirname(join(vault.path, paths.original)), { recursive: true }); +} + +export async function runCouchDbReplicationObserved( + cliBinary: string, + environment: NodeJS.ProcessEnv +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const replicator=core.services.replicator.getActiveReplicator();", + "await core.services.fileProcessing.commitPendingFileEvents();", + "const beforeSent=Number(replicator.docSent??0);", + "const beforeArrived=Number(replicator.docArrived??0);", + "const result=await core.services.replication.replicate(true);", + "return JSON.stringify({", + "succeeded:!!result,", + "sentDocuments:Number(replicator.docSent??0)-beforeSent,", + "arrivedDocuments:Number(replicator.docArrived??0)-beforeArrived,", + "});", + "})()", + ].join(""), + environment + ); +} + +export async function runJournalReplicationObserved( + cliBinary: string, + environment: NodeJS.ProcessEnv +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const replicator=core.services.replicator.getActiveReplicator();", + "const client=replicator.client;", + "const storage=client.storage;", + "const originalDownload=storage.download.bind(storage);", + "const originalUpload=storage.upload.bind(storage);", + "const downloadedJournalKeys=[];const uploadedJournalKeys=[];", + "const isJournal=(key)=>!String(key).split('/').pop().startsWith('_');", + "storage.download=async(key,...args)=>{if(isJournal(key))downloadedJournalKeys.push(String(key));return await originalDownload(key,...args);};", + "storage.upload=async(key,...args)=>{if(isJournal(key))uploadedJournalKeys.push(String(key));return await originalUpload(key,...args);};", + "let succeeded=false;", + "try{", + "await core.services.fileProcessing.commitPendingFileEvents();", + "succeeded=!!(await core.services.replication.replicate(true));", + "}finally{storage.download=originalDownload;storage.upload=originalUpload;}", + "return JSON.stringify({succeeded,downloadedJournalKeys,uploadedJournalKeys});", + "})()", + ].join(""), + environment + ); +} + +export async function readJournalCheckpoint( + cliBinary: string, + environment: NodeJS.ProcessEnv +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const replicator=core.services.replicator.getActiveReplicator();", + "const client=replicator.client;", + "const checkpoint=await client.getCheckpointInfo();", + "const sorted=(value)=>[...(value??[])].sort();", + "return JSON.stringify({", + "remoteKey:client.getRemoteKey(),lastLocalSeq:checkpoint.lastLocalSeq,journalEpoch:checkpoint.journalEpoch,", + "knownIDs:sorted(checkpoint.knownIDs),sentIDs:sorted(checkpoint.sentIDs),", + "receivedFiles:sorted(checkpoint.receivedFiles),sentFiles:sorted(checkpoint.sentFiles),", + "});", + "})()", + ].join(""), + environment + ); +} + +export async function readLocalCouchDbCheckpoints( + cliBinary: string, + environment: NodeJS.ProcessEnv, + checkpointIds: readonly string[] +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const ids=${JSON.stringify(checkpointIds)};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const database=core.localDatabase.localDatabase;", + "const checkpoints=[];", + "for(const id of ids){", + "const doc=await database.get(id).catch(()=>false);", + "if(doc&&Object.prototype.hasOwnProperty.call(doc,'last_seq')) checkpoints.push({id,lastSequence:doc.last_seq});", + "}", + "return JSON.stringify(checkpoints);", + "})()", + ].join(""), + environment + ); +} diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 8fc86665..9ecbdbb5 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -23,6 +23,7 @@ const focusedScenarios = new Set([ "hidden-file-snippet-sync", "customisation-sync", "setting-markdown-export", + "upgrade-from-stable", ]); function usage(): string { diff --git a/test/e2e-obsidian/scripts/upgrade-from-stable.ts b/test/e2e-obsidian/scripts/upgrade-from-stable.ts new file mode 100644 index 00000000..f6e75559 --- /dev/null +++ b/test/e2e-obsidian/scripts/upgrade-from-stable.ts @@ -0,0 +1,753 @@ +import { spawn } from "node:child_process"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { basename, resolve } from "node:path"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + fetchAllCouchDbDocs, + fetchCouchDbDatabaseInfo, + fetchCouchDbLocalDocs, + loadCouchDbConfig, + makeUniqueDatabaseName, + type CouchDbConfig, + type CouchDbDatabaseInfo, + type CouchDbDocument, +} from "../runner/couchdb.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + configureCouchDb, + configureObjectStorage, + createE2eCouchDbPluginData, + createE2eObjectStoragePluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + pushLocalChanges, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { + deleteObjectStoragePrefix, + ensureObjectStorageBucket, + listObjectStorageObjects, + loadObjectStorageConfig, + makeUniqueBucketPrefix, + readObjectStorageJson, + type ObjectStorageConfig, +} from "../runner/objectStorage.ts"; +import { ensurePinnedReleaseArtifact, UPGRADE_SOURCE_RELEASE } from "../runner/releaseArtifact.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { + assertCouchDbCheckpointContinuity, + assertCouchDbDocumentsUnchanged, + assertJournalCheckpointAdvanced, + assertJournalCheckpointLoaded, + assertMilestoneContinuity, + assertNoJournalReplay, + assertSomeCouchDbCheckpointAdvanced, + type CouchDbCheckpointSnapshot, + type CouchDbDocumentRevision, + type MilestoneIdentity, + type RemoteObjectSnapshot, +} from "../runner/upgradeContinuity.ts"; +import { + assertStableReleaseDefaults, + assertStableRemoteSelection, + assertUnconfiguredUpgradeReady, + assertUnconfiguredUpgradeRestarted, + assertUpgradeCompatibilityReady, + assertUpgradeRemainsReady, + configureStableRelease, + createPostUpgradeDelta, + createUpgradeScenarioPaths, + createVerifierReturnDelta, + dismissConfigDoctorIfShown, + prepareStableRemote, + readJournalCheckpoint, + readLocalCouchDbCheckpoints, + readRuntimeUpgradeState, + readRuntimeSettingsUpgradeState, + runCouchDbReplicationObserved, + runJournalReplicationObserved, + runStableFileHistory, + STABLE_RELEASE_VERSION, + verifyPostUpgradeHistory, + verifyPreUpgradeHistory, + verifyReturnDelta, + waitForPersistentNodeIdentity, + type CouchDbReplicationObservation, + type RuntimeUpgradeState, + type UpgradeTransportConfiguration, +} from "../runner/upgradeWorkflow.ts"; +import { obsidianRemoteDebuggingPort } from "../runner/ui.ts"; +import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "90000"; + +type Transport = "couchdb" | "object-storage"; + +type RemoteMilestone = CouchDbDocument & { + created?: unknown; + locked?: unknown; + accepted_nodes?: unknown; + tweak_values?: unknown; +}; + +type CouchDbRemoteSnapshot = { + checkpoints: CouchDbCheckpointSnapshot[]; + documents: CouchDbDocumentRevision[]; + info: CouchDbDatabaseInfo; + milestone: MilestoneIdentity; + preferredTweaks: Record; +}; + +type ObjectStorageRemoteSnapshot = { + journalObjects: RemoteObjectSnapshot[]; + milestone: MilestoneIdentity; + preferredTweaks: Record; +}; + +type RunnerContext = { + binary: string; + cliBinary: string; + sourceArtifactRoot: string; + targetArtifactRoot: string; + targetVersion: string; + activeSessions: Set; +}; + +type ParsedArguments = { + transports: Transport[]; + manageServices: boolean; + keepServices: boolean; +}; + +type StartSessionOptions = { + pluginData?: Record; + localStorageEntries?: Readonly>; + waitForCoreReady?: boolean; +}; + +const MILESTONE_ID = "_local/obsydian_livesync_milestone"; +const JOURNAL_MILESTONE_NAME = "_00000000-milestone.json"; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function assertEqual(actual: unknown, expected: unknown, message: string): void { + if (actual !== expected) { + throw new Error(`${message}\nExpected: ${String(expected)}\nActual: ${String(actual)}`); + } +} + +function parseArguments(argv: readonly string[]): ParsedArguments { + let transportValue = "all"; + for (let index = 0; index < argv.length; index++) { + const argument = argv[index]; + if (argument === "--transport") { + transportValue = argv[index + 1] ?? ""; + index++; + } else if (argument.startsWith("--transport=")) { + transportValue = argument.slice("--transport=".length); + } + } + const transports: Transport[] = + transportValue === "all" + ? ["couchdb", "object-storage"] + : transportValue === "couchdb" || transportValue === "object-storage" + ? [transportValue] + : (() => { + throw new Error(`Unsupported transport '${transportValue}'. Use couchdb, object-storage, or all.`); + })(); + return { + transports, + manageServices: argv.includes("--manage-services"), + keepServices: argv.includes("--keep-services"), + }; +} + +function sessionEnvironment(port: number): NodeJS.ProcessEnv { + return { ...process.env, E2E_OBSIDIAN_REMOTE_DEBUGGING_PORT: String(port) }; +} + +function sessionPorts(): readonly [number, number] { + const first = obsidianRemoteDebuggingPort(process.env); + const second = Number(process.env.E2E_OBSIDIAN_SECONDARY_REMOTE_DEBUGGING_PORT ?? first + 1); + if (!Number.isInteger(second) || second < 1 || second > 65535 || second === first) { + throw new Error(`Invalid secondary Obsidian remote debugging port: ${second}`); + } + return [first, second]; +} + +function npmBinary(): string { + return process.platform === "win32" ? "npm.cmd" : "npm"; +} + +function runNpmScript(name: string, optional = false): Promise { + return new Promise((resolvePromise, reject) => { + console.log(`\n# ${name}`); + const child = spawn(npmBinary(), ["run", name], { + cwd: process.cwd(), + env: process.env, + stdio: "inherit", + }); + child.on("error", reject); + child.on("exit", (code, signal) => { + if (code === 0 || optional) { + if (code !== 0) { + console.warn(`${name} did not complete cleanly (${signal ? `signal ${signal}` : `exit ${code}`}).`); + } + resolvePromise(); + return; + } + reject(new Error(`${name} failed (${signal ? `signal ${signal}` : `exit ${code}`}).`)); + }); + }); +} + +async function validateTargetArtifact(root: string): Promise { + await Promise.all( + ["main.js", "manifest.json", "styles.css"].map(async (name) => await access(resolve(root, name))) + ); + const manifest = JSON.parse(await readFile(resolve(root, "manifest.json"), "utf8")) as { + id?: unknown; + version?: unknown; + }; + assertEqual(manifest.id, UPGRADE_SOURCE_RELEASE.pluginId, "The target artefact has an unexpected plug-in id."); + assert(typeof manifest.version === "string" && manifest.version.length > 0, "The target manifest has no version."); + assert( + manifest.version !== STABLE_RELEASE_VERSION, + `The target artefact is still the source release ${STABLE_RELEASE_VERSION}.` + ); + return manifest.version; +} + +async function startSession( + context: RunnerContext, + vault: TemporaryVault, + port: number, + artifactRoot: string, + options: StartSessionOptions = {} +): Promise { + const session = await startObsidianLiveSyncSession({ + binary: context.binary, + cliBinary: context.cliBinary, + vault, + artifactRoot, + pluginData: options.pluginData, + localStorageEntries: options.localStorageEntries, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + env: sessionEnvironment(port), + }); + context.activeSessions.add(session); + try { + if (options.waitForCoreReady !== false) { + await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); + } + return session; + } catch (error) { + await stopSession(context, session).catch(() => undefined); + throw error; + } +} + +async function readStoredPluginData(vault: TemporaryVault): Promise> { + const path = resolve(vault.path, ".obsidian", "plugins", "obsidian-livesync", "data.json"); + return JSON.parse(await readFile(path, "utf8")) as Record; +} + +async function writeStoredPluginData(vault: TemporaryVault, data: Record): Promise { + const path = resolve(vault.path, ".obsidian", "plugins", "obsidian-livesync", "data.json"); + await writeFile(path, `${JSON.stringify(data, null, 2)}\n`); +} + +async function runUnconfiguredSettingsUpgrade(context: RunnerContext, port: number): Promise { + console.log(`\n# Upgrade from ${STABLE_RELEASE_VERSION}: unconfigured legacy settings`); + const vault = await createTemporaryVault("obsidian-livesync-upgrade-unconfigured-"); + + try { + let session = await startSession(context, vault, port, context.sourceArtifactRoot, { + pluginData: { liveSync: false }, + waitForCoreReady: false, + }); + const stableState = await readRuntimeSettingsUpgradeState(context.cliBinary, session.cliEnv); + assertStableReleaseDefaults(stableState, false); + await stopSession(context, session); + + const stableData = await readStoredPluginData(vault); + if (stableData.isConfigured !== undefined) { + assertEqual( + stableData.isConfigured, + false, + "The stable release persisted a configured state for its default-equivalent settings." + ); + } + + // 0.25.83 infers the runtime boolean, but persistence depends on an + // unrelated settings-save event. Restore the pre-flag document + // explicitly so the target proves the direct legacy migration in + // either case rather than depending on that timing. + await writeStoredPluginData(vault, { liveSync: false }); + + session = await startSession(context, vault, port, context.targetArtifactRoot, { + waitForCoreReady: false, + }); + const upgradedState = await readRuntimeSettingsUpgradeState(context.cliBinary, session.cliEnv); + assertUnconfiguredUpgradeReady(stableState, upgradedState, context.targetVersion); + await stopSession(context, session); + + const migratedData = await readStoredPluginData(vault); + assertEqual(migratedData.isConfigured, false, "The inferred unconfigured state was not saved."); + assertEqual( + migratedData.handleFilenameCaseSensitive, + false, + "The inferred case-insensitive setting was not saved." + ); + + session = await startSession(context, vault, port, context.targetArtifactRoot, { + waitForCoreReady: false, + }); + const restartedState = await readRuntimeSettingsUpgradeState(context.cliBinary, session.cliEnv); + assertUnconfiguredUpgradeRestarted(restartedState, context.targetVersion); + await stopSession(context, session); + + console.log( + `PASS unconfigured settings: ${STABLE_RELEASE_VERSION} -> ${context.targetVersion}; legacy inference, persistence, and restart idempotence verified.` + ); + } finally { + await stopSessions(context); + await vault.dispose(); + } +} + +async function stopSession(context: RunnerContext, session: ObsidianLiveSyncSession): Promise { + if (!context.activeSessions.has(session)) return; + await session.app.stop(); + context.activeSessions.delete(session); +} + +async function stopSessions(context: RunnerContext): Promise { + for (const session of [...context.activeSessions]) await stopSession(context, session); +} + +function milestoneIdentity(document: RemoteMilestone): MilestoneIdentity { + assert(document.created !== undefined && document.created !== null, "The remote milestone has no generation."); + assert(typeof document.locked === "boolean", "The remote milestone has no lock state."); + assert(Array.isArray(document.accepted_nodes), "The remote milestone has no accepted-device list."); + assert( + document.accepted_nodes.every((value) => typeof value === "string"), + "The remote milestone accepted-device list is malformed." + ); + return { + created: document.created, + locked: document.locked, + acceptedNodes: document.accepted_nodes, + }; +} + +function preferredTweaks(document: RemoteMilestone): Record { + const values = document.tweak_values; + assert(values !== null && typeof values === "object" && !Array.isArray(values), "The remote has no tweak map."); + const preferred = (values as Record).PREFERRED; + assert( + preferred !== null && typeof preferred === "object" && !Array.isArray(preferred), + "The remote has no preferred tweak settings." + ); + return { ...(preferred as Record) }; +} + +async function readCouchDbRemoteSnapshot(config: CouchDbConfig, databaseName: string): Promise { + const [allDocs, localDocs, info] = await Promise.all([ + fetchAllCouchDbDocs(config, databaseName), + fetchCouchDbLocalDocs(config, databaseName), + fetchCouchDbDatabaseInfo(config, databaseName), + ]); + const milestone = localDocs.rows.find(({ id }) => id === MILESTONE_ID)?.doc as RemoteMilestone | undefined; + assert(milestone, "The CouchDB remote milestone is missing."); + const checkpoints = localDocs.rows.flatMap(({ id, doc }) => + doc && Object.prototype.hasOwnProperty.call(doc, "last_seq") ? [{ id, lastSequence: doc.last_seq }] : [] + ); + const documents = allDocs.rows.map(({ id, value }) => ({ + id, + revision: value.rev, + deleted: value.deleted === true, + })); + return { + checkpoints, + documents, + info, + milestone: milestoneIdentity(milestone), + preferredTweaks: preferredTweaks(milestone), + }; +} + +async function readObjectStorageRemoteSnapshot( + config: ObjectStorageConfig, + prefix: string +): Promise { + const [objects, milestone] = await Promise.all([ + listObjectStorageObjects(config, prefix), + readObjectStorageJson(config, `${prefix}${JOURNAL_MILESTONE_NAME}`), + ]); + const journalObjects = objects.flatMap((object) => { + if (!object.Key || basename(object.Key).startsWith("_")) return []; + return [ + { + key: object.Key, + size: object.Size ?? 0, + etag: object.ETag ?? "", + }, + ]; + }); + return { + journalObjects, + milestone: milestoneIdentity(milestone), + preferredTweaks: preferredTweaks(milestone), + }; +} + +function assertNoOpCouchDbObservation(observation: CouchDbReplicationObservation): void { + assert(observation.succeeded, "The first post-upgrade CouchDB synchronisation failed."); + assertEqual(observation.sentDocuments, 0, "The no-op CouchDB synchronisation resent documents."); + assertEqual(observation.arrivedDocuments, 0, "The no-op CouchDB synchronisation refetched documents."); +} + +function assertNoOpCouchDbDatabase(before: CouchDbRemoteSnapshot, after: CouchDbRemoteSnapshot): void { + assertCouchDbCheckpointContinuity(before.checkpoints, after.checkpoints); + assertCouchDbDocumentsUnchanged(before.documents, after.documents); + assertEqual( + after.info.update_seq, + before.info.update_seq, + "The no-op CouchDB synchronisation advanced update_seq." + ); + assertEqual(after.info.doc_count, before.info.doc_count, "The no-op CouchDB synchronisation changed doc_count."); + assertMilestoneContinuity(before.milestone, after.milestone); +} + +function assertRestartContinuity(before: RuntimeUpgradeState, after: RuntimeUpgradeState): void { + assertEqual(after.localDatabaseName, before.localDatabaseName, "Restart opened a different local database."); + assertEqual(after.nodeId, before.nodeId, "Restart changed the device node identity."); + assertEqual( + after.settings.activeConfigurationId, + before.settings.activeConfigurationId, + "Restart changed the active remote profile." + ); +} + +async function configureFreshCouchDbVerifier( + context: RunnerContext, + session: ObsidianLiveSyncSession, + config: CouchDbConfig, + databaseName: string, + tweaks: Record +): Promise { + await configureCouchDb( + context.cliBinary, + session.cliEnv, + { uri: config.uri, username: config.username, password: config.password, dbName: databaseName }, + tweaks + ); + await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); + await prepareRemote(context.cliBinary, session.cliEnv); +} + +async function configureFreshObjectStorageVerifier( + context: RunnerContext, + session: ObsidianLiveSyncSession, + config: ObjectStorageConfig, + prefix: string, + tweaks: Record +): Promise { + await configureObjectStorage(context.cliBinary, session.cliEnv, { ...config, bucketPrefix: prefix }, tweaks); + await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); + await prepareRemote(context.cliBinary, session.cliEnv); +} + +async function runCouchDbUpgrade(context: RunnerContext, ports: readonly [number, number]): Promise { + console.log(`\n# Upgrade from ${STABLE_RELEASE_VERSION}: CouchDB`); + const config = await loadCouchDbConfig(); + const databaseName = makeUniqueDatabaseName(config.dbPrefix, "upgrade-from-stable"); + const remote: UpgradeTransportConfiguration = { kind: "couchdb", config, databaseName }; + const paths = createUpgradeScenarioPaths("couchdb"); + const upgradeVault = await createTemporaryVault("obsidian-livesync-upgrade-couchdb-"); + const verifierVault = await createTemporaryVault("obsidian-livesync-upgrade-couchdb-verifier-"); + let upgradedSession: ObsidianLiveSyncSession | undefined; + + try { + await assertCouchDbReachable(config); + await createCouchDbDatabase(config, databaseName); + + let session = await startSession(context, upgradeVault, ports[0], context.sourceArtifactRoot); + assertStableReleaseDefaults(await readRuntimeUpgradeState(context.cliBinary, session.cliEnv), false); + await configureStableRelease(context.cliBinary, session.cliEnv, remote); + const configuredStable = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + assertStableReleaseDefaults(configuredStable, true); + assertStableRemoteSelection(configuredStable, remote); + await stopSession(context, session); + + session = await startSession(context, upgradeVault, ports[0], context.sourceArtifactRoot); + const restartedStable = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + assertStableReleaseDefaults(restartedStable, true); + assertStableRemoteSelection(restartedStable, remote); + await waitForPersistentNodeIdentity(context.cliBinary, session.cliEnv); + await prepareStableRemote(context.cliBinary, session.cliEnv); + await runStableFileHistory(context.cliBinary, session.cliEnv, paths, async () => { + const result = await runCouchDbReplicationObserved(context.cliBinary, session.cliEnv); + assert(result.succeeded, "The stable CouchDB synchronisation failed."); + }); + await verifyPreUpgradeHistory(upgradeVault, paths); + + const stableState = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + const stableRemote = await readCouchDbRemoteSnapshot(config, databaseName); + const stableLocalCheckpoints = await readLocalCouchDbCheckpoints( + context.cliBinary, + session.cliEnv, + stableRemote.checkpoints.map(({ id }) => id) + ); + assertCouchDbCheckpointContinuity(stableRemote.checkpoints, stableLocalCheckpoints); + await stopSession(context, session); + + session = await startSession(context, upgradeVault, ports[0], context.targetArtifactRoot); + upgradedSession = session; + await dismissConfigDoctorIfShown(session.remoteDebuggingPort); + const upgradedState = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + assertUpgradeCompatibilityReady(stableState, upgradedState, context.targetVersion, remote); + await verifyPreUpgradeHistory(upgradeVault, paths); + + const loadedLocalCheckpoints = await readLocalCouchDbCheckpoints( + context.cliBinary, + session.cliEnv, + stableRemote.checkpoints.map(({ id }) => id) + ); + assertCouchDbCheckpointContinuity(stableLocalCheckpoints, loadedLocalCheckpoints); + const noOpObservation = await runCouchDbReplicationObserved(context.cliBinary, session.cliEnv); + assertNoOpCouchDbObservation(noOpObservation); + const noOpRemote = await readCouchDbRemoteSnapshot(config, databaseName); + assertNoOpCouchDbDatabase(stableRemote, noOpRemote); + + await createPostUpgradeDelta(context.cliBinary, session.cliEnv, paths); + const deltaObservation = await runCouchDbReplicationObserved(context.cliBinary, session.cliEnv); + assert(deltaObservation.succeeded, "The post-upgrade CouchDB delta failed."); + assert(deltaObservation.sentDocuments > 0, "The post-upgrade CouchDB delta sent no documents."); + const deltaRemote = await readCouchDbRemoteSnapshot(config, databaseName); + assertSomeCouchDbCheckpointAdvanced(noOpRemote.checkpoints, deltaRemote.checkpoints); + assertMilestoneContinuity(noOpRemote.milestone, deltaRemote.milestone); + + const verifierSettings = { + uri: config.uri, + username: config.username, + password: config.password, + dbName: databaseName, + }; + const verifier = await startSession(context, verifierVault, ports[1], context.targetArtifactRoot, { + pluginData: createE2eCouchDbPluginData(verifierSettings, deltaRemote.preferredTweaks), + localStorageEntries: createE2eObsidianDeviceLocalState(verifierVault.name), + }); + await configureFreshCouchDbVerifier(context, verifier, config, databaseName, deltaRemote.preferredTweaks); + await pushLocalChanges(context.cliBinary, verifier.cliEnv); + await verifyPostUpgradeHistory(verifierVault, paths); + await createVerifierReturnDelta(context.cliBinary, verifier.cliEnv, paths); + await pushLocalChanges(context.cliBinary, verifier.cliEnv); + + const returnObservation = await runCouchDbReplicationObserved(context.cliBinary, session.cliEnv); + assert(returnObservation.succeeded, "The upgraded CouchDB device could not receive the verifier delta."); + assert(returnObservation.arrivedDocuments > 0, "The verifier CouchDB delta did not arrive."); + await verifyReturnDelta(upgradeVault, paths); + await stopSession(context, verifier); + await stopSession(context, session); + upgradedSession = undefined; + + const restarted = await startSession(context, upgradeVault, ports[0], context.targetArtifactRoot); + const restartedState = await readRuntimeUpgradeState(context.cliBinary, restarted.cliEnv); + assertUpgradeRemainsReady(restartedState, context.targetVersion); + assertRestartContinuity(upgradedState, restartedState); + await verifyReturnDelta(upgradeVault, paths); + await stopSession(context, restarted); + + console.log( + `PASS CouchDB: ${STABLE_RELEASE_VERSION} -> ${context.targetVersion}; checkpoint lineage, no-op sync, delta sync, fresh-device round-trip, and restart continuity verified.` + ); + } finally { + if (upgradedSession) await stopSession(context, upgradedSession).catch(() => undefined); + await stopSessions(context); + await Promise.all([upgradeVault.dispose(), verifierVault.dispose()]); + if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { + await deleteCouchDbDatabase(config, databaseName).catch((error: unknown) => { + console.warn(error instanceof Error ? error.message : error); + }); + } + } +} + +async function runObjectStorageUpgrade(context: RunnerContext, ports: readonly [number, number]): Promise { + console.log(`\n# Upgrade from ${STABLE_RELEASE_VERSION}: Object Storage`); + const config = await loadObjectStorageConfig(); + const prefix = makeUniqueBucketPrefix("upgrade-from-stable"); + const remote: UpgradeTransportConfiguration = { kind: "object-storage", config, bucketPrefix: prefix }; + const paths = createUpgradeScenarioPaths("object-storage"); + const upgradeVault = await createTemporaryVault("obsidian-livesync-upgrade-object-storage-"); + const verifierVault = await createTemporaryVault("obsidian-livesync-upgrade-object-storage-verifier-"); + let upgradedSession: ObsidianLiveSyncSession | undefined; + + try { + await ensureObjectStorageBucket(config); + + let session = await startSession(context, upgradeVault, ports[0], context.sourceArtifactRoot); + assertStableReleaseDefaults(await readRuntimeUpgradeState(context.cliBinary, session.cliEnv), false); + await configureStableRelease(context.cliBinary, session.cliEnv, remote); + const configuredStable = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + assertStableReleaseDefaults(configuredStable, true); + assertStableRemoteSelection(configuredStable, remote); + await stopSession(context, session); + + session = await startSession(context, upgradeVault, ports[0], context.sourceArtifactRoot); + const restartedStable = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + assertStableReleaseDefaults(restartedStable, true); + assertStableRemoteSelection(restartedStable, remote); + await waitForPersistentNodeIdentity(context.cliBinary, session.cliEnv); + await prepareStableRemote(context.cliBinary, session.cliEnv); + await runStableFileHistory(context.cliBinary, session.cliEnv, paths, async () => { + const result = await runJournalReplicationObserved(context.cliBinary, session.cliEnv); + assert(result.succeeded, "The stable Object Storage synchronisation failed."); + }); + await verifyPreUpgradeHistory(upgradeVault, paths); + + const stableState = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + const stableCheckpoint = await readJournalCheckpoint(context.cliBinary, session.cliEnv); + const stableRemote = await readObjectStorageRemoteSnapshot(config, prefix); + await stopSession(context, session); + + session = await startSession(context, upgradeVault, ports[0], context.targetArtifactRoot); + upgradedSession = session; + await dismissConfigDoctorIfShown(session.remoteDebuggingPort); + const upgradedState = await readRuntimeUpgradeState(context.cliBinary, session.cliEnv); + assertUpgradeCompatibilityReady(stableState, upgradedState, context.targetVersion, remote); + await verifyPreUpgradeHistory(upgradeVault, paths); + + const loadedCheckpoint = await readJournalCheckpoint(context.cliBinary, session.cliEnv); + assertJournalCheckpointLoaded(stableCheckpoint, loadedCheckpoint); + const noOpObservation = await runJournalReplicationObserved(context.cliBinary, session.cliEnv); + assert(noOpObservation.succeeded, "The first post-upgrade Object Storage synchronisation failed."); + const noOpCheckpoint = await readJournalCheckpoint(context.cliBinary, session.cliEnv); + const noOpRemote = await readObjectStorageRemoteSnapshot(config, prefix); + assertNoJournalReplay( + stableCheckpoint, + noOpCheckpoint, + stableRemote.journalObjects, + noOpRemote.journalObjects, + noOpObservation + ); + assertMilestoneContinuity(stableRemote.milestone, noOpRemote.milestone); + + await createPostUpgradeDelta(context.cliBinary, session.cliEnv, paths); + const deltaObservation = await runJournalReplicationObserved(context.cliBinary, session.cliEnv); + assert(deltaObservation.succeeded, "The post-upgrade Object Storage delta failed."); + const deltaCheckpoint = await readJournalCheckpoint(context.cliBinary, session.cliEnv); + assertJournalCheckpointAdvanced(noOpCheckpoint, deltaCheckpoint, deltaObservation); + const deltaRemote = await readObjectStorageRemoteSnapshot(config, prefix); + assertMilestoneContinuity(noOpRemote.milestone, deltaRemote.milestone); + + const verifierSettings = { ...config, bucketPrefix: prefix }; + const verifier = await startSession(context, verifierVault, ports[1], context.targetArtifactRoot, { + pluginData: createE2eObjectStoragePluginData(verifierSettings, deltaRemote.preferredTweaks), + localStorageEntries: createE2eObsidianDeviceLocalState(verifierVault.name), + }); + await configureFreshObjectStorageVerifier(context, verifier, config, prefix, deltaRemote.preferredTweaks); + await pushLocalChanges(context.cliBinary, verifier.cliEnv); + await verifyPostUpgradeHistory(verifierVault, paths); + await createVerifierReturnDelta(context.cliBinary, verifier.cliEnv, paths); + await pushLocalChanges(context.cliBinary, verifier.cliEnv); + + const returnObservation = await runJournalReplicationObserved(context.cliBinary, session.cliEnv); + assert(returnObservation.succeeded, "The upgraded Object Storage device could not receive the verifier delta."); + assert(returnObservation.downloadedJournalKeys.length > 0, "The verifier Object Storage delta did not arrive."); + await verifyReturnDelta(upgradeVault, paths); + await stopSession(context, verifier); + await stopSession(context, session); + upgradedSession = undefined; + + const restarted = await startSession(context, upgradeVault, ports[0], context.targetArtifactRoot); + const restartedState = await readRuntimeUpgradeState(context.cliBinary, restarted.cliEnv); + assertUpgradeRemainsReady(restartedState, context.targetVersion); + assertRestartContinuity(upgradedState, restartedState); + await verifyReturnDelta(upgradeVault, paths); + await stopSession(context, restarted); + + console.log( + `PASS Object Storage: ${STABLE_RELEASE_VERSION} -> ${context.targetVersion}; checkpoint lineage, no replay, delta sync, fresh-device round-trip, and restart continuity verified.` + ); + } finally { + if (upgradedSession) await stopSession(context, upgradedSession).catch(() => undefined); + await stopSessions(context); + await Promise.all([upgradeVault.dispose(), verifierVault.dispose()]); + if (process.env.E2E_OBSIDIAN_KEEP_OBJECT_STORAGE !== "true") { + await deleteObjectStoragePrefix(config, prefix).catch((error: unknown) => { + console.warn(error instanceof Error ? error.message : error); + }); + } + } +} + +async function startManagedServices(transports: readonly Transport[]): Promise { + if (transports.includes("couchdb")) { + await runNpmScript("test:docker-couchdb:stop", true); + await runNpmScript("test:docker-couchdb:start"); + } + if (transports.includes("object-storage")) { + await runNpmScript("test:docker-s3:stop", true); + await runNpmScript("test:docker-s3:start"); + } +} + +async function stopManagedServices(transports: readonly Transport[]): Promise { + if (transports.includes("object-storage")) await runNpmScript("test:docker-s3:stop", true); + if (transports.includes("couchdb")) await runNpmScript("test:docker-couchdb:stop", true); +} + +async function main(): Promise { + const arguments_ = parseArguments(process.argv.slice(2)); + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + + const targetArtifactRoot = resolve(process.env.E2E_LIVESYNC_TARGET_ARTIFACT_ROOT?.trim() || process.cwd()); + const targetVersion = await validateTargetArtifact(targetArtifactRoot); + const sourceArtifactRoot = await ensurePinnedReleaseArtifact(); + const context: RunnerContext = { + binary, + cliBinary: cli.binary, + sourceArtifactRoot, + targetArtifactRoot, + targetVersion, + activeSessions: new Set(), + }; + const ports = sessionPorts(); + let managedServicesStarted = false; + + console.log(`Using exact source release: ${STABLE_RELEASE_VERSION}`); + console.log(`Using target release candidate: ${targetVersion}`); + console.log(`Source artefact cache: ${sourceArtifactRoot}`); + console.log(`Target artefact root: ${targetArtifactRoot}`); + + try { + await runUnconfiguredSettingsUpgrade(context, ports[0]); + if (arguments_.manageServices) { + await startManagedServices(arguments_.transports); + managedServicesStarted = true; + } + for (const transport of arguments_.transports) { + if (transport === "couchdb") await runCouchDbUpgrade(context, ports); + else await runObjectStorageUpgrade(context, ports); + } + } finally { + await stopSessions(context); + if (managedServicesStarted && !arguments_.keepServices) { + await stopManagedServices(arguments_.transports); + } + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +});