import { execFile } from "node:child_process"; import { randomBytes } from "node:crypto"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; import { Worker } from "node:worker_threads"; import { encodeSettingsToQRCodeData } from "@vrtmrz/livesync-commonlib/compat/API/processSetting"; import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/setup-uri"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { evalObsidianJson } from "../runner/cli.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { assertEqual, assertE2eCompatibilityUnpaused, pushLocalChanges, type ConfiguredSettings, waitForLiveSyncCoreReady, waitForLocalDatabaseEntry, } from "../runner/liveSyncWorkflow.ts"; import { deleteObjectStoragePrefix, ensureObjectStorageBucket, listObjectStorageObjects, loadObjectStorageConfig, makeUniqueBucketPrefix, type ObjectStorageConfig, } from "../runner/objectStorage.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { acknowledgeDisabledOptionalFeatures, captureAndStartInitialisation, confirmFastFetch, confirmRebuild, enterSetupURI, finishInitialisation, generateSetupURIFromDevice, continueWithoutRemoteSettings, captureGuideDialogue, modalByTitle, selectRadioOption, type SetupArtifact, type SetupCaptureNames, } from "../runner/setupUri.ts"; import { captureObsidianElement, captureObsidianPage, obsidianRemoteDebuggingPort, withObsidianPage, } from "../runner/ui.ts"; import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "90000"; const execFileAsync = promisify(execFile); const useCustomRequestHandler = process.argv.includes("--custom-http-handler"); const useQRCode = process.argv.includes("--qr"); const uriMode = process.argv.includes("--compatible") ? "persistent" : "ephemeral"; const captures: SetupCaptureNames = useQRCode ? { scenario: "object-storage-qr", guide: "object-storage-qr-setup" } : uriMode === "persistent" ? { scenario: "object-storage-compatible-uri", guide: "object-storage-compatible-setup" } : useCustomRequestHandler ? { scenario: "object-storage-custom-http-handler-setup-uri", guide: "object-storage-custom-http-handler-setup", } : { scenario: "object-storage-setup-uri", guide: "object-storage-setup" }; const noteFromFirst = "E2E/object-storage/from-first.md"; const noteFromSecond = "E2E/object-storage/from-second.md"; const firstContent = "# Object Storage from the first device\n\nThis note travelled through the first device's Setup URI.\n"; const secondContent = "# Object Storage from the second device\n\nThis note completed the return journey.\n"; type RunnerContext = { binary: string; cliBinary: string; activeSessions: Set; }; type StartupSchedulingState = ConfiguredSettings & { periodicReplication: boolean; }; function sessionEnvironment(port: number): NodeJS.ProcessEnv { return { ...process.env, E2E_OBSIDIAN_REMOTE_DEBUGGING_PORT: String(port) }; } function sessionPorts(): readonly [number, number] { const first = obsidianRemoteDebuggingPort(process.env); const second = Number(process.env.E2E_OBSIDIAN_SECONDARY_REMOTE_DEBUGGING_PORT ?? first + 1); if (!Number.isInteger(second) || second < 1 || second > 65535 || second === first) { throw new Error(`Invalid secondary Obsidian remote debugging port: ${second}`); } return [first, second]; } async function runDeno(script: string, environment: NodeJS.ProcessEnv): Promise { const { stdout } = await execFileAsync( "deno", [ "run", "--minimum-dependency-age=0", "--config=utils/flyio/deno.jsonc", "--frozen", "--lock=utils/flyio/deno.lock", "--allow-env", script, ], { cwd: process.cwd(), env: environment, maxBuffer: 4 * 1024 * 1024 } ); return stdout; } async function generateBootstrapSetupURI( objectStorage: ObjectStorageConfig, bucketPrefix: string, useCustomRequestHandler: boolean ): Promise { const setupPassphrase = randomBytes(24).toString("base64url"); const output = await runDeno("utils/setup/generate_setup_uri.ts", { ...process.env, remote_type: "s3", endpoint: objectStorage.endpoint, access_key: objectStorage.accessKey, secret_key: objectStorage.secretKey, bucket: objectStorage.bucket, region: objectStorage.region, force_path_style: String(objectStorage.forcePathStyle), bucket_prefix: bucketPrefix, ...(useCustomRequestHandler ? { use_custom_request_handler: "true" } : {}), passphrase: randomBytes(24).toString("base64url"), uri_passphrase: setupPassphrase, uri_mode: uriMode, }); const setupURI = output.split(/\r?\n/u).find((line) => line.startsWith("obsidian://setuplivesync?settings=")); if (!setupURI) throw new Error("The public Setup URI generator did not emit an Object Storage Setup URI."); return { setupURI, setupPassphrase }; } async function expiredSetupURI(settings: ObsidianLiveSyncSettings, setupPassphrase: string): Promise { // Only this fixture worker uses a past clock; Obsidian and the runner keep real time. const worker = new Worker( ` const { parentPort, workerData } = require('node:worker_threads'); Date.now = () => Date.UTC(2024, 0, 1); import('@vrtmrz/livesync-commonlib/setup-uri').then(async ({ encodeTimeBoundSetupURI }) => { const result = await encodeTimeBoundSetupURI(workerData.settings, workerData.setupPassphrase); parentPort.postMessage(result.uri); }); `, { eval: true, workerData: { settings, setupPassphrase } } ); try { const setupURI = await new Promise((resolve, reject) => { worker.once("message", resolve); worker.once("error", reject); worker.once("exit", (code) => reject(new Error(`The expired URI fixture exited with ${code}.`))); }); return { setupURI, setupPassphrase }; } finally { await worker.terminate(); } } async function assertIndependentIdKey( context: RunnerContext, session: ObsidianLiveSyncSession, vault: TemporaryVault, expectedKey: string ): Promise { const matches = await evalObsidianJson( context.cliBinary, `(() => { const settings = app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings(); return JSON.stringify(settings.encrypt && settings.usePathObfuscation && settings.idDerivationVersion === 1 && settings.idDerivationKey === ${JSON.stringify(expectedKey)}); })()`, session.cliEnv ); assertEqual(matches, true, "The device did not retain the shared independent ID key and Path Obfuscation."); const raw = await readFile(join(vault.path, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8"); const saved = JSON.parse(raw) as Record; assertEqual(saved.idDerivationVersion, 1, "The independent ID version was not saved."); assertEqual(saved.idDerivationKey, "", "The ID key was saved in plain text."); if (!saved.encryptedIdDerivationKey || raw.includes(expectedKey)) { throw new Error("The shared ID key was not encrypted in local settings."); } } async function startSession( context: RunnerContext, vault: TemporaryVault, port: number ): Promise { const session = await startObsidianLiveSyncSession({ binary: context.binary, cliBinary: context.cliBinary, vault, startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), env: sessionEnvironment(port), }); context.activeSessions.add(session); return session; } async function stopSessions(context: RunnerContext): Promise { for (const session of [...context.activeSessions]) { await stopSession(context, session); } } async function stopSession(context: RunnerContext, session: ObsidianLiveSyncSession): Promise { if (!context.activeSessions.has(session)) return; await session.app.stop(); context.activeSessions.delete(session); } async function writeNote( cliBinary: string, environment: NodeJS.ProcessEnv, path: string, content: string ): Promise { await evalObsidianJson( cliBinary, [ "(async()=>{", `const path=${JSON.stringify(path)};`, `const content=${JSON.stringify(content)};`, "const folder=path.split('/').slice(0,-1).join('/');", "if(folder&&!(await app.vault.adapter.exists(folder))) await app.vault.createFolder(folder);", "const existing=app.vault.getAbstractFileByPath(path);", "if(existing) await app.vault.modify(existing,content);", "else await app.vault.create(path,content);", "return JSON.stringify({ok:true});", "})()", ].join(""), environment ); await waitForLocalDatabaseEntry(cliBinary, environment, path); } async function waitForPathContent(vault: TemporaryVault, path: string, expected: string): Promise { const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ?? 30000); let lastContent = ""; while (Date.now() < deadline) { try { lastContent = await readFile(join(vault.path, path), "utf8"); if (lastContent === expected) return; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } await new Promise((resolve) => setTimeout(resolve, 250)); } throw new Error(`Timed out waiting for ${path}. Last content:\n${lastContent}`); } async function waitForObjectStorageData(config: ObjectStorageConfig, prefix: string): Promise { const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_OBJECT_STORAGE_TIMEOUT_MS ?? 30000); while (Date.now() < deadline) { if ((await listObjectStorageObjects(config, prefix)).length > 0) return; await new Promise((resolve) => setTimeout(resolve, 500)); } throw new Error(`Timed out waiting for Object Storage data under ${prefix}.`); } async function configureMigratedStartupScheduling( cliBinary: string, environment: NodeJS.ProcessEnv ): Promise { return await evalObsidianJson( cliBinary, [ "(async()=>{", "const plugin=app.plugins.plugins['obsidian-livesync'];", "const core=plugin.core;", // Persist only the migration-shaped scheduling flags and leave the // generated URI unchanged. Device A stops before B creates the // return note, so save-triggered reconciliation cannot satisfy the // later start-up assertion. "await core.services.setting.applyExternalSettings({liveSync:true,syncOnStart:true,periodicReplication:false},true);", "const current=core.services.setting.currentSettings();", "return JSON.stringify({", "isConfigured:current.isConfigured,", "liveSync:current.liveSync,", "syncOnStart:current.syncOnStart,", "syncOnSave:current.syncOnSave,", "periodicReplication:current.periodicReplication,", "remoteType:current.remoteType,", "couchDB_URI:current.couchDB_URI,", "couchDB_DBNAME:current.couchDB_DBNAME,", "endpoint:current.endpoint,", "bucket:current.bucket,", "bucketPrefix:current.bucketPrefix,", "});", "})()", ].join(""), environment ); } async function captureNote(port: number, path: string, text: string, filename: string): Promise { await withObsidianPage(port, async (page) => { await page.evaluate((notePath) => { const obsidian = globalThis as typeof globalThis & { app?: { workspace?: { openLinkText(path: string, sourcePath: string, newLeaf: boolean): Promise }; }; }; return obsidian.app?.workspace?.openLinkText(notePath, "", false); }, path); }); await captureObsidianPage(port, `${filename}.full.png`, async (page) => { await page.getByText(text, { exact: false }).first().waitFor({ state: "visible", timeout: 30000 }); }); return await captureObsidianElement(port, filename, (page) => page.locator(".workspace-leaf.mod-active").first()); } async function importQRCode(port: number, qrData: string): Promise { await withObsidianPage(port, async (page) => { await page.evaluate((data) => { const obsidian = globalThis as typeof globalThis & { app: { plugins: { plugins: Record< string, { core: { modules: { decodeQR?: (qr: string) => Promise }[] } } >; }; }; }; const setup = obsidian.app.plugins.plugins["obsidian-livesync"].core.modules.find( (module) => typeof module.decodeQR === "function" ); if (!setup?.decodeQR) throw new Error("The QR settings decoder is unavailable."); // The decoder remains pending while the real setup dialogues run. void setup.decodeQR(data); }, qrData); }); const title = "Mostly Complete: Decision Required"; const screenshot = await captureGuideDialogue(port, `guide-${captures.guide}-join-choice.png`, title); await withObsidianPage(port, async (page) => { const modal = modalByTitle(page, title); await selectRadioOption(modal, "🔗 Join this device"); await modal.getByRole("button", { name: "Proceed to the next step.", exact: true }).click(); }); return screenshot; } async function main(): Promise { const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); const objectStorage = await loadObjectStorageConfig(); const bucketPrefix = makeUniqueBucketPrefix("setup-uri-workflow"); const bootstrapArtifact = await generateBootstrapSetupURI(objectStorage, bucketPrefix, useCustomRequestHandler); const bootstrapSettings = await decodeSettingsFromSetupURI( bootstrapArtifact.setupURI, bootstrapArtifact.setupPassphrase ); if (!bootstrapSettings || bootstrapSettings.idDerivationVersion !== 1 || !bootstrapSettings.idDerivationKey) { throw new Error("The public Setup URI generator did not configure an independent ID key."); } const idKey = bootstrapSettings.idDerivationKey; const rejectedArtifacts = [ { ...bootstrapArtifact, setupPassphrase: "incorrect-setup-passphrase" }, await expiredSetupURI(bootstrapSettings as ObsidianLiveSyncSettings, bootstrapArtifact.setupPassphrase), ]; const vaultA = await createTemporaryVault(); const vaultB = await createTemporaryVault(); const [portA, portB] = sessionPorts(); const context: RunnerContext = { binary, cliBinary: cli.binary, activeSessions: new Set() }; const screenshots: string[] = []; try { await ensureObjectStorageBucket(objectStorage); console.log(`Temporary Object Storage target: ${objectStorage.bucket}/${bucketPrefix}`); const sessionA = await startSession(context, vaultA, portA); screenshots.push(await enterSetupURI(portA, "new", bootstrapArtifact, captures, rejectedArtifacts)); screenshots.push(await captureAndStartInitialisation(portA, "new", captures)); screenshots.push(await confirmRebuild(portA, captures)); screenshots.push(await continueWithoutRemoteSettings(portA, captures)); screenshots.push(await acknowledgeDisabledOptionalFeatures(portA, captures)); const firstState = await finishInitialisation(portA, context.cliBinary, sessionA.cliEnv); await assertE2eCompatibilityUnpaused(context.cliBinary, sessionA.cliEnv, portA); await assertIndependentIdKey(context, sessionA, vaultA, idKey); assertEqual( firstState.endpoint, objectStorage.endpoint, "The first device did not activate the Object Storage endpoint." ); assertEqual( firstState.bucket, objectStorage.bucket, "The first device did not activate the Object Storage bucket." ); assertEqual( firstState.bucketPrefix, bucketPrefix, "The first device did not activate the unique bucket prefix." ); assertEqual( firstState.useCustomRequestHandler, useCustomRequestHandler, "The first device did not preserve the expected Custom HTTP Handler setting." ); await writeNote(context.cliBinary, sessionA.cliEnv, noteFromFirst, firstContent); const firstEntry = await waitForLocalDatabaseEntry(context.cliBinary, sessionA.cliEnv, noteFromFirst); if ( !/^f:[0-9a-f]{64}$/u.test(firstEntry.id) || firstEntry.children.length === 0 || firstEntry.children.some((id) => !/^h:\+[0-9a-f]{64}$/u.test(id)) ) { throw new Error("The source note did not use independent document and Chunk IDs."); } await pushLocalChanges(context.cliBinary, sessionA.cliEnv); await waitForObjectStorageData(objectStorage, bucketPrefix); const generated = await generateSetupURIFromDevice( portA, randomBytes(24).toString("base64url"), captures, uriMode ); if (generated.artifact.setupURI === bootstrapArtifact.setupURI) { throw new Error("The first device returned the bootstrap Setup URI instead of generating a new one."); } screenshots.push(...generated.screenshots); const qrSettings = useQRCode ? await evalObsidianJson( context.cliBinary, "JSON.stringify(app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings())", sessionA.cliEnv ) : undefined; if (qrSettings) { // Represent QR settings from a device with a different database // suffix; isolated Obsidian profiles can share the default suffix. qrSettings.additionalSuffixOfDatabaseName = "qr-source"; } const startupState = await configureMigratedStartupScheduling(context.cliBinary, sessionA.cliEnv); assertEqual(startupState.liveSync, true, "The first device did not persist its Continuous setting."); assertEqual(startupState.syncOnStart, true, "The first device did not persist syncOnStart."); assertEqual( startupState.periodicReplication, false, "Periodic replication could mask the syncOnStart return journey." ); assertEqual( startupState.endpoint, objectStorage.endpoint, "Enabling syncOnStart changed the Object Storage endpoint." ); assertEqual( startupState.bucketPrefix, bucketPrefix, "Enabling syncOnStart changed the Object Storage bucket prefix." ); await stopSession(context, sessionA); const sessionB = await startSession(context, vaultB, portB); const initialMarker = await assertE2eCompatibilityUnpaused(context.cliBinary, sessionB.cliEnv, portB); if (qrSettings) { assertEqual( initialMarker.additionalSuffix === `-${qrSettings.additionalSuffixOfDatabaseName}`, false, "The QR fixture must start with distinct source and receiver database suffixes." ); } screenshots.push( qrSettings ? await importQRCode(portB, encodeSettingsToQRCodeData(qrSettings)) : await enterSetupURI(portB, "existing", generated.artifact, captures) ); screenshots.push(await captureAndStartInitialisation(portB, "existing", captures)); if (qrSettings) { // Inspect the imported namespace before Fetch resets the local // database and restores this device's own suffix. await withObsidianPage(portB, async (page) => { await modalByTitle(page, "Data retrieval scheduled").waitFor({ state: "visible", timeout: 30000 }); }); const importedMarker = await assertE2eCompatibilityUnpaused(context.cliBinary, sessionB.cliEnv, portB); assertEqual( importedMarker.expectedStorageKey === initialMarker.expectedStorageKey, false, "The QR fixture did not exercise a changed device-local namespace after settings import." ); assertEqual( importedMarker.additionalSuffix, `-${qrSettings.additionalSuffixOfDatabaseName}`, "The second device did not import the QR source's database suffix." ); } screenshots.push(...(await confirmFastFetch(portB, captures))); const secondState = await finishInitialisation(portB, context.cliBinary, sessionB.cliEnv); const fetchedMarker = await assertE2eCompatibilityUnpaused(context.cliBinary, sessionB.cliEnv, portB); await assertIndependentIdKey(context, sessionB, vaultB, idKey); assertEqual( secondState.endpoint, objectStorage.endpoint, "The second device did not import the Object Storage endpoint." ); assertEqual( secondState.bucketPrefix, bucketPrefix, "The second device did not import the unique bucket prefix." ); assertEqual( secondState.useCustomRequestHandler, useCustomRequestHandler, "The second device did not import the expected Custom HTTP Handler setting." ); await pushLocalChanges(context.cliBinary, sessionB.cliEnv); await waitForPathContent(vaultB, noteFromFirst, firstContent); const importedEntry = await waitForLocalDatabaseEntry(context.cliBinary, sessionB.cliEnv, noteFromFirst); assertEqual(importedEntry.id, firstEntry.id, "Import changed the obfuscated document ID."); assertEqual( JSON.stringify(importedEntry.children), JSON.stringify(firstEntry.children), "Import changed the Chunk IDs." ); screenshots.push( await captureNote( portB, noteFromFirst, "Object Storage from the first device", `guide-${captures.guide}-first-to-second.png` ) ); await writeNote(context.cliBinary, sessionB.cliEnv, noteFromSecond, secondContent); const secondEntry = await waitForLocalDatabaseEntry(context.cliBinary, sessionB.cliEnv, noteFromSecond); await pushLocalChanges(context.cliBinary, sessionB.cliEnv); await stopSession(context, sessionB); const returningSessionB = await startSession(context, vaultB, portB); await waitForLiveSyncCoreReady(context.cliBinary, returningSessionB.cliEnv); await assertIndependentIdKey(context, returningSessionB, vaultB, idKey); const restartedMarker = await assertE2eCompatibilityUnpaused( context.cliBinary, returningSessionB.cliEnv, portB ); assertEqual( restartedMarker.expectedStorageKey, fetchedMarker.expectedStorageKey, "Restart changed the device-local namespace selected by Fetch." ); await waitForPathContent(vaultB, noteFromFirst, firstContent); await stopSession(context, returningSessionB); const returningSessionA = await startSession(context, vaultA, portA); await waitForLiveSyncCoreReady(context.cliBinary, returningSessionA.cliEnv); await assertE2eCompatibilityUnpaused(context.cliBinary, returningSessionA.cliEnv, portA); await assertIndependentIdKey(context, returningSessionA, vaultA, idKey); // Deliberately omit manual replication here. Object Storage reports // Continuous as not applicable, so startup scheduling must honour the // retained syncOnStart setting by running an unattended OneShot. await waitForPathContent(vaultA, noteFromSecond, secondContent); const returnedEntry = await waitForLocalDatabaseEntry( context.cliBinary, returningSessionA.cliEnv, noteFromSecond ); assertEqual(returnedEntry.id, secondEntry.id, "The return journey changed the obfuscated document ID."); assertEqual( JSON.stringify(returnedEntry.children), JSON.stringify(secondEntry.children), "The return journey changed the Chunk IDs." ); screenshots.push( await captureNote( portA, noteFromSecond, "Object Storage from the second device", `guide-${captures.guide}-second-to-first.png` ) ); console.log( `Object Storage ${useQRCode ? "QR" : "Setup URI"} and two-device roundtrip succeeded with the ${ useCustomRequestHandler ? "Custom HTTP Handler" : "default HTTP handler" }. Screenshots: ${screenshots.join(", ")}` ); } catch (error) { for (const session of context.activeSessions) { await captureObsidianPage(session.remoteDebuggingPort, `${captures.scenario}-failure.png`, async (page) => { console.error( "Visible dialogue titles:", await page.locator(".modal-container:visible .modal-title").allTextContents() ); console.error( "Initialisation state:", await evalObsidianJson( context.cliBinary, `(()=>{ const core=app.plugins.plugins['obsidian-livesync'].core; const settings=core.services.setting.currentSettings(); return JSON.stringify({configured:settings.isConfigured, databaseReady:core.services.database.isDatabaseReady(),appReady:core.services.appLifecycle.isReady(), suspended:core.services.appLifecycle.isSuspended(),versionUpFlash:settings.versionUpFlash, activeConfigurationId:settings.activeConfigurationId, remoteConfigurationCount:Object.keys(settings.remoteConfigurations||{}).length});})()`, session.cliEnv ) ); }).catch((diagnosticError: unknown) => console.error("Could not capture initialisation failure:", diagnosticError) ); } throw error; } finally { await stopSessions(context).catch((error: unknown) => { console.warn(error instanceof Error ? error.message : error); }); await vaultA.dispose(); await vaultB.dispose(); if (process.env.E2E_OBSIDIAN_KEEP_OBJECT_STORAGE !== "true") { await deleteObjectStoragePrefix(objectStorage, bucketPrefix).catch((error: unknown) => { console.warn(error instanceof Error ? error.message : error); }); } } } main().catch((error: unknown) => { console.error(error instanceof Error ? error.stack : error); process.exit(1); });