obsidian-livesync/test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts

import { randomBytes } from "node:crypto";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { DEVICE_ID_PREFERRED, MILESTONE_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { evalObsidianJson } from "../runner/cli.ts";
import {
    assertCouchDbReachable,
    deleteCouchDbDatabase,
    fetchCouchDbDocument,
    loadCouchDbConfig,
    makeUniqueDatabaseName,
    putCouchDbDocument,
    waitForCouchDbDocs,
    type CouchDbConfig,
} from "../runner/couchdb.ts";
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
import { assertEqual, pushLocalChanges, waitForLocalDatabaseEntry } from "../runner/liveSyncWorkflow.ts";
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
import {
    acknowledgeDisabledOptionalFeatures,
    captureAndStartInitialisation,
    captureGuideDialogue,
    confirmFastFetch,
    confirmRebuild,
    enterSetupURI,
    finishInitialisation,
    generateSetupURIFromDevice,
    modalByTitle,
    resumeCompatibilityReviewIfShown,
    selectRadioOption,
    continueWithoutRemoteSettings,
    type SetupArtifact,
} from "../runner/setupUri.ts";
import { captureObsidianPage, openLiveSyncSettings, withObsidianPage } from "../runner/ui.ts";
import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts";

process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "90000";
process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "30000";

const uiTimeoutMs = Number(process.env.E2E_OBSIDIAN_SETUP_URI_TIMEOUT_MS ?? 30000);
const notePath = "E2E/manual-couchdb/from-first-device.md";
const noteContent = "# Manual CouchDB setup\n\nThis note was sent by the manually configured first device.\n";
const returnNotePath = "E2E/manual-couchdb/from-second-device.md";
const returnNoteContent =
    "# Manual CouchDB return journey\n\nThis note returned through a Setup URI generated by the first device.\n";
const captures = {
    scenario: "couchdb-manual-setup-workflow",
    guide: "couchdb-manual",
} as const;
const e2eeRebuildCaptures = {
    scenario: "couchdb-manual-setup-workflow",
    guide: "couchdb-manual-e2ee-rebuild",
} as const;

type RunnerContext = {
    binary: string;
    cliBinary: string;
    couchDb: CouchDbConfig;
    dbName: string;
    activeSessions: Set<ObsidianLiveSyncSession>;
};

async function startUnconfiguredSession(
    context: RunnerContext,
    vault: TemporaryVault
): Promise<ObsidianLiveSyncSession> {
    const session = await startObsidianLiveSyncSession({
        binary: context.binary,
        cliBinary: context.cliBinary,
        vault,
        startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
    });
    context.activeSessions.add(session);
    return session;
}

async function stopTrackedSession(context: RunnerContext, session: ObsidianLiveSyncSession): Promise<void> {
    if (!context.activeSessions.has(session)) return;
    await session.app.stop();
    context.activeSessions.delete(session);
}

async function stopTrackedSessions(context: RunnerContext): Promise<void> {
    for (const session of [...context.activeSessions]) {
        await stopTrackedSession(context, session);
    }
}

async function captureFailure(session: ObsidianLiveSyncSession, label: string): Promise<void> {
    const screenshot = await captureObsidianPage(
        session.remoteDebuggingPort,
        `couchdb-manual-${label}-failure.png`,
        async () => undefined
    ).catch(() => undefined);
    if (screenshot) {
        console.error(`Manual CouchDB failure screenshot: ${screenshot}`);
    }
}

async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig, dbName: string): Promise<string[]> {
    const screenshots: string[] = [];
    await withObsidianPage(port, async (page) => {
        const invitation = page.locator(".notice").filter({ hasText: "Welcome to Self-hosted LiveSync" });
        await invitation.waitFor({ state: "visible", timeout: uiTimeoutMs });
        await invitation.locator(".sls-onboarding-invitation-action").click({ timeout: uiTimeoutMs });

        const intro = modalByTitle(page, "Welcome to Self-hosted LiveSync");
        await intro.waitFor({ state: "visible", timeout: uiTimeoutMs });
        await selectRadioOption(intro, "I am setting this up for the first time");
        await intro
            .getByRole("button", { name: "Yes, I want to set up a new synchronisation" })
            .click({ timeout: uiTimeoutMs });
    });

    screenshots.push(
        await captureGuideDialogue(port, "guide-couchdb-manual-connection-method.png", "Connection Method")
    );
    await withObsidianPage(port, async (page) => {
        const method = modalByTitle(page, "Connection Method");
        await selectRadioOption(method, "Configure a remote manually");
        await method.getByRole("button", { name: "Proceed with manual configuration" }).click({ timeout: uiTimeoutMs });

        const encryption = modalByTitle(page, "End-to-End Encryption");
        await encryption.waitFor({ state: "visible", timeout: uiTimeoutMs });
        await encryption
            .locator("label.row")
            .filter({ hasText: "End-to-End Encryption" })
            .locator('input[type="checkbox"]')
            .first()
            .check({ timeout: uiTimeoutMs });
        await encryption
            .locator("label.row")
            .filter({ hasText: "Obfuscate Properties" })
            .locator('input[type="checkbox"]')
            .first()
            .check({ timeout: uiTimeoutMs });
        await encryption.locator('input[name="e2ee-passphrase"]').fill(randomBytes(24).toString("base64url"));
    });
    screenshots.push(await captureGuideDialogue(port, "guide-couchdb-manual-encryption.png", "End-to-End Encryption"));
    await withObsidianPage(port, async (page) => {
        const encryption = modalByTitle(page, "End-to-End Encryption");
        await encryption.getByRole("button", { name: "Proceed", exact: true }).click({ timeout: uiTimeoutMs });
    });

    screenshots.push(
        await captureGuideDialogue(port, "guide-couchdb-manual-remote-selection.png", "Choose a synchronisation remote")
    );
    await withObsidianPage(port, async (page) => {
        const remoteSelection = modalByTitle(page, "Choose a synchronisation remote");
        await selectRadioOption(remoteSelection, "CouchDB");
        await remoteSelection
            .getByRole("button", { name: "Continue to CouchDB setup", exact: true })
            .click({ timeout: uiTimeoutMs });

        const couchDB = modalByTitle(page, "CouchDB Configuration");
        await couchDB.waitFor({ state: "visible", timeout: uiTimeoutMs });
        await couchDB.locator('input[name="couchdb-url"]').fill(couchDb.uri);
        await couchDB.locator('input[name="couchdb-username"]').fill(couchDb.username);
        await couchDB.locator('input[name="couchdb-password"]').fill(couchDb.password);
        await couchDB.locator('input[name="couchdb-database"]').fill(dbName);
    });
    screenshots.push(
        await captureGuideDialogue(port, "guide-couchdb-manual-connection-details.png", "CouchDB Configuration")
    );

    await withObsidianPage(port, async (page) => {
        const couchDB = modalByTitle(page, "CouchDB Configuration");
        await couchDB
            .getByRole("button", { name: "Check server requirements", exact: true })
            .click({ timeout: uiTimeoutMs });
        const summary = couchDB.locator(".check-results summary");
        await summary.waitFor({ state: "visible", timeout: uiTimeoutMs });
        await summary
            .filter({ hasText: /All checks passed successfully!|issue\(s\) detected!/u })
            .waitFor({ state: "visible", timeout: uiTimeoutMs });
        const errors = couchDB.locator(".check-result.error");
        if ((await errors.count()) > 0) {
            const messages = await errors.locator(".message").allTextContents();
            throw new Error(`The documented CouchDB fixture failed its server requirements: ${messages.join(" | ")}`);
        }
        const details = couchDB.locator(".check-results details");
        if (!(await details.evaluate((element) => (element as HTMLDetailsElement).open))) {
            await summary.click({ timeout: uiTimeoutMs });
        }
    });
    screenshots.push(
        await captureGuideDialogue(port, "guide-couchdb-manual-server-requirements.png", "CouchDB Configuration")
    );

    await withObsidianPage(port, async (page) => {
        const couchDB = modalByTitle(page, "CouchDB Configuration");
        await couchDB
            .getByRole("button", { name: "Create or connect to database and continue", exact: true })
            .click({ timeout: uiTimeoutMs });
        await modalByTitle(page, "Setup Complete: Preparing to Initialise Server").waitFor({
            state: "visible",
            timeout: uiTimeoutMs,
        });
    });
    return screenshots;
}

async function writeNoteViaObsidian(
    cliBinary: string,
    environment: NodeJS.ProcessEnv,
    path: string,
    content: string
): Promise<void> {
    await evalObsidianJson<unknown>(
        cliBinary,
        [
            "(async()=>{",
            `const path=${JSON.stringify(path)};`,
            `const content=${JSON.stringify(content)};`,
            "const folder=path.split('/').slice(0,-1).join('/');",
            "if(folder&&!(await app.vault.adapter.exists(folder))) await app.vault.createFolder(folder);",
            "const existing=app.vault.getAbstractFileByPath(path);",
            "if(existing) await app.vault.modify(existing,content);",
            "else await app.vault.create(path,content);",
            "return JSON.stringify({ok:true});",
            "})()",
        ].join(""),
        environment
    );
}

async function waitForVaultFile(
    vault: TemporaryVault,
    path: string,
    expected: string,
    timeoutMs = Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ?? 30000)
): Promise<void> {
    const deadline = Date.now() + timeoutMs;
    let lastContent = "";
    while (Date.now() < deadline) {
        try {
            lastContent = await readFile(join(vault.path, path), "utf8");
            if (lastContent === expected) return;
        } catch (error) {
            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
        }
        await new Promise((resolve) => setTimeout(resolve, 250));
    }
    throw new Error(`Timed out waiting for ${path}. Last content:\n${lastContent}`);
}

async function waitForRemoteEntry(context: RunnerContext, entry: { id: string; children: string[] }): Promise<void> {
    await waitForCouchDbDocs(context.couchDb, context.dbName, (docs) => {
        const ids = new Set(docs.map((doc) => doc._id));
        return ids.has(entry.id) && entry.children.every((childId) => ids.has(childId));
    });
}

async function assertPersistedE2EE(vault: TemporaryVault): Promise<void> {
    const persisted = JSON.parse(
        await readFile(join(vault.path, ".obsidian", "plugins", "obsidian-livesync", "data.json"), "utf8")
    ) as {
        encrypt?: unknown;
        encryptedPassphrase?: unknown;
        passphrase?: unknown;
    };
    assertEqual(persisted.encrypt, true, "Manual CouchDB setup did not persist E2EE as enabled.");
    assertEqual(persisted.passphrase, "", "Manual CouchDB setup persisted the E2EE passphrase in plain text.");
    if (typeof persisted.encryptedPassphrase !== "string" || persisted.encryptedPassphrase.length === 0) {
        throw new Error("Manual CouchDB setup did not persist an encrypted E2EE passphrase.");
    }
}

async function setRemotePreferredE2EEDisabled(context: RunnerContext): Promise<void> {
    const milestone = await fetchCouchDbDocument(context.couchDb, context.dbName, MILESTONE_DOCID);
    const tweakValues = milestone.tweak_values;
    if (typeof tweakValues !== "object" || tweakValues === null || Array.isArray(tweakValues)) {
        throw new Error("The existing CouchDB milestone did not contain synchronisation settings.");
    }
    const preferred = (tweakValues as Record<string, unknown>)[DEVICE_ID_PREFERRED];
    if (typeof preferred !== "object" || preferred === null || Array.isArray(preferred)) {
        throw new Error("The existing CouchDB milestone did not contain preferred synchronisation settings.");
    }
    await putCouchDbDocument(context.couchDb, context.dbName, {
        ...milestone,
        tweak_values: {
            ...tweakValues,
            [DEVICE_ID_PREFERRED]: {
                ...(preferred as Record<string, unknown>),
                encrypt: false,
            },
        },
    });
}

async function assertRemotePreferredE2EE(context: RunnerContext, expected: boolean): Promise<void> {
    const milestone = await fetchCouchDbDocument(context.couchDb, context.dbName, MILESTONE_DOCID);
    const tweakValues = milestone.tweak_values;
    const preferred =
        typeof tweakValues === "object" && tweakValues !== null && !Array.isArray(tweakValues)
            ? (tweakValues as Record<string, unknown>)[DEVICE_ID_PREFERRED]
            : undefined;
    const encrypt =
        typeof preferred === "object" && preferred !== null && !Array.isArray(preferred)
            ? (preferred as Record<string, unknown>).encrypt
            : undefined;
    assertEqual(encrypt, expected, `The remote preferred E2EE setting was not ${expected ? "enabled" : "disabled"}.`);
}

async function scheduleRemoteOverwrite(port: number): Promise<void> {
    await withObsidianPage(port, async (page) => {
        const settingsNavigator = await openLiveSyncSettings(page, uiTimeoutMs);
        const maintenance = await settingsNavigator.openPage("Maintenance");
        const overwrite = maintenance
            .locator(".setting-item")
            .filter({ hasText: "Overwrite Server Data with This Device's Files" });
        await overwrite
            .getByRole("button", { name: "Schedule and Restart", exact: true })
            .click({ timeout: uiTimeoutMs });
    });
}

async function assertRemoteEntryEncrypted(
    context: RunnerContext,
    entry: { id: string; path: string; children: string[] },
    plaintextPath: string,
    plaintext: string
): Promise<void> {
    const remoteMetadata = await fetchCouchDbDocument(context.couchDb, context.dbName, entry.id);
    const serialisedMetadata = JSON.stringify(remoteMetadata);
    if (
        !remoteMetadata._id.startsWith("f:") ||
        typeof remoteMetadata.path !== "string" ||
        !remoteMetadata.path.startsWith("/\\:") ||
        remoteMetadata.path === entry.path ||
        serialisedMetadata.includes(plaintextPath) ||
        !Array.isArray(remoteMetadata.children) ||
        remoteMetadata.children.length !== 0 ||
        remoteMetadata.mtime !== 0 ||
        remoteMetadata.ctime !== 0 ||
        remoteMetadata.size !== 0
    ) {
        throw new Error("The directly fetched CouchDB Metadata document did not protect its properties.");
    }

    const childId = entry.children[0];
    if (!childId) {
        throw new Error("The local E2EE test entry did not reference a Chunk document.");
    }
    if (!childId.startsWith("h:+")) {
        throw new Error(`The E2EE test entry used an unencrypted Chunk identifier: ${childId}`);
    }
    const remoteChunk = await fetchCouchDbDocument(context.couchDb, context.dbName, childId);
    assertEqual(remoteChunk.e_, true, "The directly fetched CouchDB Chunk was not marked as encrypted.");
    if (
        typeof remoteChunk.data !== "string" ||
        remoteChunk.data === plaintext ||
        remoteChunk.data.includes(plaintext)
    ) {
        throw new Error("The directly fetched CouchDB Chunk contained readable Vault content.");
    }
}

async function main(): Promise<void> {
    const binary = requireObsidianBinary();
    const cli = discoverObsidianCli();
    if (!cli.binary) {
        throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`);
    }
    const couchDb = await loadCouchDbConfig();
    const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "manual-setup");
    const vaultA = await createTemporaryVault();
    const vaultB = await createTemporaryVault();
    const context: RunnerContext = {
        binary,
        cliBinary: cli.binary,
        couchDb,
        dbName,
        activeSessions: new Set(),
    };
    const screenshots: string[] = [];
    let secondDeviceArtifact: SetupArtifact | undefined;

    try {
        await assertCouchDbReachable(couchDb);
        console.log(`Using Obsidian executable: ${binary}`);
        console.log(`Temporary Vault A: ${vaultA.path}`);
        console.log(`Temporary Vault B: ${vaultB.path}`);
        console.log(`CouchDB database to be created by the onboarding dialogue: ${dbName}`);

        let session = await startUnconfiguredSession(context, vaultA);
        try {
            screenshots.push(...(await enterManualCouchDBSettings(session.remoteDebuggingPort, couchDb, dbName)));
            screenshots.push(await captureAndStartInitialisation(session.remoteDebuggingPort, "new", captures));
            screenshots.push(await confirmRebuild(session.remoteDebuggingPort, captures));
            screenshots.push(await continueWithoutRemoteSettings(session.remoteDebuggingPort, captures));
            screenshots.push(await acknowledgeDisabledOptionalFeatures(session.remoteDebuggingPort, captures));
            const state = await finishInitialisation(session.remoteDebuggingPort, context.cliBinary, session.cliEnv);
            await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
            assertEqual(state.activeConfigurationId !== "", true, "Manual CouchDB setup did not activate a profile.");
            assertEqual(
                state.remoteConfigurationCount,
                1,
                "Manual CouchDB setup did not persist exactly one remote profile."
            );
            await assertPersistedE2EE(vaultA);

            await writeNoteViaObsidian(context.cliBinary, session.cliEnv, notePath, noteContent);
            const entry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, notePath);
            await pushLocalChanges(context.cliBinary, session.cliEnv);
            await waitForRemoteEntry(context, entry);
        } catch (error) {
            await captureFailure(session, "first-device");
            throw error;
        } finally {
            await stopTrackedSession(context, session);
        }

        await setRemotePreferredE2EEDisabled(context);
        await assertRemotePreferredE2EE(context, false);

        session = await startUnconfiguredSession(context, vaultA);
        try {
            await scheduleRemoteOverwrite(session.remoteDebuggingPort);
            screenshots.push(await confirmRebuild(session.remoteDebuggingPort, e2eeRebuildCaptures));
            screenshots.push(
                await acknowledgeDisabledOptionalFeatures(session.remoteDebuggingPort, e2eeRebuildCaptures)
            );
            await finishInitialisation(session.remoteDebuggingPort, context.cliBinary, session.cliEnv);
            await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
            await assertPersistedE2EE(vaultA);

            const rebuiltEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, notePath);
            await waitForRemoteEntry(context, rebuiltEntry);
            await assertRemoteEntryEncrypted(context, rebuiltEntry, notePath, noteContent);
            await assertRemotePreferredE2EE(context, true);

            const generated = await generateSetupURIFromDevice(
                session.remoteDebuggingPort,
                randomBytes(24).toString("base64url"),
                captures
            );
            secondDeviceArtifact = generated.artifact;
            screenshots.push(...generated.screenshots);
        } catch (error) {
            await captureFailure(session, "e2ee-rebuild");
            throw error;
        } finally {
            await stopTrackedSession(context, session);
        }

        session = await startUnconfiguredSession(context, vaultB);
        try {
            if (!secondDeviceArtifact) {
                throw new Error("The manually configured first device did not generate a Setup URI.");
            }
            screenshots.push(
                await enterSetupURI(session.remoteDebuggingPort, "existing", secondDeviceArtifact, captures)
            );
            screenshots.push(await captureAndStartInitialisation(session.remoteDebuggingPort, "existing", captures));
            screenshots.push(...(await confirmFastFetch(session.remoteDebuggingPort, captures)));
            await finishInitialisation(session.remoteDebuggingPort, context.cliBinary, session.cliEnv);
            await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
            await pushLocalChanges(context.cliBinary, session.cliEnv);
            await waitForVaultFile(vaultB, notePath, noteContent);

            await writeNoteViaObsidian(context.cliBinary, session.cliEnv, returnNotePath, returnNoteContent);
            const returnEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, returnNotePath);
            await pushLocalChanges(context.cliBinary, session.cliEnv);
            await waitForRemoteEntry(context, returnEntry);
        } catch (error) {
            await captureFailure(session, "second-device");
            throw error;
        } finally {
            await stopTrackedSession(context, session);
        }

        session = await startUnconfiguredSession(context, vaultA);
        try {
            await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
            await pushLocalChanges(context.cliBinary, session.cliEnv);
            await waitForVaultFile(vaultA, returnNotePath, returnNoteContent);
        } catch (error) {
            await captureFailure(session, "return-journey");
            throw error;
        } finally {
            await stopTrackedSession(context, session);
        }

        console.log(
            `Manual CouchDB onboarding created and tested its database, generated a second-device Setup URI, and completed a bidirectional note round-trip. Screenshots: ${screenshots.join(", ")}`
        );
    } finally {
        await stopTrackedSessions(context).catch((error: unknown) => {
            console.warn(error instanceof Error ? error.message : error);
        });
        await vaultA.dispose();
        await vaultB.dispose();
        if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
            await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => {
                console.warn(error instanceof Error ? error.message : error);
            });
        }
    }
}

main().catch((error: unknown) => {
    console.error(error instanceof Error ? error.stack : error);
    process.exit(1);
});