Merge main and verify ID keys through Setup URI and QR

This commit is contained in:
vorotamoroz
2026-09-29 14:28:51 +00:00
51 changed files with 2823 additions and 181 deletions
@@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { DEVICE_ID_PREFERRED, MILESTONE_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { deriveIdKey, formatIdRecoveryCode } from "@vrtmrz/livesync-commonlib/settings";
import { evalObsidianJson } from "../runner/cli.ts";
import {
assertCouchDbReachable,
@@ -14,7 +15,12 @@ import {
type CouchDbConfig,
} from "../runner/couchdb.ts";
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
import { assertEqual, pushLocalChanges, waitForLocalDatabaseEntry } from "../runner/liveSyncWorkflow.ts";
import {
assertEqual,
assertE2eCompatibilityUnpaused,
pushLocalChanges,
waitForLocalDatabaseEntry,
} from "../runner/liveSyncWorkflow.ts";
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
import {
acknowledgeDisabledOptionalFeatures,
@@ -26,7 +32,6 @@ import {
finishInitialisation,
generateSetupURIFromDevice,
modalByTitle,
resumeCompatibilityReviewIfShown,
selectRadioOption,
continueWithoutRemoteSettings,
type SetupArtifact,
@@ -99,7 +104,12 @@ async function captureFailure(session: ObsidianLiveSyncSession, label: string):
}
}
async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig, dbName: string): Promise<string[]> {
async function enterManualCouchDBSettings(
port: number,
couchDb: CouchDbConfig,
dbName: string,
independentIdSource?: string
): Promise<string[]> {
const screenshots: string[] = [];
await withObsidianPage(port, async (page) => {
const invitation = page.locator(".notice").filter({ hasText: "Welcome to Self-hosted LiveSync" });
@@ -134,12 +144,41 @@ async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig,
0,
"The Obfuscate Properties row was present before end-to-end encryption was enabled."
);
const disabledIdChoices = encryption.locator("fieldset.sls-id-choices").first();
assertEqual(
await disabledIdChoices.evaluate((element) => element.hasAttribute("disabled")),
true,
"The ID configuration was enabled while E2EE was off."
);
for (const value of ["keep", "random", "custom"]) {
assertEqual(
await disabledIdChoices.locator(`input[value="${value}"]`).isDisabled(),
true,
`The ${value} ID configuration was enabled while E2EE was off.`
);
}
const disabledIdScreenshot = join(
process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e",
"guide-couchdb-manual-id-generation-disabled.png"
);
await disabledIdChoices.screenshot({ path: disabledIdScreenshot });
screenshots.push(disabledIdScreenshot);
assertEqual(
await disabledIdChoices.evaluate((element) => Number(getComputedStyle(element).opacity) < 1),
true,
"The disabled ID configuration did not look disabled in the default theme."
);
await encryption
.locator("label.row")
.filter({ hasText: "End-to-End Encryption" })
.locator('input[type="checkbox"]')
.first()
.check({ timeout: uiTimeoutMs });
assertEqual(
await disabledIdChoices.evaluate((element) => Number(getComputedStyle(element).opacity)),
1,
"The ID configuration remained dimmed after E2EE was enabled."
);
const passphraseInput = encryption.locator('input[name="e2ee-passphrase"]');
await passphraseInput.waitFor({ state: "visible", timeout: uiTimeoutMs });
await encryption
@@ -149,7 +188,85 @@ async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig,
.first()
.check({ timeout: uiTimeoutMs });
await passphraseInput.fill(e2eePassphrase);
const passwordToggle = encryption.locator("button.sls-password-toggle");
const idChoices = encryption.locator('input[type="radio"][name="id-derivation-choice"]');
assertEqual(await idChoices.count(), 3, "The three ID configurations were not all shown.");
for (const value of ["keep", "random", "custom"]) {
assertEqual(
await encryption.locator(`input[name="id-derivation-choice"][value="${value}"]`).isVisible(),
true,
`The ${value} ID configuration was not visible.`
);
}
const keepChoice = encryption.locator('input[name="id-derivation-choice"][value="keep"]');
const randomChoice = encryption.locator('input[name="id-derivation-choice"][value="random"]');
const customChoice = encryption.locator('input[name="id-derivation-choice"][value="custom"]');
assertEqual(await randomChoice.isChecked(), true, "The default ID configuration was not random.");
assertEqual(
await encryption.getByText("Keep current configuration", { exact: true }).count(),
1,
"The current-configuration choice was not labelled consistently."
);
assertEqual(
await encryption.getByText("Current configuration: no ID key is saved.", { exact: false }).count(),
1,
"The current legacy configuration was not explained."
);
await keepChoice.check({ timeout: uiTimeoutMs });
assertEqual(
await encryption.getByText("Changing the E2EE passphrase changes IDs", { exact: false }).count(),
1,
"Keeping legacy IDs did not explain the effect of changing the E2EE passphrase."
);
await randomChoice.check({ timeout: uiTimeoutMs });
if (independentIdSource) {
await customChoice.check({ timeout: uiTimeoutMs });
const customChoices = encryption.locator('input[type="radio"][name="id-custom-choice"]');
assertEqual(await customChoices.count(), 3, "The three custom ID inputs were not all shown.");
for (const value of ["passphrase", "source", "recovery"]) {
assertEqual(
await encryption.locator(`input[name="id-custom-choice"][value="${value}"]`).isVisible(),
true,
`The ${value} custom ID input was not visible.`
);
}
const sourceChoice = encryption.locator('input[name="id-custom-choice"][value="source"]');
assertEqual(await sourceChoice.isChecked(), true, "The custom ID input was not selected by default.");
await encryption
.locator('input[name="id-custom-choice"][value="passphrase"]')
.check({ timeout: uiTimeoutMs });
assertEqual(
await encryption.locator('input[name="id-derivation-source"]').count(),
0,
"The E2EE passphrase choice exposed a second source input."
);
await encryption
.locator('input[name="id-custom-choice"][value="recovery"]')
.check({ timeout: uiTimeoutMs });
assertEqual(
await encryption.locator('input[name="id-derivation-source"]').getAttribute("placeholder"),
"Enter an ID recovery code",
"The recovery-code choice did not request a recovery code."
);
const recoveryChoice = encryption.locator('input[name="id-custom-choice"][value="recovery"]');
await sourceChoice.check({ timeout: uiTimeoutMs });
const sourceInput = encryption.locator('input[name="id-derivation-source"]');
await sourceInput.fill("");
await encryption.getByRole("button", { name: "Proceed", exact: true }).click({ timeout: uiTimeoutMs });
assertEqual(
await encryption.getByText("An ID source is required to enable this option.", { exact: false }).count(),
1,
"A first-time independent ID configuration did not require a source."
);
assertEqual(
await encryption.isVisible(),
true,
"The E2EE dialogue closed after a first-time ID source was omitted."
);
await recoveryChoice.check({ timeout: uiTimeoutMs });
await sourceChoice.check({ timeout: uiTimeoutMs });
await sourceInput.fill(independentIdSource);
}
const passwordToggle = passphraseInput.locator("..").locator("button.sls-password-toggle");
await passwordToggle.click({ timeout: uiTimeoutMs });
assertEqual(
await passphraseInput.getAttribute("type"),
@@ -167,16 +284,13 @@ async function enterManualCouchDBSettings(port: number, couchDb: CouchDbConfig,
"password",
"Toggling visibility again did not re-mask the passphrase."
);
assertEqual(
await passphraseInput.inputValue(),
e2eePassphrase,
"Re-masking the passphrase changed its value."
);
assertEqual(await passphraseInput.inputValue(), e2eePassphrase, "Re-masking the passphrase changed its value.");
});
screenshots.push(await captureGuideDialogue(port, "guide-couchdb-manual-encryption.png", "End-to-End Encryption"));
await withObsidianPage(port, async (page) => {
const encryption = modalByTitle(page, "End-to-End Encryption");
await encryption.getByRole("button", { name: "Proceed", exact: true }).click({ timeout: uiTimeoutMs });
await encryption.waitFor({ state: "hidden", timeout: uiTimeoutMs });
});
screenshots.push(
@@ -288,13 +402,18 @@ async function waitForRemoteEntry(context: RunnerContext, entry: { id: string; c
});
}
async function assertPersistedE2EE(vault: TemporaryVault): Promise<void> {
const persisted = JSON.parse(
await readFile(join(vault.path, ".obsidian", "plugins", "obsidian-livesync", "data.json"), "utf8")
) as {
async function assertPersistedE2EE(vault: TemporaryVault, independentIdSource?: string): Promise<void> {
const rawSettings = await readFile(
join(vault.path, ".obsidian", "plugins", "obsidian-livesync", "data.json"),
"utf8"
);
const persisted = JSON.parse(rawSettings) as {
encrypt?: unknown;
encryptedPassphrase?: unknown;
passphrase?: unknown;
idDerivationVersion?: unknown;
idDerivationKey?: unknown;
encryptedIdDerivationKey?: unknown;
};
assertEqual(persisted.encrypt, true, "Manual CouchDB setup did not persist E2EE as enabled.");
assertEqual(persisted.passphrase, "", "Manual CouchDB setup persisted the E2EE passphrase in plain text.");
@@ -304,6 +423,14 @@ async function assertPersistedE2EE(vault: TemporaryVault): Promise<void> {
if (JSON.stringify(persisted).includes(e2eePassphrase)) {
throw new Error("Manual CouchDB setup persisted the E2EE passphrase in plain text.");
}
assertEqual(persisted.idDerivationVersion, 1, "The independent ID mode was not persisted.");
assertEqual(persisted.idDerivationKey, "", "The derived ID key was stored in plain text.");
if (typeof persisted.encryptedIdDerivationKey !== "string" || !persisted.encryptedIdDerivationKey) {
throw new Error("The derived ID key was not encrypted in local settings.");
}
if (independentIdSource) {
if (rawSettings.includes(independentIdSource)) throw new Error("The ID source was stored in local settings.");
}
}
async function assertRestoredE2EEPassphrase(session: ObsidianLiveSyncSession, cliBinary: string): Promise<void> {
@@ -320,6 +447,126 @@ async function assertRestoredE2EEPassphrase(session: ObsidianLiveSyncSession, cl
assertEqual(restored, true, "The E2EE passphrase was not restored after Obsidian restarted.");
}
async function assertCurrentIdDerivationKey(
session: ObsidianLiveSyncSession,
cliBinary: string,
expected: string,
context: string
): Promise<void> {
const settings = await evalObsidianJson<{ idDerivationVersion: number; idDerivationKey: string }>(
cliBinary,
[
"(()=>{",
"const settings=app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings();",
"return JSON.stringify({idDerivationVersion:settings.idDerivationVersion,idDerivationKey:settings.idDerivationKey});",
"})()",
].join(""),
session.cliEnv
);
assertEqual(settings.idDerivationVersion, 1, `${context}: the independent ID version was not retained.`);
assertEqual(settings.idDerivationKey, expected, `${context}: the saved ID key changed.`);
}
async function assertRecoveryCodeCanBeRevealed(
session: ObsidianLiveSyncSession,
cliBinary: string,
source: string
): Promise<void> {
const port = session.remoteDebuggingPort;
const expected = formatIdRecoveryCode(await deriveIdKey(source));
await withObsidianPage(port, async (page) => {
const settingsNavigator = await openLiveSyncSettings(page, uiTimeoutMs);
const remotePage = await settingsNavigator.openPage("Remote Configuration");
await remotePage
.locator(".setting-item")
.filter({ hasText: "Configure E2EE" })
.getByRole("button", { name: "Configure", exact: true })
.click({ timeout: uiTimeoutMs });
const encryption = modalByTitle(page, "End-to-End Encryption");
await encryption.waitFor({ state: "visible", timeout: uiTimeoutMs });
assertEqual(
await encryption.locator('input[name="id-derivation-choice"][value="keep"]').isChecked(),
true,
"An existing ID key was not selected for reuse."
);
assertEqual(
await encryption.getByText("Current configuration: a saved ID key is used.").count(),
1,
"The saved ID key was not explained."
);
await encryption.getByRole("button", { name: "Show current recovery code" }).click({ timeout: uiTimeoutMs });
assertEqual(
await encryption.getByRole("textbox", { name: "Current ID recovery code" }).inputValue(),
expected,
"The displayed recovery code did not contain the saved ID key."
);
await encryption.locator('input[name="id-derivation-choice"][value="custom"]').check({ timeout: uiTimeoutMs });
await encryption.locator('input[name="id-custom-choice"][value="recovery"]').check({ timeout: uiTimeoutMs });
const recoveryInput = encryption.locator('input[name="id-derivation-source"]');
await recoveryInput.fill("not-a-recovery-code");
await encryption.getByRole("button", { name: "Proceed" }).click({ timeout: uiTimeoutMs });
assertEqual(
await encryption.getByText("The ID source or recovery code is invalid.", { exact: false }).count(),
1,
"The recovery-code choice accepted an ordinary source string."
);
await recoveryInput.fill(expected);
await encryption.getByRole("button", { name: "Proceed" }).click({ timeout: uiTimeoutMs });
await encryption.waitFor({ state: "hidden", timeout: uiTimeoutMs });
assertEqual(
await modalByTitle(page, "Mostly Complete: Decision Required").count(),
0,
"Recovering the existing ID key opened a new setup decision."
);
});
const expectedIdKey = await deriveIdKey(source);
await assertCurrentIdDerivationKey(session, cliBinary, expectedIdKey, "Recovering the saved ID key");
await withObsidianPage(port, async (page) => {
const settingsNavigator = await openLiveSyncSettings(page, uiTimeoutMs);
const remotePage = await settingsNavigator.openPage("Remote Configuration");
await remotePage
.locator(".setting-item")
.filter({ hasText: "Configure E2EE" })
.getByRole("button", { name: "Configure", exact: true })
.click({ timeout: uiTimeoutMs });
const encryption = modalByTitle(page, "End-to-End Encryption");
await encryption.waitFor({ state: "visible", timeout: uiTimeoutMs });
await encryption.locator('input[name="id-derivation-choice"][value="custom"]').check({ timeout: uiTimeoutMs });
await encryption.locator('input[name="id-custom-choice"][value="source"]').check({ timeout: uiTimeoutMs });
const sourceInput = encryption.locator('input[name="id-derivation-source"]');
await sourceInput.fill("");
assertEqual(await sourceInput.inputValue(), "", "The independent ID source input was not empty.");
assertEqual(
await encryption.getByText("Leave this input empty to keep the saved ID key.", { exact: true }).count(),
1,
"The configured ID source did not explain that an empty input keeps the saved ID key."
);
await encryption.getByRole("button", { name: "Proceed", exact: true }).click({ timeout: uiTimeoutMs });
await encryption.waitFor({ state: "hidden", timeout: uiTimeoutMs });
assertEqual(
await modalByTitle(page, "Mostly Complete: Decision Required").count(),
0,
"Keeping the saved ID key after an empty source opened a new setup decision."
);
});
await assertCurrentIdDerivationKey(session, cliBinary, expectedIdKey, "Saving an empty custom ID source");
}
async function readDocumentId(cliBinary: string, environment: NodeJS.ProcessEnv, path: string): Promise<string> {
return await evalObsidianJson<string>(
cliBinary,
[
"(async()=>{",
`const path=${JSON.stringify(path)};`,
"const core=app.plugins.plugins['obsidian-livesync'].core;",
"return JSON.stringify(await core.services.path.path2id(path));",
"})()",
].join(""),
environment
);
}
async function setRemotePreferredE2EEDisabled(context: RunnerContext): Promise<void> {
const milestone = await fetchCouchDbDocument(context.couchDb, context.dbName, MILESTONE_DOCID);
const tweakValues = milestone.tweak_values;
@@ -430,6 +677,10 @@ async function main(): Promise<void> {
};
const screenshots: string[] = [];
let secondDeviceArtifact: SetupArtifact | undefined;
const independentIdSource =
process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true" ? randomBytes(32).toString("base64url") : undefined;
let firstEntryId: string | undefined;
let returnEntryId: string | undefined;
try {
await assertCouchDbReachable(couchDb);
@@ -440,23 +691,29 @@ async function main(): Promise<void> {
let session = await startUnconfiguredSession(context, vaultA);
try {
screenshots.push(...(await enterManualCouchDBSettings(session.remoteDebuggingPort, couchDb, dbName)));
screenshots.push(
...(await enterManualCouchDBSettings(session.remoteDebuggingPort, couchDb, dbName, independentIdSource))
);
screenshots.push(await captureAndStartInitialisation(session.remoteDebuggingPort, "new", captures));
screenshots.push(await confirmRebuild(session.remoteDebuggingPort, captures));
screenshots.push(await continueWithoutRemoteSettings(session.remoteDebuggingPort, captures));
screenshots.push(await acknowledgeDisabledOptionalFeatures(session.remoteDebuggingPort, captures));
const state = await finishInitialisation(session.remoteDebuggingPort, context.cliBinary, session.cliEnv);
await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
await assertE2eCompatibilityUnpaused(context.cliBinary, session.cliEnv, session.remoteDebuggingPort);
assertEqual(state.activeConfigurationId !== "", true, "Manual CouchDB setup did not activate a profile.");
assertEqual(
state.remoteConfigurationCount,
1,
"Manual CouchDB setup did not persist exactly one remote profile."
);
await assertPersistedE2EE(vaultA);
await assertPersistedE2EE(vaultA, independentIdSource);
if (independentIdSource) {
await assertRecoveryCodeCanBeRevealed(session, context.cliBinary, independentIdSource);
}
await writeNoteViaObsidian(context.cliBinary, session.cliEnv, notePath, noteContent);
const entry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, notePath);
firstEntryId = entry.id;
await pushLocalChanges(context.cliBinary, session.cliEnv);
await waitForRemoteEntry(context, entry);
} catch (error) {
@@ -478,10 +735,13 @@ async function main(): Promise<void> {
await acknowledgeDisabledOptionalFeatures(session.remoteDebuggingPort, e2eeRebuildCaptures)
);
await finishInitialisation(session.remoteDebuggingPort, context.cliBinary, session.cliEnv);
await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
await assertPersistedE2EE(vaultA);
await assertE2eCompatibilityUnpaused(context.cliBinary, session.cliEnv, session.remoteDebuggingPort);
await assertPersistedE2EE(vaultA, independentIdSource);
const rebuiltEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, notePath);
if (independentIdSource) {
assertEqual(rebuiltEntry.id, firstEntryId, "Rebuild changed the configured document ID.");
}
await waitForRemoteEntry(context, rebuiltEntry);
await assertRemoteEntryEncrypted(context, rebuiltEntry, notePath, noteContent);
await assertRemotePreferredE2EE(context, true);
@@ -511,12 +771,21 @@ async function main(): Promise<void> {
screenshots.push(await captureAndStartInitialisation(session.remoteDebuggingPort, "existing", captures));
screenshots.push(...(await confirmFastFetch(session.remoteDebuggingPort, captures)));
await finishInitialisation(session.remoteDebuggingPort, context.cliBinary, session.cliEnv);
await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
await assertE2eCompatibilityUnpaused(context.cliBinary, session.cliEnv, session.remoteDebuggingPort);
await assertPersistedE2EE(vaultB, independentIdSource);
await pushLocalChanges(context.cliBinary, session.cliEnv);
await waitForVaultFile(vaultB, notePath, noteContent);
if (independentIdSource) {
assertEqual(
await readDocumentId(context.cliBinary, session.cliEnv, notePath),
firstEntryId,
"The Setup URI did not restore the document ID key on the second device."
);
}
await writeNoteViaObsidian(context.cliBinary, session.cliEnv, returnNotePath, returnNoteContent);
const returnEntry = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, returnNotePath);
returnEntryId = returnEntry.id;
await pushLocalChanges(context.cliBinary, session.cliEnv);
await waitForRemoteEntry(context, returnEntry);
} catch (error) {
@@ -528,9 +797,16 @@ async function main(): Promise<void> {
session = await startUnconfiguredSession(context, vaultA);
try {
await resumeCompatibilityReviewIfShown(session.remoteDebuggingPort);
await assertE2eCompatibilityUnpaused(context.cliBinary, session.cliEnv, session.remoteDebuggingPort);
await pushLocalChanges(context.cliBinary, session.cliEnv);
await waitForVaultFile(vaultA, returnNotePath, returnNoteContent);
if (independentIdSource) {
assertEqual(
await readDocumentId(context.cliBinary, session.cliEnv, returnNotePath),
returnEntryId,
"The first device did not retain the shared document ID key."
);
}
} catch (error) {
await captureFailure(session, "return-journey");
throw error;
+4
View File
@@ -35,6 +35,10 @@ const testSteps: Step[] = [
name: "Object Storage Setup URI workflow",
args: ["run", "test:e2e:obsidian:object-storage-setup-uri-workflow"],
},
{
name: "Object Storage Compatible Setup URI workflow",
args: ["run", "test:e2e:obsidian:object-storage-compatible-setup-uri-workflow"],
},
{
name: "Object Storage QR workflow",
args: ["run", "test:e2e:obsidian:object-storage-qr-workflow"],
+41 -4
View File
@@ -15,6 +15,8 @@
* Separate successes would not prove that those observations belonged to the
* same upload.
*/
import { randomBytes } from "node:crypto";
import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings";
import { evalObsidianJson } from "../runner/cli.ts";
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
import {
@@ -33,6 +35,7 @@ import {
listObjectStorageObjects,
loadObjectStorageConfig,
makeUniqueBucketPrefix,
readObjectStorageJson,
} from "../runner/objectStorage.ts";
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
import { createTemporaryVault } from "../runner/vault.ts";
@@ -41,6 +44,8 @@ import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../r
process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
const notePath = "E2E/minio-upload.md";
const useIndependentIds = process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true";
const useCustomRequestHandler = process.env.E2E_OBSIDIAN_CUSTOM_HTTP_HANDLER === "true";
const noteContent = [
"# Object Storage upload from real Obsidian",
"",
@@ -127,10 +132,23 @@ async function main(): Promise<void> {
});
await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
const configured = await configureObjectStorage(cli.binary, session.cliEnv, {
...objectStorage,
bucketPrefix,
});
const configured = await configureObjectStorage(
cli.binary,
session.cliEnv,
{ ...objectStorage, bucketPrefix },
{
...(useIndependentIds
? {
encrypt: true,
usePathObfuscation: true,
passphrase: randomBytes(32).toString("base64url"),
idDerivationVersion: 1,
idDerivationKey: await deriveIdKey(randomBytes(32).toString("base64url")),
}
: {}),
...(useCustomRequestHandler ? { useCustomRequestHandler: true } : {}),
}
);
await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
assertEqual(configured.isConfigured, true, "Self-hosted LiveSync was not marked as configured.");
assertEqual(configured.remoteType, "MINIO", "Remote type was not Object Storage.");
@@ -145,6 +163,16 @@ async function main(): Promise<void> {
REMOTE_ACTIVITY_EXPECTED_STATE.idle
);
const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv);
if (useIndependentIds) {
if (
!/^f:[0-9a-f]{64}$/u.test(localEntry.id) ||
localEntry.children.some((child) => !/^h:\+[0-9a-f]{64}$/u.test(child))
) {
throw new Error(
`The real Obsidian Journal upload did not use independent document and Chunk IDs (document length ${localEntry.id.length}, Chunk lengths ${localEntry.children.map((child) => child.length).join(",")}).`
);
}
}
await pushLocalChanges(cli.binary, session.cliEnv);
const activityAfterUpload = await waitForRemoteActivityState(
session.remoteDebuggingPort,
@@ -160,6 +188,15 @@ async function main(): Promise<void> {
);
const keys = await waitForObjectStorageObjects(bucketPrefix);
if (useIndependentIds) {
const milestone = await readObjectStorageJson<{ encrypted_id_derivation_proof?: string }>(
objectStorage,
`${bucketPrefix}_00000000-milestone.json`
);
if (!milestone.encrypted_id_derivation_proof) {
throw new Error("The Journal milestone did not retain an encrypted ID agreement proof.");
}
}
console.log(
`Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}`
@@ -3,13 +3,14 @@ import { randomBytes } from "node:crypto";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { promisify } from "node:util";
import { Worker } from "node:worker_threads";
import { encodeSettingsToQRCodeData } from "@vrtmrz/livesync-commonlib/compat/API/processSetting";
import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/setup-uri";
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { evalObsidianJson } from "../runner/cli.ts";
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
import {
assertEqual,
assertE2eCompatibilityMarker,
assertE2eCompatibilityUnpaused,
pushLocalChanges,
type ConfiguredSettings,
@@ -53,11 +54,17 @@ process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "90000";
const execFileAsync = promisify(execFile);
const useCustomRequestHandler = process.argv.includes("--custom-http-handler");
const useQRCode = process.argv.includes("--qr");
const uriMode = process.argv.includes("--compatible") ? "persistent" : "ephemeral";
const captures: SetupCaptureNames = useQRCode
? { scenario: "object-storage-qr", guide: "object-storage-qr-setup" }
: useCustomRequestHandler
? { scenario: "object-storage-custom-http-handler-setup-uri", guide: "object-storage-custom-http-handler-setup" }
: { scenario: "object-storage-setup-uri", guide: "object-storage-setup" };
: uriMode === "persistent"
? { scenario: "object-storage-compatible-uri", guide: "object-storage-compatible-setup" }
: useCustomRequestHandler
? {
scenario: "object-storage-custom-http-handler-setup-uri",
guide: "object-storage-custom-http-handler-setup",
}
: { scenario: "object-storage-setup-uri", guide: "object-storage-setup" };
const noteFromFirst = "E2E/object-storage/from-first.md";
const noteFromSecond = "E2E/object-storage/from-second.md";
const firstContent =
@@ -123,12 +130,63 @@ async function generateBootstrapSetupURI(
...(useCustomRequestHandler ? { use_custom_request_handler: "true" } : {}),
passphrase: randomBytes(24).toString("base64url"),
uri_passphrase: setupPassphrase,
uri_mode: uriMode,
});
const setupURI = output.split(/\r?\n/u).find((line) => line.startsWith("obsidian://setuplivesync?settings="));
if (!setupURI) throw new Error("The public Setup URI generator did not emit an Object Storage Setup URI.");
return { setupURI, setupPassphrase };
}
async function expiredSetupURI(settings: ObsidianLiveSyncSettings, setupPassphrase: string): Promise<SetupArtifact> {
// Only this fixture worker uses a past clock; Obsidian and the runner keep real time.
const worker = new Worker(
`
const { parentPort, workerData } = require('node:worker_threads');
Date.now = () => Date.UTC(2024, 0, 1);
import('@vrtmrz/livesync-commonlib/setup-uri').then(async ({ encodeTimeBoundSetupURI }) => {
const result = await encodeTimeBoundSetupURI(workerData.settings, workerData.setupPassphrase);
parentPort.postMessage(result.uri);
});
`,
{ eval: true, workerData: { settings, setupPassphrase } }
);
try {
const setupURI = await new Promise<string>((resolve, reject) => {
worker.once("message", resolve);
worker.once("error", reject);
worker.once("exit", (code) => reject(new Error(`The expired URI fixture exited with ${code}.`)));
});
return { setupURI, setupPassphrase };
} finally {
await worker.terminate();
}
}
async function assertIndependentIdKey(
context: RunnerContext,
session: ObsidianLiveSyncSession,
vault: TemporaryVault,
expectedKey: string
): Promise<void> {
const matches = await evalObsidianJson<boolean>(
context.cliBinary,
`(() => {
const settings = app.plugins.plugins['obsidian-livesync'].core.services.setting.currentSettings();
return JSON.stringify(settings.encrypt && settings.usePathObfuscation &&
settings.idDerivationVersion === 1 && settings.idDerivationKey === ${JSON.stringify(expectedKey)});
})()`,
session.cliEnv
);
assertEqual(matches, true, "The device did not retain the shared independent ID key and Path Obfuscation.");
const raw = await readFile(join(vault.path, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8");
const saved = JSON.parse(raw) as Record<string, unknown>;
assertEqual(saved.idDerivationVersion, 1, "The independent ID version was not saved.");
assertEqual(saved.idDerivationKey, "", "The ID key was saved in plain text.");
if (!saved.encryptedIdDerivationKey || raw.includes(expectedKey)) {
throw new Error("The shared ID key was not encrypted in local settings.");
}
}
async function startSession(
context: RunnerContext,
vault: TemporaryVault,
@@ -297,6 +355,18 @@ async function main(): Promise<void> {
const objectStorage = await loadObjectStorageConfig();
const bucketPrefix = makeUniqueBucketPrefix("setup-uri-workflow");
const bootstrapArtifact = await generateBootstrapSetupURI(objectStorage, bucketPrefix, useCustomRequestHandler);
const bootstrapSettings = await decodeSettingsFromSetupURI(
bootstrapArtifact.setupURI,
bootstrapArtifact.setupPassphrase
);
if (!bootstrapSettings || bootstrapSettings.idDerivationVersion !== 1 || !bootstrapSettings.idDerivationKey) {
throw new Error("The public Setup URI generator did not configure an independent ID key.");
}
const idKey = bootstrapSettings.idDerivationKey;
const rejectedArtifacts = [
{ ...bootstrapArtifact, setupPassphrase: "incorrect-setup-passphrase" },
await expiredSetupURI(bootstrapSettings as ObsidianLiveSyncSettings, bootstrapArtifact.setupPassphrase),
];
const vaultA = await createTemporaryVault();
const vaultB = await createTemporaryVault();
const [portA, portB] = sessionPorts();
@@ -308,13 +378,14 @@ async function main(): Promise<void> {
console.log(`Temporary Object Storage target: ${objectStorage.bucket}/${bucketPrefix}`);
const sessionA = await startSession(context, vaultA, portA);
screenshots.push(await enterSetupURI(portA, "new", bootstrapArtifact, captures));
screenshots.push(await enterSetupURI(portA, "new", bootstrapArtifact, captures, rejectedArtifacts));
screenshots.push(await captureAndStartInitialisation(portA, "new", captures));
screenshots.push(await confirmRebuild(portA, captures));
screenshots.push(await continueWithoutRemoteSettings(portA, captures));
screenshots.push(await acknowledgeDisabledOptionalFeatures(portA, captures));
const firstState = await finishInitialisation(portA, context.cliBinary, sessionA.cliEnv);
await assertE2eCompatibilityUnpaused(context.cliBinary, sessionA.cliEnv, portA);
await assertIndependentIdKey(context, sessionA, vaultA, idKey);
assertEqual(
firstState.endpoint,
objectStorage.endpoint,
@@ -337,9 +408,22 @@ async function main(): Promise<void> {
);
await writeNote(context.cliBinary, sessionA.cliEnv, noteFromFirst, firstContent);
const firstEntry = await waitForLocalDatabaseEntry(context.cliBinary, sessionA.cliEnv, noteFromFirst);
if (
!/^f:[0-9a-f]{64}$/u.test(firstEntry.id) ||
firstEntry.children.length === 0 ||
firstEntry.children.some((id) => !/^h:\+[0-9a-f]{64}$/u.test(id))
) {
throw new Error("The source note did not use independent document and Chunk IDs.");
}
await pushLocalChanges(context.cliBinary, sessionA.cliEnv);
await waitForObjectStorageData(objectStorage, bucketPrefix);
const generated = await generateSetupURIFromDevice(portA, randomBytes(24).toString("base64url"), captures);
const generated = await generateSetupURIFromDevice(
portA,
randomBytes(24).toString("base64url"),
captures,
uriMode
);
if (generated.artifact.setupURI === bootstrapArtifact.setupURI) {
throw new Error("The first device returned the bootstrap Setup URI instead of generating a new one.");
}
@@ -377,7 +461,7 @@ async function main(): Promise<void> {
await stopSession(context, sessionA);
const sessionB = await startSession(context, vaultB, portB);
const initialMarker = await assertE2eCompatibilityMarker(context.cliBinary, sessionB.cliEnv);
const initialMarker = await assertE2eCompatibilityUnpaused(context.cliBinary, sessionB.cliEnv, portB);
if (qrSettings) {
assertEqual(
initialMarker.additionalSuffix === `-${qrSettings.additionalSuffixOfDatabaseName}`,
@@ -412,6 +496,7 @@ async function main(): Promise<void> {
screenshots.push(...(await confirmFastFetch(portB, captures)));
const secondState = await finishInitialisation(portB, context.cliBinary, sessionB.cliEnv);
const fetchedMarker = await assertE2eCompatibilityUnpaused(context.cliBinary, sessionB.cliEnv, portB);
await assertIndependentIdKey(context, sessionB, vaultB, idKey);
assertEqual(
secondState.endpoint,
objectStorage.endpoint,
@@ -429,6 +514,13 @@ async function main(): Promise<void> {
);
await pushLocalChanges(context.cliBinary, sessionB.cliEnv);
await waitForPathContent(vaultB, noteFromFirst, firstContent);
const importedEntry = await waitForLocalDatabaseEntry(context.cliBinary, sessionB.cliEnv, noteFromFirst);
assertEqual(importedEntry.id, firstEntry.id, "Import changed the obfuscated document ID.");
assertEqual(
JSON.stringify(importedEntry.children),
JSON.stringify(firstEntry.children),
"Import changed the Chunk IDs."
);
screenshots.push(
await captureNote(
portB,
@@ -439,10 +531,12 @@ async function main(): Promise<void> {
);
await writeNote(context.cliBinary, sessionB.cliEnv, noteFromSecond, secondContent);
const secondEntry = await waitForLocalDatabaseEntry(context.cliBinary, sessionB.cliEnv, noteFromSecond);
await pushLocalChanges(context.cliBinary, sessionB.cliEnv);
await stopSession(context, sessionB);
const returningSessionB = await startSession(context, vaultB, portB);
await waitForLiveSyncCoreReady(context.cliBinary, returningSessionB.cliEnv);
await assertIndependentIdKey(context, returningSessionB, vaultB, idKey);
const restartedMarker = await assertE2eCompatibilityUnpaused(
context.cliBinary,
returningSessionB.cliEnv,
@@ -459,10 +553,22 @@ async function main(): Promise<void> {
const returningSessionA = await startSession(context, vaultA, portA);
await waitForLiveSyncCoreReady(context.cliBinary, returningSessionA.cliEnv);
await assertE2eCompatibilityUnpaused(context.cliBinary, returningSessionA.cliEnv, portA);
await assertIndependentIdKey(context, returningSessionA, vaultA, idKey);
// Deliberately omit manual replication here. Object Storage reports
// Continuous as not applicable, so startup scheduling must honour the
// retained syncOnStart setting by running an unattended OneShot.
await waitForPathContent(vaultA, noteFromSecond, secondContent);
const returnedEntry = await waitForLocalDatabaseEntry(
context.cliBinary,
returningSessionA.cliEnv,
noteFromSecond
);
assertEqual(returnedEntry.id, secondEntry.id, "The return journey changed the obfuscated document ID.");
assertEqual(
JSON.stringify(returnedEntry.children),
JSON.stringify(secondEntry.children),
"The return journey changed the Chunk IDs."
);
screenshots.push(
await captureNote(
portA,
+112 -54
View File
@@ -167,7 +167,8 @@ async function captureReadinessFailure(
async function openHarness(): Promise<void> {
const opened = await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => {
return await page.evaluate(
(commandId) => (globalThis as ReviewHarnessTestGlobal).app?.commands?.executeCommandById(commandId) === true,
(commandId) =>
(globalThis as ReviewHarnessTestGlobal).app?.commands?.executeCommandById(commandId) === true,
"obsidian-livesync:open-review-harness"
);
});
@@ -206,11 +207,54 @@ async function runAutomaticScenarios(): Promise<void> {
});
}
async function runIdBenchmark(): Promise<void> {
await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => {
const snapshotSettings = () =>
page.evaluate(() => {
const plugin = (globalThis as ReviewHarnessTestGlobal).app?.plugins?.plugins["obsidian-livesync"] as {
core: { services: { setting: { currentSettings(): unknown } } };
};
return JSON.stringify(plugin.core.services.setting.currentSettings());
});
const before = await snapshotSettings();
const harness = page.locator('[data-testid="review-harness"]');
await harness
.locator('[data-testid="review-harness-run-id-generation-performance"]')
.click({ timeout: uiTimeoutMs });
const result = harness.locator('[data-testid="review-harness-result-id-generation-performance"]');
await result.getByText("Passed:", { exact: false }).waitFor({ state: "visible", timeout: uiTimeoutMs * 4 });
const observations = await result.locator("li").allTextContents();
for (const label of [
"Chunk IDs, 256 B",
"Chunk IDs, 4096 B",
"Chunk IDs, 32768 B",
"Obfuscated document IDs",
]) {
for (const mode of ["legacy", "independent"]) {
if (
!observations.some(
(line) =>
line.startsWith(`${label}, ${mode}: 1000 IDs total median=`) && line.includes("; per ID=")
)
) {
throw new Error(`Missing benchmark timing and units: ${label}, ${mode}`);
}
}
}
if (
!observations.some((line) => line.startsWith("ID key derivation at save time:")) ||
!observations.some((line) => line.startsWith("JavaScript heap:"))
) {
throw new Error("The benchmark did not report derivation and heap observations.");
}
if ((await snapshotSettings()) !== before) throw new Error("The benchmark changed the live settings.");
await assertNoHorizontalOverflow(page, harness, { label: "ID benchmark results" });
});
}
async function runVaultFixture(): Promise<string> {
await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => {
await page
.locator('[data-testid="review-harness-run-vault-round-trip"]')
.click({ timeout: uiTimeoutMs });
await page.locator('[data-testid="review-harness-run-vault-round-trip"]').click({ timeout: uiTimeoutMs });
const confirmation = page.locator(".modal-container").filter({
has: page.getByText("Review Harness: Vault fixture access", { exact: true }),
});
@@ -273,27 +317,22 @@ async function restartAndResumeHarness(): Promise<string> {
});
await keepCompatibilityPaused();
await waitForHarness();
return await captureObsidianDialogue(
obsidianRemoteDebuggingPort(),
"review-harness-resumed.png",
async (page) => {
const harness = page.locator('[data-testid="review-harness"]');
await harness
.locator('[data-testid="review-harness-resumed"]')
.waitFor({ state: "visible", timeout: uiTimeoutMs });
const continuationRemoved = await page.evaluate((stateKey) => {
const plugin = (globalThis as ReviewHarnessTestGlobal).app?.plugins?.plugins["obsidian-livesync"];
if (typeof plugin !== "object" || plugin === null || !("core" in plugin)) {
throw new Error("Self-hosted LiveSync is unavailable after restart.");
}
const core = (plugin as { core: { services: { setting: { getSmallConfig(key: string): string } } } })
.core;
return core.services.setting.getSmallConfig(stateKey) === "";
}, REVIEW_HARNESS_STATE_KEY);
if (!continuationRemoved) throw new Error("The one-shot continuation was not removed before use.");
await assertNoHorizontalOverflow(page, harness, { label: "resumed Review Harness" });
}
);
return await captureObsidianDialogue(obsidianRemoteDebuggingPort(), "review-harness-resumed.png", async (page) => {
const harness = page.locator('[data-testid="review-harness"]');
await harness
.locator('[data-testid="review-harness-resumed"]')
.waitFor({ state: "visible", timeout: uiTimeoutMs });
const continuationRemoved = await page.evaluate((stateKey) => {
const plugin = (globalThis as ReviewHarnessTestGlobal).app?.plugins?.plugins["obsidian-livesync"];
if (typeof plugin !== "object" || plugin === null || !("core" in plugin)) {
throw new Error("Self-hosted LiveSync is unavailable after restart.");
}
const core = (plugin as { core: { services: { setting: { getSmallConfig(key: string): string } } } }).core;
return core.services.setting.getSmallConfig(stateKey) === "";
}, REVIEW_HARNESS_STATE_KEY);
if (!continuationRemoved) throw new Error("The one-shot continuation was not removed before use.");
await assertNoHorizontalOverflow(page, harness, { label: "resumed Review Harness" });
});
}
async function completeResumedCompatibilityStep(): Promise<void> {
@@ -332,9 +371,7 @@ async function copyAndReadReport(): Promise<string> {
undefined,
{ timeout: uiTimeoutMs }
);
return await page.evaluate(
() => (globalThis as ReviewHarnessTestGlobal).reviewHarnessCopiedReport ?? ""
);
return await page.evaluate(() => (globalThis as ReviewHarnessTestGlobal).reviewHarnessCopiedReport ?? "");
});
}
@@ -348,35 +385,36 @@ async function verifyMobileHarness(): Promise<string> {
if (typeof plugin !== "object" || plugin === null || !("core" in plugin)) {
throw new Error("Self-hosted LiveSync is unavailable in mobile test mode.");
}
const core = (plugin as {
core: { services: { API: { showWindow(type: string): Promise<void> } } };
}).core;
const core = (
plugin as {
core: { services: { API: { showWindow(type: string): Promise<void> } } };
}
).core;
await core.services.API.showWindow(viewType);
}, "self-hosted-livesync-review-harness");
});
return await captureObsidianDialogue(
obsidianRemoteDebuggingPort(),
"review-harness-mobile.png",
async (page) => {
const harness = page.locator('[data-testid="review-harness"]');
await harness.waitFor({ state: "visible", timeout: uiTimeoutMs });
await assertNoHorizontalOverflow(page, harness, { label: "mobile Review Harness" });
const heading = harness.getByRole("heading", { name: "Self-hosted LiveSync review harness" });
await assertLocatorWithinSafeArea(page, heading, {
label: "mobile Review Harness heading",
safeAreaInsets: iPhoneSafeArea,
await runIdBenchmark();
return await captureObsidianDialogue(obsidianRemoteDebuggingPort(), "review-harness-mobile.png", async (page) => {
const harness = page.locator('[data-testid="review-harness"]');
await harness.waitFor({ state: "visible", timeout: uiTimeoutMs });
await harness.getByRole("heading", { name: "Self-hosted LiveSync review harness" }).scrollIntoViewIfNeeded();
await assertNoHorizontalOverflow(page, harness, { label: "mobile Review Harness" });
const heading = harness.getByRole("heading", { name: "Self-hosted LiveSync review harness" });
await assertLocatorWithinSafeArea(page, heading, {
label: "mobile Review Harness heading",
safeAreaInsets: iPhoneSafeArea,
});
for (const testId of [
"review-harness-run-automatic",
"review-harness-run-full",
"review-harness-copy-report",
"review-harness-run-id-generation-performance",
]) {
await assertLocatorHasMinimumTouchTarget(page, harness.locator(`[data-testid="${testId}"]`), {
label: testId,
});
for (const testId of [
"review-harness-run-automatic",
"review-harness-run-full",
"review-harness-copy-report",
]) {
await assertLocatorHasMinimumTouchTarget(page, harness.locator(`[data-testid="${testId}"]`), {
label: testId,
});
}
}
);
});
}
async function main(): Promise<void> {
@@ -392,6 +430,7 @@ async function main(): Promise<void> {
vault,
startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
pluginData: {
// Config Doctor is covered by settings-ui; this fixture exercises the Harness.
doctorProcessedVersion: DoctorRegulation.version,
settingVersion: CURRENT_SETTING_VERSION,
isConfigured: true,
@@ -442,15 +481,34 @@ async function main(): Promise<void> {
const vaultConfirmationScreenshot = await runVaultFixture();
const resumedScreenshot = await restartAndResumeHarness();
await completeResumedCompatibilityStep();
await runIdBenchmark();
const report = await copyAndReadReport();
if (!report.includes("## Self-hosted LiveSync Review Harness report")) {
throw new Error("The copied Review Harness report was not Markdown evidence.");
}
for (const forbidden of [vault.name, REVIEW_HARNESS_FIXTURE_ROOT]) {
if (report.includes(forbidden)) throw new Error(`The Review Harness report exposed local state: ${forbidden}`);
for (const expected of [
"1000 IDs total median=",
"; per ID=",
"ID key derivation at save time:",
"JavaScript heap:",
]) {
if (!report.includes(expected)) throw new Error(`Missing copied benchmark observation: ${expected}`);
}
for (const forbidden of [
vault.name,
REVIEW_HARNESS_FIXTURE_ROOT,
"ab".repeat(32),
"Self-hosted LiveSync ID benchmark passphrase",
"Self-hosted LiveSync ID benchmark source",
]) {
if (report.includes(forbidden))
throw new Error(`The Review Harness report exposed local state: ${forbidden}`);
}
const mobileScreenshot = await verifyMobileHarness();
const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e";
await mkdir(outputDirectory, { recursive: true });
await writeFile(join(outputDirectory, "review-harness-report.md"), report, "utf8");
console.log(
`Review Harness passed one-shot, fixture, report, and mobile checks. Screenshots: ${[
initialScreenshot,
+1
View File
@@ -21,6 +21,7 @@ const focusedScenarios = new Set([
"cli-to-obsidian-sync",
"minio-upload",
"object-storage-setup-uri-workflow",
"object-storage-compatible-setup-uri-workflow",
"object-storage-qr-workflow",
"object-storage-custom-http-handler-setup-uri-workflow",
"p2p-setup-uri-workflow",
@@ -1,5 +1,6 @@
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings";
import { evalObsidianJson } from "../runner/cli.ts";
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
import { assertEqual } from "../runner/liveSyncWorkflow.ts";
@@ -34,7 +35,11 @@ async function waitForFileContaining(
throw new Error(`Timed out waiting for setting Markdown: ${fullPath}\nLast error: ${String(lastError)}`);
}
async function configureSettingMarkdown(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
async function configureSettingMarkdown(
cliBinary: string,
env: NodeJS.ProcessEnv,
idDerivationKey: string
): Promise<void> {
await evalObsidianJson<unknown>(
cliBinary,
[
@@ -46,6 +51,8 @@ async function configureSettingMarkdown(cliBinary: string, env: NodeJS.ProcessEn
"couchDB_USER:'e2e-user',",
"couchDB_PASSWORD:'e2e-password',",
"passphrase:'e2e-passphrase',",
"idDerivationVersion:1,",
`idDerivationKey:${JSON.stringify(idDerivationKey)},`,
"showVerboseLog:true,",
"},true);",
"await core.services.setting.saveSettingData();",
@@ -64,6 +71,7 @@ async function main(): Promise<void> {
}
const vault = await createTemporaryVault();
const idDerivationKey = await deriveIdKey("setting-markdown-export-independent-id-key-fixture");
let session: ObsidianLiveSyncSession | undefined;
try {
console.log(`Using Obsidian executable: ${binary}`);
@@ -77,19 +85,39 @@ async function main(): Promise<void> {
});
// The export is available while an unconfigured Vault remains outside
// application readiness; the session helper has already loaded the plug-in.
await configureSettingMarkdown(cli.binary, session.cliEnv);
await configureSettingMarkdown(cli.binary, session.cliEnv, idDerivationKey);
const content = await waitForFileContaining(vault.path, settingPath, [
(value) => value.includes("````yaml:livesync-setting"),
(value) => value.includes(`settingSyncFile: ${settingPath}`),
(value) => value.includes("showVerboseLog: true"),
]);
const persisted = JSON.parse(
await readFile(join(vault.path, ".obsidian", "plugins", "obsidian-livesync", "data.json"), "utf-8")
) as {
idDerivationVersion?: unknown;
idDerivationKey?: unknown;
encryptedIdDerivationKey?: unknown;
};
assertEqual(persisted.idDerivationVersion, 1, "The independent ID key fixture was not persisted.");
assertEqual(persisted.idDerivationKey, "", "The independent ID key was stored in plain text locally.");
const encryptedIdDerivationKey = persisted.encryptedIdDerivationKey;
if (typeof encryptedIdDerivationKey !== "string" || encryptedIdDerivationKey.length === 0) {
throw new Error("The independent ID key fixture was not saved in encrypted local settings.");
}
assertEqual(
content.includes("couchDB_PASSWORD: e2e-password"),
false,
"Credential leaked into setting Markdown."
);
assertEqual(content.includes("passphrase: e2e-passphrase"), false, "Passphrase leaked into setting Markdown.");
assertEqual(content.includes(idDerivationKey), false, "Plaintext ID key leaked into setting Markdown.");
assertEqual(
content.includes(encryptedIdDerivationKey),
false,
"Encrypted ID key leaked into setting Markdown."
);
console.log(`Generated setting Markdown without credentials: ${settingPath}`);
} finally {
+232 -21
View File
@@ -1,11 +1,16 @@
import { mkdir, readFile, rename as renameFilesystemPath, rm, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { SALT_OF_PASSPHRASE } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { encryptString } from "@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption";
import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings";
import { CENTRAL_COMPATIBILITY_REJECTION_REASONS } from "@vrtmrz/livesync-commonlib/replication";
import { evalObsidianJson } from "../runner/cli.ts";
import {
assertCouchDbReachable,
createCouchDbDatabase,
deleteCouchDbDatabase,
fetchAllCouchDbDocs,
fetchCouchDbLocalDocs,
loadCouchDbConfig,
makeUniqueDatabaseName,
waitForCouchDbDocs,
@@ -18,6 +23,7 @@ import {
assertE2eCompatibilityUnpaused,
configureCouchDb,
createE2eCouchDbPluginData,
createE2eObsidianDeviceLocalState,
prepareRemote,
pushLocalChanges,
waitForLiveSyncCoreReady,
@@ -439,7 +445,8 @@ async function renameNoteViaObsidian(cliBinary: string, env: NodeJS.ProcessEnv,
async function startConfiguredSession(
context: RunnerContext,
vault: TemporaryVault,
overrides: Record<string, unknown> = {}
overrides: Record<string, unknown> = {},
persistedOverrides: Record<string, unknown> = overrides
): Promise<ObsidianLiveSyncSession> {
const couchDbSettings = {
uri: context.couchDb.uri,
@@ -452,7 +459,7 @@ async function startConfiguredSession(
cliBinary: context.cliBinary,
vault,
startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
pluginData: createE2eCouchDbPluginData(couchDbSettings, overrides),
pluginData: createE2eCouchDbPluginData(couchDbSettings, persistedOverrides),
});
context.activeSessions.add(session);
try {
@@ -961,6 +968,193 @@ async function runEncryptedRoundTrip(
console.log("Two-vault encrypted note synchronisation round-tripped.");
}
async function runIndependentIdRoundTrip(
context: RunnerContext,
vaultA: TemporaryVault,
vaultB: TemporaryVault
): Promise<void> {
const source = "real-obsidian-e2e-independent-id-source";
const key = await deriveIdKey(source);
const content = "# Shared content with an independent ID key.\n";
const pathA = "E2E/independent-ids/from-a.md";
const pathB = "E2E/independent-ids/from-b.md";
const overrides = {
encrypt: true,
passphrase: "real-obsidian-e2e-independent-passphrase",
usePathObfuscation: true,
E2EEAlgorithm: "v2",
idDerivationVersion: 1,
idDerivationKey: key,
};
const persistedOverrides = {
...overrides,
idDerivationKey: "",
encryptedIdDerivationKey: await encryptString(key, `*${SALT_OF_PASSPHRASE}`),
};
let session = await startConfiguredSession(context, vaultA, overrides, persistedOverrides);
await writeNoteViaObsidian(context.cliBinary, session.cliEnv, pathA, content);
const entryA = await uploadNote(context, session, pathA);
if (entryA.children.length === 0) throw new Error("Independent ID mode produced no Chunks.");
await stopTrackedSession(context, session);
session = await startConfiguredSession(context, vaultB, overrides, persistedOverrides);
await syncAndApply(context, session);
await waitForPathContent(vaultB.path, pathA, (received) => received === content);
const receivedA = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, pathA);
assertEqual(receivedA.id, entryA.id, "The second device did not preserve the obfuscated document ID.");
await writeNoteViaObsidian(context.cliBinary, session.cliEnv, pathB, content);
const entryB = await uploadNote(context, session, pathB);
assertEqual(
JSON.stringify(entryB.children),
JSON.stringify(entryA.children),
"The second device did not reuse the same content-derived Chunk IDs."
);
await stopTrackedSession(context, session);
session = await startConfiguredSession(context, vaultA, overrides, persistedOverrides);
await syncAndApply(context, session);
await waitForPathContent(vaultA.path, pathB, (received) => received === content);
const receivedB = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, pathB);
assertEqual(receivedB.id, entryB.id, "The first device did not preserve the return document ID.");
await stopTrackedSession(context, session);
console.log("Two real Obsidian devices shared independent document and Chunk IDs in both directions.");
for (const [label, candidateKey] of [
["different key", "cd".repeat(32)],
["legacy IDs", ""],
] as const) {
const remoteBefore = await fetchAllCouchDbDocs(context.couchDb, context.dbName);
const checkpointsBefore = await fetchCouchDbLocalDocs(context.couchDb, context.dbName);
const rejectedVault = await createTemporaryVault();
let rejectedSession: ObsidianLiveSyncSession | undefined;
try {
rejectedSession = await startObsidianLiveSyncSession({
binary: context.binary,
cliBinary: context.cliBinary,
vault: rejectedVault,
localStorageEntries: createE2eObsidianDeviceLocalState(rejectedVault.name),
pluginData: createE2eCouchDbPluginData(
{ ...context.couchDb, dbName: context.dbName },
{
...overrides,
idDerivationVersion: candidateKey ? 1 : 0,
idDerivationKey: "",
encryptedIdDerivationKey: candidateKey
? await encryptString(candidateKey, `*${SALT_OF_PASSPHRASE}`)
: "",
}
),
});
context.activeSessions.add(rejectedSession);
await waitForLiveSyncCoreReady(context.cliBinary, rejectedSession.cliEnv);
const unsentPath = "E2E/independent-ids/rejected.md";
await writeNoteViaObsidian(context.cliBinary, rejectedSession.cliEnv, unsentPath, content);
await waitForLocalDatabaseEntry(context.cliBinary, rejectedSession.cliEnv, unsentPath);
const attempt = await evalObsidianJson<{ admitted: boolean; reason: string; replicated: boolean }>(
context.cliBinary,
[
"(async()=>{",
"const core=app.plugins.plugins['obsidian-livesync'].core;",
"const replicator=core.services.replicator.getActiveReplicator();",
"const settings=core.services.setting.currentSettings();",
"let reason='';",
"const connection=await replicator.checkReplicationConnectivity(settings,false,false,false,false,undefined,(decision)=>{reason=decision.reason??'';});",
"if(connection) await connection.close();",
"const replicated=await core.services.replication.replicate(true);",
"return JSON.stringify({admitted:!!connection,reason,replicated:!!replicated});",
"})()",
].join(""),
rejectedSession.cliEnv
);
assertEqual(attempt.admitted, false, `CouchDB admitted ${label} for obfuscated document IDs.`);
assertEqual(
attempt.reason,
CENTRAL_COMPATIBILITY_REJECTION_REASONS.ID_DERIVATION_MISMATCH,
`CouchDB rejected ${label} for an unrelated reason.`
);
assertEqual(attempt.replicated, false, `Ordinary replication accepted ${label}.`);
assertEqual(
await pathExists(rejectedVault.path, pathA),
false,
"A rejected device received a remote note."
);
await stopTrackedSession(context, rejectedSession);
rejectedSession = undefined;
assertEqual(
JSON.stringify(await fetchAllCouchDbDocs(context.couchDb, context.dbName)),
JSON.stringify(remoteBefore),
`A rejected ${label} connection changed remote documents.`
);
assertEqual(
JSON.stringify(await fetchCouchDbLocalDocs(context.couchDb, context.dbName)),
JSON.stringify(checkpointsBefore),
`A rejected ${label} connection changed remote checkpoints.`
);
} finally {
if (rejectedSession) await stopTrackedSession(context, rejectedSession);
await rejectedVault.dispose();
}
}
console.log("Ordinary CouchDB replication rejected different and legacy document ID keys without remote writes.");
}
async function runDifferentChunkIdKeysRoundTrip(
context: RunnerContext,
vaultA: TemporaryVault,
vaultB: TemporaryVault
): Promise<void> {
const keyA = await deriveIdKey("real-obsidian-e2e-chunk-source-a");
const keyB = await deriveIdKey("real-obsidian-e2e-chunk-source-b");
const content = "# Shared content with different Chunk ID keys.\n";
const pathA = "E2E/chunk-id-keys/from-a.md";
const pathB = "E2E/chunk-id-keys/from-b.md";
const commonSettings = {
encrypt: true,
passphrase: "real-obsidian-e2e-chunk-passphrase",
usePathObfuscation: false,
E2EEAlgorithm: "v2",
idDerivationVersion: 1,
};
const settingsFor = (key: string) => ({ ...commonSettings, idDerivationKey: key });
const persistedSettingsFor = async (key: string) => ({
...settingsFor(key),
idDerivationKey: "",
encryptedIdDerivationKey: await encryptString(key, `*${SALT_OF_PASSPHRASE}`),
});
const persistedA = await persistedSettingsFor(keyA);
const persistedB = await persistedSettingsFor(keyB);
let session = await startConfiguredSession(context, vaultA, settingsFor(keyA), persistedA);
await writeNoteViaObsidian(context.cliBinary, session.cliEnv, pathA, content);
const entryA = await uploadNote(context, session, pathA);
if (entryA.children.length === 0) throw new Error("The first device produced no Chunks.");
await stopTrackedSession(context, session);
session = await startConfiguredSession(context, vaultB, settingsFor(keyB), persistedB);
await syncAndApply(context, session);
await waitForPathContent(vaultB.path, pathA, (received) => received === content);
const receivedA = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, pathA);
assertEqual(receivedA.id, entryA.id, "The second device changed the visible document ID.");
await writeNoteViaObsidian(context.cliBinary, session.cliEnv, pathB, content);
const entryB = await uploadNote(context, session, pathB);
if (entryB.children.length === 0) throw new Error("The second device produced no Chunks.");
if (JSON.stringify(entryB.children) === JSON.stringify(entryA.children)) {
throw new Error("Different ID keys unexpectedly generated the same Chunk IDs.");
}
await stopTrackedSession(context, session);
session = await startConfiguredSession(context, vaultA, settingsFor(keyA), persistedA);
await syncAndApply(context, session);
await waitForPathContent(vaultA.path, pathB, (received) => received === content);
const receivedB = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, pathB);
assertEqual(receivedB.id, entryB.id, "The first device changed the return document ID.");
await stopTrackedSession(context, session);
console.log("Two real Obsidian devices exchanged notes with different Chunk ID keys and visible document paths.");
}
async function runMarkdownAutoMerge(
context: RunnerContext,
vaultA: TemporaryVault,
@@ -1085,10 +1279,9 @@ async function runConflictTimeStorageOperations(
showMergeDialogOnlyOnActive: true,
handleFilenameCaseSensitive: false,
};
const baseContent = Object.fromEntries(paths.map((path) => [path, `# Conflict operation\n\nBase for ${path}.\n`])) as Record<
(typeof paths)[number],
string
>;
const baseContent = Object.fromEntries(
paths.map((path) => [path, `# Conflict operation\n\nBase for ${path}.\n`])
) as Record<(typeof paths)[number], string>;
const leftContent = Object.fromEntries(
paths.map((path) => [path, `${baseContent[path]}\nEdit made on Vault A.\n`])
) as Record<(typeof paths)[number], string>;
@@ -1129,7 +1322,9 @@ async function runConflictTimeStorageOperations(
const initialBranchRevisions = new Map<string, Set<string>>();
for (const path of paths) {
const state = await waitForFileConflict(context.cliBinary, session.cliEnv, path);
const displayedBranch = state.branches.find((branch) => branch.content === rightContent[path] && !branch.deleted);
const displayedBranch = state.branches.find(
(branch) => branch.content === rightContent[path] && !branch.deleted
);
if (!displayedBranch) {
throw new Error(`Could not identify the branch displayed by Vault B: ${path}; ${JSON.stringify(state)}`);
}
@@ -1170,12 +1365,7 @@ async function runConflictTimeStorageOperations(
"A conflict-time deletion did not extend the displayed revision."
);
await renameNoteViaObsidian(
context.cliBinary,
session.cliEnv,
conflictCaseFromPath,
conflictCaseToPath
);
await renameNoteViaObsidian(context.cliBinary, session.cliEnv, conflictCaseFromPath, conflictCaseToPath);
const caseRenamedBranch = await waitForConflictBranch(
context.cliBinary,
session.cliEnv,
@@ -1216,12 +1406,7 @@ async function runConflictTimeStorageOperations(
"A conflict-time case-only rename did not record the new displayed revision."
);
await renameNoteViaObsidian(
context.cliBinary,
session.cliEnv,
conflictRenameFromPath,
conflictRenameToPath
);
await renameNoteViaObsidian(context.cliBinary, session.cliEnv, conflictRenameFromPath, conflictRenameToPath);
const renamedTarget = await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, conflictRenameToPath);
const renamedSourceDeletion = await waitForConflictBranch(
context.cliBinary,
@@ -1367,10 +1552,13 @@ async function main(): Promise<void> {
const couchDb = await loadCouchDbConfig();
const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "two-vault-sync");
const encryptedDbName = makeUniqueDatabaseName(couchDb.dbPrefix, "two-vault-sync-e2ee");
const independentDbName = makeUniqueDatabaseName(couchDb.dbPrefix, "two-vault-sync-independent-ids");
const vaultA = await createTemporaryVault();
const vaultB = await createTemporaryVault();
const encryptedVaultA = await createTemporaryVault();
const encryptedVaultB = await createTemporaryVault();
const independentVaultA = await createTemporaryVault();
const independentVaultB = await createTemporaryVault();
const context: RunnerContext = {
binary,
cliBinary: cli.binary,
@@ -1385,11 +1573,19 @@ async function main(): Promise<void> {
dbName: encryptedDbName,
activeSessions: new Set(),
};
const independentContext: RunnerContext = {
binary,
cliBinary: cli.binary,
couchDb,
dbName: independentDbName,
activeSessions: new Set(),
};
try {
await assertCouchDbReachable(couchDb);
await createCouchDbDatabase(couchDb, dbName);
await createCouchDbDatabase(couchDb, encryptedDbName);
await createCouchDbDatabase(couchDb, independentDbName);
console.log(`Using Obsidian executable: ${binary}`);
console.log(`Temporary vault A: ${vaultA.path}`);
@@ -1398,11 +1594,13 @@ async function main(): Promise<void> {
console.log(`Temporary encrypted CouchDB database: ${encryptedDbName}`);
const onlyParentCaseDeletion = process.env.E2E_OBSIDIAN_ONLY_PARENT_CASE_DELETION === "true";
const onlyIndependentIds = process.env.E2E_OBSIDIAN_ONLY_INDEPENDENT_IDS === "true";
const onlyDifferentChunkIdKeys = process.env.E2E_OBSIDIAN_ONLY_DIFFERENT_CHUNK_ID_KEYS === "true";
if (onlyParentCaseDeletion) {
await runParentCaseDeletionProtection(context, vaultA, vaultB);
}
const onlyConflictOperations = process.env.E2E_OBSIDIAN_ONLY_CONFLICT_OPERATIONS === "true";
if (!onlyParentCaseDeletion && !onlyConflictOperations) {
if (!onlyParentCaseDeletion && !onlyConflictOperations && !onlyIndependentIds && !onlyDifferentChunkIdKeys) {
await runCreateUpdateDelete(context, vaultA, vaultB);
await runRename(context, vaultA, vaultB);
await runCaseOnlyRename(context, vaultA, vaultB);
@@ -1416,17 +1614,27 @@ async function main(): Promise<void> {
) {
await runConflictTimeStorageOperations(context, vaultA, vaultB);
}
if (!onlyParentCaseDeletion && !onlyConflictOperations) {
if (!onlyParentCaseDeletion && !onlyConflictOperations && !onlyIndependentIds && !onlyDifferentChunkIdKeys) {
await runTargetMismatch(context, vaultA, vaultB);
await runEncryptedRoundTrip(encryptedContext, encryptedVaultA, encryptedVaultB);
}
if (!onlyParentCaseDeletion && !onlyConflictOperations) {
if (onlyDifferentChunkIdKeys) {
await runDifferentChunkIdKeysRoundTrip(independentContext, independentVaultA, independentVaultB);
} else {
await runIndependentIdRoundTrip(independentContext, independentVaultA, independentVaultB);
}
}
} finally {
await stopTrackedSessions(context);
await stopTrackedSessions(encryptedContext);
await stopTrackedSessions(independentContext);
await vaultA.dispose();
await vaultB.dispose();
await encryptedVaultA.dispose();
await encryptedVaultB.dispose();
await independentVaultA.dispose();
await independentVaultB.dispose();
if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => {
console.warn(error instanceof Error ? error.message : error);
@@ -1434,6 +1642,9 @@ async function main(): Promise<void> {
await deleteCouchDbDatabase(couchDb, encryptedDbName).catch((error: unknown) => {
console.warn(error instanceof Error ? error.message : error);
});
await deleteCouchDbDatabase(couchDb, independentDbName).catch((error: unknown) => {
console.warn(error instanceof Error ? error.message : error);
});
}
}
}