From 0890a972220c82691ec38e57e53a76a61dccb223 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sat, 1 Aug 2026 03:52:57 +0000 Subject: [PATCH] test(obsidian): cover adaptive S3 journal upload --- package.json | 1 + test/e2e-obsidian/README.md | 4 +- test/e2e-obsidian/runner/liveSyncWorkflow.ts | 6 ++ test/e2e-obsidian/scripts/local-suite.ts | 1 + test/e2e-obsidian/scripts/minio-upload.ts | 100 ++++++++++++++++--- test/e2e-obsidian/scripts/run-focused.ts | 1 + 6 files changed, 96 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index e88fd61a..08549a82 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "test:e2e:obsidian:couchdb-manual-setup-workflow": "tsx test/e2e-obsidian/scripts/couchdb-manual-setup-workflow.ts", "test:e2e:obsidian:cli-to-obsidian-sync": "tsx test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts", "test:e2e:obsidian:minio-upload": "tsx test/e2e-obsidian/scripts/minio-upload.ts", + "test:e2e:obsidian:adaptive-s3": "tsx test/e2e-obsidian/scripts/minio-upload.ts --adaptive", "test:e2e:obsidian:object-storage-setup-uri-workflow": "tsx test/e2e-obsidian/scripts/object-storage-setup-uri-workflow.ts", "test:e2e:obsidian:p2p-setup-uri-workflow": "tsx test/e2e-obsidian/scripts/p2p-setup-uri-workflow.ts", "pretest:e2e:obsidian:p2p-connection-check": "npm run build && npm run build --workspace webpeer", diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 864da52f..9af96072 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -114,7 +114,7 @@ The mobile pass uses Obsidian's `app.emulateMobile(true)`, a 390 by 844 CSS-pixe `test:e2e:obsidian:p2p-pane` starts one configured CouchDB-only session with no P2P profile and separate configured P2P sessions for desktop and mobile. It proves that the command remains registered while the retired command, automatic pane, and ribbon entry without a P2P configuration are absent. For the configured P2P profiles, it verifies that the desktop ribbon is available, the current status command reaches the pane without it opening at start-up, checks its connection control and horizontal layout, and captures unobstructed desktop and mobile screenshots. The mobile session uses a fresh Vault, profile, and Obsidian process, enters `app.emulateMobile(true)` through `lifecycle.beforePluginStart`, and requires the P2P view to belong to the right drawer rather than inheriting desktop workspace state. It deliberately uses no relay or peer: replacement of the active replicator is covered by focused unit tests, the Deno and Compose CLI P2P lifecycle suite covers the headless transport, and `p2p-setup-uri-workflow` owns the visible transfer path between two real Obsidian sessions. -`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Object Storage upload and Setup URI round-trip, P2P Setup URI round-trip, startup scan, provisioned CouchDB Setup URI, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB, MinIO, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run. +`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Opaque and Adaptive Object Storage uploads, the Object Storage Setup URI round-trip, the P2P Setup URI round-trip, startup scan, the provisioned CouchDB Setup URI, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB, MinIO, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run. `test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, starts from configured plug-in data without the device-local compatibility marker, and verifies the copied-or-restored Vault explanation in the actual compatibility dialogue. It captures the summary and details, resumes explicitly, confirms that the marker was recorded, creates a note in real Obsidian, commits the note into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the metadata document and its chunk documents. @@ -147,6 +147,8 @@ LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) -- `test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service and starts with isolated Object Storage settings and the device-local compatibility acknowledgement already in place, keeping the scenario focused on upload rather than unconfigured start-up or setup. It confirms those settings through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. Adapter tests separately observe an in-progress SDK command, while this real-runtime workflow verifies the resulting request counters advance and rebalance. +`test:e2e:obsidian:adaptive-s3` reuses that one-device upload workflow with the experimental Adaptive Journal format and Range retrieval selected explicitly. It requires real Obsidian to retain those settings, publish a Chunk-backed note through one-shot synchronisation, and produce an authenticated manifest, a writer record, and an immutable Commit Bundle under a disposable MinIO prefix without writing the Opaque Journal milestone. Commonlib tests own the storage protocol and failure classification, while the CLI E2E owns bidirectional synchronisation, external Packs, and both Pack retrieval policies; this scenario verifies only the Obsidian composition boundary. + `test:e2e:obsidian:object-storage-setup-uri-workflow` uses the public Commonlib-backed tool to generate the initial Setup URI for a unique MinIO prefix, completes visible initialisation on the first device, and then asks that working real Obsidian device to create a new Setup URI through the registered command. A second real Obsidian device imports only the device-generated URI. The workflow verifies A-to-B and B-to-A notes, captures the documented onboarding choices, and removes the Object Storage prefix only after both sessions have stopped. `test:e2e:obsidian:p2p-setup-uri-workflow` runs two concurrent isolated real Obsidian sessions against the local Compose Nostr relay fixture. The first device imports a generated initial Setup URI and completes its signalling test with zero peers, creates a Setup URI for the second device through the registered command, and remains online while the second device imports it. The second device must select the expected online source before Fetch can rebuild its local database. The workflow accepts each connection request visibly on the receiving device, verifies the initial A-to-B fetch, checks that the menu for the three persistent per-peer actions remains within the viewport, reconnects both P2P sessions in join order, and verifies the B-to-A return journey. Every started session remains tracked until teardown completes. diff --git a/test/e2e-obsidian/runner/liveSyncWorkflow.ts b/test/e2e-obsidian/runner/liveSyncWorkflow.ts index 896fb058..fedd230d 100644 --- a/test/e2e-obsidian/runner/liveSyncWorkflow.ts +++ b/test/e2e-obsidian/runner/liveSyncWorkflow.ts @@ -19,6 +19,9 @@ export type ConfiguredSettings = { endpoint?: string; bucket?: string; bucketPrefix?: string; + expectedRepositoryId?: string; + journalFormat?: string; + packReadPolicy?: string; }; export type CoreReadiness = { @@ -320,6 +323,9 @@ export async function configureObjectStorage( "endpoint:current.endpoint,", "bucket:current.bucket,", "bucketPrefix:current.bucketPrefix,", + "expectedRepositoryId:current.expectedRepositoryId,", + "journalFormat:current.journalFormat,", + "packReadPolicy:current.packReadPolicy,", "});", "})()", ].join(""), diff --git a/test/e2e-obsidian/scripts/local-suite.ts b/test/e2e-obsidian/scripts/local-suite.ts index 9dfc225f..4fb16a38 100644 --- a/test/e2e-obsidian/scripts/local-suite.ts +++ b/test/e2e-obsidian/scripts/local-suite.ts @@ -30,6 +30,7 @@ const testSteps: Step[] = [ args: ["run", "test:e2e:obsidian:cli-to-obsidian-sync"], }, { name: "Object Storage upload", args: ["run", "test:e2e:obsidian:minio-upload"] }, + { name: "Adaptive S3 upload", args: ["run", "test:e2e:obsidian:adaptive-s3"] }, { name: "Object Storage Setup URI workflow", args: ["run", "test:e2e:obsidian:object-storage-setup-uri-workflow"], diff --git a/test/e2e-obsidian/scripts/minio-upload.ts b/test/e2e-obsidian/scripts/minio-upload.ts index b0899f12..08aadbfb 100644 --- a/test/e2e-obsidian/scripts/minio-upload.ts +++ b/test/e2e-obsidian/scripts/minio-upload.ts @@ -33,6 +33,7 @@ import { listObjectStorageObjects, loadObjectStorageConfig, makeUniqueBucketPrefix, + readObjectStorageJson, } from "../runner/objectStorage.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { createTemporaryVault } from "../runner/vault.ts"; @@ -40,9 +41,21 @@ import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../r process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000"; -const notePath = "E2E/minio-upload.md"; +const adaptive = process.argv.includes("--adaptive"); +const unsupportedArguments = process.argv.slice(2).filter((argument) => argument !== "--adaptive"); +if (unsupportedArguments.length > 0) { + throw new Error(`Unsupported Object Storage upload argument: ${unsupportedArguments.join(", ")}`); +} +const journalSettings = adaptive + ? { + journalFormat: "adaptive-v1", + packReadPolicy: "range", + } + : {}; +const scenarioName = adaptive ? "Adaptive S3" : "Object Storage"; +const notePath = adaptive ? "E2E/adaptive-s3-upload.md" : "E2E/minio-upload.md"; const noteContent = [ - "# Object Storage upload from real Obsidian", + `# ${scenarioName} upload from real Obsidian`, "", "This note is created through Obsidian and uploaded by Self-hosted LiveSync to S3-compatible Object Storage.", "The test is intentionally small, but it crosses the real Obsidian, Journal Sync, and AWS SDK boundary.", @@ -78,7 +91,7 @@ async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.Proces ); } -async function waitForObjectStorageObjects(prefix: string): Promise { +async function waitForObjectStorageObjects(prefix: string, requiredKeyPrefix?: string): Promise { const objectStorage = await loadObjectStorageConfig(); const timeoutMs = Number(process.env.E2E_OBSIDIAN_OBJECT_STORAGE_TIMEOUT_MS ?? 20000); const deadline = Date.now() + timeoutMs; @@ -86,12 +99,48 @@ async function waitForObjectStorageObjects(prefix: string): Promise { while (Date.now() < deadline) { const objects = await listObjectStorageObjects(objectStorage, prefix); keys = objects.flatMap((object) => (object.Key ? [object.Key] : [])); - if (keys.length > 0) { + if (keys.length > 0 && (!requiredKeyPrefix || keys.some((key) => key.startsWith(requiredKeyPrefix)))) { return keys; } await new Promise((resolve) => setTimeout(resolve, 500)); } - throw new Error(`Timed out waiting for Object Storage objects under ${prefix}. Last keys: ${keys.join(", ")}`); + throw new Error( + `Timed out waiting for Object Storage objects under ${prefix}${requiredKeyPrefix ? ` with prefix ${requiredKeyPrefix}` : ""}. Last keys: ${keys.join(", ")}` + ); +} + +async function assertAdaptiveObjects(prefix: string, keys: string[]): Promise { + const objectStorage = await loadObjectStorageConfig(); + const manifestKey = `${prefix}a1~manifest.json`; + const requiredPrefixes = [`${prefix}a1~writer~`, `${prefix}a1~commit~`]; + if (!keys.includes(manifestKey)) { + throw new Error(`Adaptive Journal manifest is missing. Keys: ${keys.join(", ")}`); + } + for (const requiredPrefix of requiredPrefixes) { + if (!keys.some((key) => key.startsWith(requiredPrefix))) { + throw new Error(`Adaptive Journal object prefix ${requiredPrefix} is missing. Keys: ${keys.join(", ")}`); + } + } + if (keys.includes(`${prefix}_00000000-milestone.json`)) { + throw new Error("Adaptive Journal wrote the legacy Opaque Journal milestone."); + } + + const manifest = await readObjectStorageJson<{ + format?: unknown; + formatVersion?: unknown; + manifestAuth?: unknown; + objectLayout?: unknown; + repositoryId?: unknown; + }>(objectStorage, manifestKey); + assertEqual(manifest.format, "adaptive-journal", "Unexpected Adaptive Journal manifest format."); + assertEqual(manifest.formatVersion, 1, "Unexpected Adaptive Journal manifest version."); + assertEqual(manifest.objectLayout, "commit-bundle-v1", "Unexpected Adaptive Journal object layout."); + if (typeof manifest.repositoryId !== "string" || manifest.repositoryId.length === 0) { + throw new Error("Adaptive Journal manifest did not contain a repository ID."); + } + if (typeof manifest.manifestAuth !== "string" || manifest.manifestAuth.length === 0) { + throw new Error("Adaptive Journal manifest did not contain its authentication value."); + } } async function main(): Promise { @@ -102,7 +151,7 @@ async function main(): Promise { } const objectStorage = await loadObjectStorageConfig(); - const bucketPrefix = makeUniqueBucketPrefix("minio-upload"); + const bucketPrefix = makeUniqueBucketPrefix(adaptive ? "adaptive-s3-upload" : "minio-upload"); const vault = await createTemporaryVault(); let session: ObsidianLiveSyncSession | undefined; @@ -119,18 +168,26 @@ async function main(): Promise { cliBinary: cli.binary, vault, startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), - pluginData: createE2eObjectStoragePluginData({ - ...objectStorage, - bucketPrefix, - }), + pluginData: createE2eObjectStoragePluginData( + { + ...objectStorage, + bucketPrefix, + }, + journalSettings + ), localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), }); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); - const configured = await configureObjectStorage(cli.binary, session.cliEnv, { - ...objectStorage, - bucketPrefix, - }); + const configured = await configureObjectStorage( + cli.binary, + session.cliEnv, + { + ...objectStorage, + bucketPrefix, + }, + journalSettings + ); await waitForLiveSyncCoreReady(cli.binary, session.cliEnv); assertEqual(configured.isConfigured, true, "Self-hosted LiveSync was not marked as configured."); assertEqual(configured.remoteType, "MINIO", "Remote type was not Object Storage."); @@ -138,6 +195,11 @@ async function main(): Promise { assertEqual(configured.bucket, objectStorage.bucket, "Configured Object Storage bucket did not match."); assertEqual(configured.bucketPrefix, bucketPrefix, "Configured Object Storage bucket prefix did not match."); assertEqual(configured.liveSync, false, "LiveSync should remain disabled during this one-shot workflow."); + if (adaptive) { + assertEqual(configured.journalFormat, "adaptive-v1", "Adaptive Journal format was not retained."); + assertEqual(configured.packReadPolicy, "range", "Adaptive Journal Pack retrieval was not retained."); + assertEqual(configured.expectedRepositoryId, "", "A new Adaptive repository should not be pre-bound."); + } await prepareRemote(cli.binary, session.cliEnv); const activityBeforeUpload = await waitForRemoteActivityState( @@ -159,10 +221,16 @@ async function main(): Promise { "Object Storage remote-request counters did not rebalance after synchronisation." ); - const keys = await waitForObjectStorageObjects(bucketPrefix); + const keys = await waitForObjectStorageObjects( + bucketPrefix, + adaptive ? `${bucketPrefix}a1~commit~` : undefined + ); + if (adaptive) { + await assertAdaptiveObjects(bucketPrefix, keys); + } console.log( - `Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}` + `Uploaded ${localEntry.path} through ${scenarioName} Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}` ); } finally { if (session) { diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index 957d6fd8..5e4da85a 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -18,6 +18,7 @@ const focusedScenarios = new Set([ "couchdb-manual-setup-workflow", "cli-to-obsidian-sync", "minio-upload", + "adaptive-s3", "object-storage-setup-uri-workflow", "p2p-setup-uri-workflow", "startup-scan",