diff --git a/package.json b/package.json index 5f2074ca..151ccd01 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "test:contract:context:obsidian": "npm run build && npm run test:e2e:obsidian:smoke", "test:e2e:cli": "npm run test:e2e:ci --workspace self-hosted-livesync-cli", "test:e2e:cli:p2p": "npm run test:e2e:p2p --workspace self-hosted-livesync-cli", + "test:e2e:cli:adaptive-s3": "npm run test:e2e:adaptive-s3 --workspace self-hosted-livesync-cli", "test:e2e:cli:all": "npm run test:e2e:all --workspace self-hosted-livesync-cli", "test:integration": "npx dotenv-cli -e .env -e .test.env -- vitest run --config vitest.config.integration.ts", "test:unit:coverage": "vitest run --config vitest.config.unit.ts --coverage", diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 5e027a66..dbfbe740 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -22,6 +22,8 @@ "pretest:e2e:ci": "npm run build", "test:e2e:ci": "deno task --cwd testdeno test:ci", "test:e2e:p2p": "deno task --cwd testdeno test:p2p:compose", + "pretest:e2e:adaptive-s3": "npm run build", + "test:e2e:adaptive-s3": "deno task --cwd testdeno test:adaptive-journal-s3", "test:e2e:mirror": "bash test/test-mirror-linux.sh", "test:e2e:remote-commands": "bash test/test-remote-commands-linux.sh", "pretest:e2e:all": "npm run build", diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json index 7cb8a80d..62be474f 100644 --- a/src/apps/cli/testdeno/deno.json +++ b/src/apps/cli/testdeno/deno.json @@ -34,7 +34,8 @@ "test:e2e-matrix:couchdb-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc0' test-e2e-two-vaults-matrix.ts", "test:e2e-matrix:couchdb-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc1' test-e2e-two-vaults-matrix.ts", "test:e2e-matrix:minio-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc0' test-e2e-two-vaults-matrix.ts", - "test:e2e-matrix:minio-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc1' test-e2e-two-vaults-matrix.ts" + "test:e2e-matrix:minio-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc1' test-e2e-two-vaults-matrix.ts", + "test:adaptive-journal-s3": "deno test --env-file=.test.env -A --no-check test-adaptive-journal-s3.ts" }, "imports": { "@std/assert": "jsr:@std/assert@^1.0.13", diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/src/apps/cli/testdeno/helpers/docker.ts index 3c0e6ce1..c0188aaa 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/src/apps/cli/testdeno/helpers/docker.ts @@ -466,6 +466,43 @@ export async function stopMinio(): Promise { untrackContainer(MINIO_CONTAINER); } +export async function listMinioObjectKeys( + minioEndpoint: string, + accessKey: string, + secretKey: string, + bucket: string +): Promise { + const cmd = + `mc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` + + `mc ls --recursive --json myminio/${shQuote(bucket)}`; + const result = await docker( + "run", + "--rm", + "--network", + "host", + "--entrypoint", + "/bin/sh", + MINIO_MC_IMAGE, + "-c", + cmd + ); + if (result.code !== 0) { + throw new Error(`Could not list MinIO objects: ${result.stderr.trim()}`); + } + + return result.stdout + .split(/\r?\n/u) + .filter((line) => line.trim().length > 0) + .map((line) => JSON.parse(line) as { key?: unknown }) + .map(({ key }) => { + if (typeof key !== "string") { + throw new Error("MinIO returned an object without a string key"); + } + return key; + }) + .sort(); +} + async function initMinioBucket( minioEndpoint: string, accessKey: string, diff --git a/src/apps/cli/testdeno/helpers/settings.ts b/src/apps/cli/testdeno/helpers/settings.ts index 524622c0..c54b19d1 100644 --- a/src/apps/cli/testdeno/helpers/settings.ts +++ b/src/apps/cli/testdeno/helpers/settings.ts @@ -125,6 +125,9 @@ export async function applyRemoteSyncSettings( passphrase?: string; enableCompression?: boolean; usePathObfuscation?: boolean; + journalFormat?: "adaptive-v1" | "opaque-v1"; + expectedRepositoryId?: string; + packReadPolicy?: "range" | "whole-pack"; } ): Promise { const data = JSON.parse(await Deno.readTextFile(settingsFile)); @@ -143,6 +146,15 @@ export async function applyRemoteSyncSettings( data.secretKey = options.minioSecretKey; data.region = "auto"; data.forcePathStyle = true; + if (options.journalFormat !== undefined) { + data.journalFormat = options.journalFormat; + } + if (options.expectedRepositoryId !== undefined) { + data.expectedRepositoryId = options.expectedRepositoryId; + } + if (options.packReadPolicy !== undefined) { + data.packReadPolicy = options.packReadPolicy; + } } data.liveSync = true; diff --git a/src/apps/cli/testdeno/run-ci-suite.ts b/src/apps/cli/testdeno/run-ci-suite.ts index 7b5824ca..5c2a654d 100644 --- a/src/apps/cli/testdeno/run-ci-suite.ts +++ b/src/apps/cli/testdeno/run-ci-suite.ts @@ -11,6 +11,7 @@ const TASKS = [ "test:e2e-matrix:couchdb-enc1", "test:e2e-matrix:minio-enc0", "test:e2e-matrix:minio-enc1", + "test:adaptive-journal-s3", ] as const; for (const [index, task] of TASKS.entries()) { diff --git a/src/apps/cli/testdeno/test-adaptive-journal-s3.ts b/src/apps/cli/testdeno/test-adaptive-journal-s3.ts new file mode 100644 index 00000000..91df3fff --- /dev/null +++ b/src/apps/cli/testdeno/test-adaptive-journal-s3.ts @@ -0,0 +1,113 @@ +import { assert, assertEquals } from "@std/assert"; +import { TempDir } from "./helpers/temp.ts"; +import { assertFilesEqual, runCli, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; +import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; +import { listMinioObjectKeys, startMinio, stopMinio } from "./helpers/docker.ts"; + +function requireEnv(...keys: string[]): string { + for (const key of keys) { + const value = Deno.env.get(key)?.trim(); + if (value) return value; + } + throw new Error(`Required environment variable is missing: ${keys.join(" or ")}`); +} + +Deno.test("e2e: two CLI vaults synchronise through Adaptive Journal S3", async () => { + const suffix = `${Date.now()}-${Math.floor(Math.random() * 100000)}`; + const endpoint = requireEnv("MINIO_ENDPOINT", "minioEndpoint").replace(/\/$/u, ""); + const accessKey = requireEnv("MINIO_ACCESS_KEY", "accessKey"); + const secretKey = requireEnv("MINIO_SECRET_KEY", "secretKey"); + const bucket = `${requireEnv("MINIO_BUCKET_NAME", "bucketName")}-${suffix}`; + const passphrase = "adaptive-journal-cli-e2e-passphrase"; + + await using workDir = await TempDir.create("livesync-cli-adaptive-journal-s3"); + const vaultA = workDir.join("vault-a"); + const vaultB = workDir.join("vault-b"); + const settingsA = workDir.join("settings-a.json"); + const settingsB = workDir.join("settings-b.json"); + const binarySource = workDir.join("source.bin"); + const binaryDestination = workDir.join("destination.bin"); + await Deno.mkdir(vaultA, { recursive: true }); + await Deno.mkdir(vaultB, { recursive: true }); + + const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + await startMinio(endpoint, accessKey, secretKey, bucket); + + try { + await initSettingsFile(settingsA); + await initSettingsFile(settingsB); + await applyRemoteSyncSettings(settingsA, { + remoteType: "MINIO", + minioBucket: bucket, + minioEndpoint: endpoint, + minioAccessKey: accessKey, + minioSecretKey: secretKey, + encrypt: true, + passphrase, + journalFormat: "adaptive-v1", + packReadPolicy: "whole-pack", + }); + await applyRemoteSyncSettings(settingsB, { + remoteType: "MINIO", + minioBucket: bucket, + minioEndpoint: endpoint, + minioAccessKey: accessKey, + minioSecretKey: secretKey, + encrypt: true, + passphrase, + journalFormat: "adaptive-v1", + packReadPolicy: "range", + }); + + const textPath = "adaptive/text.md"; + const binaryPath = "adaptive/data.bin"; + await runCliWithInputOrFail(`created-by-a-${suffix}\n`, vaultA, "--settings", settingsA, "put", textPath); + const binary = Uint8Array.from({ length: 8192 }, (_, index) => (index * 31 + 17) % 256); + await Deno.writeFile(binarySource, binary); + await runCliOrFail(vaultA, "--settings", settingsA, "push", binarySource, binaryPath); + + await runCliOrFail(vaultA, "--settings", settingsA, "sync"); + await runCliOrFail(vaultB, "--settings", settingsB, "sync"); + assertEquals( + sanitiseCatStdout(await runCliOrFail(vaultB, "--settings", settingsB, "cat", textPath)).trimEnd(), + `created-by-a-${suffix}` + ); + await runCliOrFail(vaultB, "--settings", settingsB, "pull", binaryPath, binaryDestination); + await assertFilesEqual(binarySource, binaryDestination, "Adaptive Journal binary transfer differs"); + + await runCliWithInputOrFail(`updated-by-b-${suffix}\n`, vaultB, "--settings", settingsB, "put", textPath); + await runCliOrFail(vaultB, "--settings", settingsB, "sync"); + await runCliOrFail(vaultA, "--settings", settingsA, "sync"); + assertEquals( + sanitiseCatStdout(await runCliOrFail(vaultA, "--settings", settingsA, "cat", textPath)).trimEnd(), + `updated-by-b-${suffix}` + ); + + await runCliOrFail(vaultA, "--settings", settingsA, "rm", binaryPath); + await runCliOrFail(vaultA, "--settings", settingsA, "sync"); + await runCliOrFail(vaultB, "--settings", settingsB, "sync"); + const deleted = await runCli(vaultB, "--settings", settingsB, "cat", binaryPath); + assert(deleted.code !== 0, `Deleted binary remained readable:\n${deleted.combined}`); + + const objectKeys = await listMinioObjectKeys(endpoint, accessKey, secretKey, bucket); + assert(objectKeys.includes("a1~manifest.json"), `Adaptive manifest is missing:\n${objectKeys.join("\n")}`); + for (const prefix of ["a1~writer~", "a1~pack~", "a1~index~", "a1~delta~", "a1~metadata~", "a1~commit~"]) { + assert( + objectKeys.some((key) => key.startsWith(prefix)), + `Adaptive object with prefix ${prefix} is missing:\n${objectKeys.join("\n")}` + ); + } + assert( + !objectKeys.some((key) => key.startsWith("a1~probe~")), + `Adaptive capability probe objects were not removed:\n${objectKeys.join("\n")}` + ); + assert( + !objectKeys.includes("_00000000-milestone.json"), + `Legacy Journal milestone was written into the Adaptive repository:\n${objectKeys.join("\n")}` + ); + } finally { + if (!keepDocker) { + await stopMinio().catch(() => {}); + } + } +});