Run CLI database continuity regression in standard CI

test/e2e-cli/ci-tasks.json

@@ -15,5 +15,6 @@
     "test:e2e-matrix:couchdb-enc1",
     "test:e2e-matrix:minio-enc0",
     "test:e2e-matrix:minio-enc1",
-    "test:journal-chunk-recovery"
+    "test:journal-chunk-recovery",
+    "test:fetch-database-continuity"
 ]

test/e2e-cli/helpers/docker.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import { writeSync } from "node:fs";
 import { spawnProcess } from "./process.ts";
 /**
@@ -577,6 +578,59 @@ export async function startMinio(
     await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket);
 }
 
+export type IsolatedMinioFixture = {
+    endpoint: string;
+    accessKey: string;
+    secretKey: string;
+    stop: () => Promise<void>;
+};
+
+/** Start an S3 fixture owned by this caller without touching the shared test container. */
+export async function startIsolatedMinio(accessKey: string, secretKey: string): Promise<IsolatedMinioFixture> {
+    const container = `livesync-cli-continuity-${randomUUID()}`;
+    console.log(`[INFO] starting isolated S3 test container: ${container}`);
+    let tracked = false;
+    const stop = async () => {
+        await stopAndRemoveContainer(container);
+        if (tracked) untrackContainer(container);
+    };
+
+    try {
+        await dockerOrFail(
+            "run",
+            "-d",
+            "--name",
+            container,
+            "-p",
+            "127.0.0.1::9000",
+            "-e",
+            `RUSTFS_ACCESS_KEY=${accessKey}`,
+            "-e",
+            `RUSTFS_SECRET_KEY=${secretKey}`,
+            "-e",
+            "RUSTFS_CONSOLE_ENABLE=true",
+            "-e",
+            "RUSTFS_CORS_ALLOWED_ORIGINS=*",
+            S3_IMAGE,
+            "/data"
+        );
+        trackContainer(container);
+        tracked = true;
+
+        const mapping = await dockerOrFail("port", container, "9000/tcp");
+        const port = Number(mapping.trim().match(/:(\d+)$/)?.[1]);
+        if (!Number.isInteger(port) || port < 1 || port > 65535) {
+            throw new Error(`Could not determine the isolated S3 fixture port from: ${mapping.trim()}`);
+        }
+        const endpoint = `http://127.0.0.1:${port}`;
+        await fetchRetry(endpoint, { method: "GET" }, 30, 1000, [403, 404]);
+        return { endpoint, accessKey, secretKey, stop };
+    } catch (error) {
+        await stop();
+        throw error;
+    }
+}
+
 // ---------------------------------------------------------------------------
 // P2P relay (strfry)
 // ---------------------------------------------------------------------------

test/e2e-cli/tasks.json

@@ -1,7 +1,8 @@
 {
   "test:helpers": {
     "test": [
-      "helpers/process.test.ts"
+      "helpers/process.test.ts",
+      "test-ci-task-contract.ts"
     ]
   },
   "test": {

test/e2e-cli/test-ci-task-contract.ts

@@ -0,0 +1,18 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import { test } from "node:test";
+
+test("fetch database continuity is exercised by standard CLI CI", async () => {
+    const ciTasks = JSON.parse(await readFile(new URL("./ci-tasks.json", import.meta.url), "utf8")) as string[];
+    const tasks = JSON.parse(await readFile(new URL("./tasks.json", import.meta.url), "utf8")) as Record<
+        string,
+        { test?: string[] }
+    >;
+    const workflow = await readFile(new URL("../../.github/workflows/cli-node-tests.yml", import.meta.url), "utf8");
+
+    assert(ciTasks.includes("test:helpers"), "the standard CI suite must run the task registration contract");
+    assert(tasks["test:helpers"]?.test?.includes("test-ci-task-contract.ts"));
+    assert(ciTasks.includes("test:fetch-database-continuity"), "the continuity task must be in the standard CI suite");
+    assert.deepEqual(tasks["test:fetch-database-continuity"]?.test, ["test-fetch-database-continuity.ts"]);
+    assert.match(workflow, /test\/e2e-cli\/ci-tasks\.json/, "the automatic CI matrix must read the authoritative task list");
+});

test/e2e-cli/test-fetch-database-continuity.ts

@@ -1,9 +1,12 @@
 import assert from "node:assert/strict";
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
-import { test } from "node:test";
+import { after, before, test } from "node:test";
+import { CreateBucketCommand, S3Client } from "@aws-sdk/client-s3";
 import { CLI_DIR, CLI_DIST } from "./helpers/cli.ts";
+import { startIsolatedMinio, type IsolatedMinioFixture } from "./helpers/docker.ts";
 import { applyRemoteSyncSettings, generateSetupUriFromSettings, initSettingsFile } from "./helpers/settings.ts";
 import { TempDir } from "./helpers/temp.ts";
 import { spawnProcess, type RunningProcess } from "./helpers/process.ts";
@@ -24,20 +27,89 @@ const RESOLUTION_MARKER = "Resolving [same]: Conflicted revision has been delete
 const INITIAL_REPLICATION_COMPLETE = "[Daemon] Initial replication complete";
 const INITIAL_REPLICATION_FAILED = "[Daemon] Initial replication failed";
 const VERIFY_LEGACY_FAILURE = process.env["LIVESYNC_ISSUE1304_VERIFY_LEGACY_FAILURE"] === "1";
+const EXPLICIT_FIXTURE_VARIABLES = [
+    "LIVESYNC_ISSUE1304_MINIO_ENDPOINT",
+    "LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY",
+    "LIVESYNC_ISSUE1304_MINIO_SECRET_KEY",
+    "LIVESYNC_ISSUE1304_MINIO_BUCKET",
+];
+const EXPLICIT_FIXTURE = EXPLICIT_FIXTURE_VARIABLES.some((name) => process.env[name]?.trim());
 
-function requiredEnv(name: string): string {
-    const value = process.env[name]?.trim();
-    if (!value) throw new Error(`Required environment variable is missing: ${name}`);
-    return value;
+if (EXPLICIT_FIXTURE && EXPLICIT_FIXTURE_VARIABLES.some((name) => !process.env[name]?.trim())) {
+    throw new Error(`External fixture mode requires all of: ${EXPLICIT_FIXTURE_VARIABLES.join(", ")}`);
 }
 
-function remoteSettings(bucketOverride?: string): RemoteSettings {
+let managedFixture: IsolatedMinioFixture | undefined;
+if (!EXPLICIT_FIXTURE) {
+    before(async () => {
+        managedFixture = await startIsolatedMinio(
+            process.env["accessKey"]?.trim() ?? "minioadmin",
+            process.env["secretKey"]?.trim() ?? "minioadmin"
+        );
+    });
+    after(async () => {
+        await managedFixture?.stop();
+    });
+}
+
+function requiredEnv(...names: string[]): string {
+    for (const name of names) {
+        const value = process.env[name]?.trim();
+        if (value) return value;
+    }
+    throw new Error(`Required environment variable is missing: ${names.join(" or ")}`);
+}
+
+function evidenceDirectory(): string {
+    return resolve(
+        process.env["LIVESYNC_ISSUE1304_EVIDENCE_DIR"]?.trim() || join(tmpdir(), "livesync-cli-issue1304-evidence")
+    );
+}
+
+function remoteSettings(): RemoteSettings {
     return {
         endpoint: requiredEnv("LIVESYNC_ISSUE1304_MINIO_ENDPOINT").replace(/\/$/, ""),
         accessKey: requiredEnv("LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY"),
         secretKey: requiredEnv("LIVESYNC_ISSUE1304_MINIO_SECRET_KEY"),
-        bucket: bucketOverride ?? requiredEnv("LIVESYNC_ISSUE1304_MINIO_BUCKET"),
-        evidenceDir: resolve(requiredEnv("LIVESYNC_ISSUE1304_EVIDENCE_DIR")),
+        bucket: requiredEnv("LIVESYNC_ISSUE1304_MINIO_BUCKET"),
+        evidenceDir: evidenceDirectory(),
+    };
+}
+
+async function createManagedBucket(remote: Pick<RemoteSettings, "endpoint" | "accessKey" | "secretKey">, mode: ScenarioMode): Promise<string> {
+    const shortMode = mode === "reproduction" ? "repro" : "control";
+    const bucket = `livesync-ci-1304-${shortMode}-${randomUUID().replaceAll("-", "")}`;
+    const client = new S3Client({
+        endpoint: remote.endpoint,
+        region: "us-east-1",
+        forcePathStyle: true,
+        credentials: { accessKeyId: remote.accessKey, secretAccessKey: remote.secretKey },
+    });
+    try {
+        await client.send(new CreateBucketCommand({ Bucket: bucket }));
+    } finally {
+        client.destroy();
+    }
+    return bucket;
+}
+
+async function scenarioRemoteSettings(mode: ScenarioMode): Promise<RemoteSettings> {
+    if (EXPLICIT_FIXTURE) {
+        const remote = remoteSettings();
+        if (mode === "write-settings-control") remote.bucket = requiredEnv("LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET");
+        return remote;
+    }
+
+    if (!managedFixture) throw new Error("The managed S3 fixture was not started");
+    const remote = {
+        endpoint: managedFixture.endpoint,
+        accessKey: managedFixture.accessKey,
+        secretKey: managedFixture.secretKey,
+    };
+    return {
+        ...remote,
+        bucket: await createManagedBucket(remote, mode),
+        evidenceDir: evidenceDirectory(),
     };
 }
 
@@ -265,8 +337,7 @@ async function settingsSummary(settingsPath: string): Promise<Record<string, unk
     }
 }
 
-async function runScenario(mode: ScenarioMode, bucket: string): Promise<void> {
-    const remote = remoteSettings(bucket);
+async function runScenario(mode: ScenarioMode, remote: RemoteSettings): Promise<void> {
     const evidenceRoot = remote.evidenceDir;
     const evidenceName = `${mode}-${Date.now()}-${process.pid}`;
     const evidenceDir = join(evidenceRoot, evidenceName);
@@ -556,13 +627,11 @@ async function runScenario(mode: ScenarioMode, bucket: string): Promise<void> {
 }
 
 test("CLI fetch: files remain after same-content duplicate root resolution", async () => {
-    const remote = remoteSettings();
-    await runScenario("reproduction", remote.bucket);
+    await runScenario("reproduction", await scenarioRemoteSettings("reproduction"));
 });
 
-if (process.env["LIVESYNC_ISSUE1304_POSITIVE_CONTROL"] === "1") {
+if (!EXPLICIT_FIXTURE || process.env["LIVESYNC_ISSUE1304_POSITIVE_CONTROL"] === "1") {
     test("CLI fetch: --write-settings keeps the selected database across daemon startup", async () => {
-        const controlBucket = requiredEnv("LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET");
-        await runScenario("write-settings-control", controlBucket);
+        await runScenario("write-settings-control", await scenarioRemoteSettings("write-settings-control"));
     });
 }