Add explicit stopped-Journal recovery to Obsidian and CLI

Offer a confirmation in Maintenance and the journal-skip command to skip only the current stopped Journal and resume receipt. Keep Commonlib notices visible on stderr during ordinary CLI use. Document the missing-content risk and replay procedure, and cover cancellation, stale failures, subsequent failures, and receipt with unit and plain and encrypted CLI and Obsidian end-to-end tests. Requires the Commonlib stopped-Journal recovery API. Adopt its exact published package version before integrating this consumer change.
2026-11-05 15:58:19 +00:00 · 2026-11-04 15:25:30 +00:00
1 parent bf2241e5ee
commit a94c3fef90
@@ -16,5 +16,6 @@
    "test:e2e-matrix:minio-enc0",
    "test:e2e-matrix:minio-enc1",
    "test:fetch-database-continuity",
-    "test:journal-chunk-recovery"
+    "test:journal-chunk-recovery",
+    "test:journal-receive-failure"
 ]
@@ -1,209 +1,145 @@
 {
-  "test:helpers": {
-    "test": [
-      "helpers/process.test.ts",
-      "test-ci-task-contract.ts"
-    ]
-  },
-  "test": {
-    "tasks": [
-      "test:ci"
-    ]
-  },
-  "test:ci": {
-    "script": "run-ci-suite.ts"
-  },
-  "test:p2p:compose": {
-    "script": "run-compose-p2p.ts"
-  },
-  "test:local": {
-    "tasks": [
-      "test:setup-put-cat",
-      "test:mirror",
-      "test:daemon"
-    ]
-  },
-  "test:daemon": {
-    "test": [
-      "test-daemon.ts"
-    ]
-  },
-  "test:daemon-startup": {
-    "test": [
-      "test-daemon-startup.ts"
-    ]
-  },
-  "test:decoupled-vault": {
-    "test": [
-      "test-decoupled-vault.ts"
-    ]
-  },
-  "test:remote-commands": {
-    "test": [
-      "test-remote-commands.ts"
-    ]
-  },
-  "test:settings-writeback": {
-    "test": [
-      "test-settings-writeback.ts"
-    ]
-  },
-  "test:fetch-database-continuity": {
-    "test": [
-      "test-fetch-database-continuity.ts"
-    ]
-  },
-  "test:remote-administration-exit-codes": {
-    "test": [
-      "test-remote-administration-exit-codes.ts"
-    ]
-  },
-  "test:push-pull": {
-    "test": [
-      "test-push-pull.ts"
-    ]
-  },
-  "test:setup-put-cat": {
-    "test": [
-      "test-setup-put-cat.ts"
-    ]
-  },
-  "test:mirror": {
-    "test": [
-      "test-mirror.ts"
-    ]
-  },
-  "test:sync-two-local": {
-    "test": [
-      "test-sync-two-local-databases.ts"
-    ]
-  },
-  "test:sync-locked-remote": {
-    "test": [
-      "test-sync-locked-remote.ts"
-    ]
-  },
-  "test:p2p-host": {
-    "test": [
-      "test-p2p-host.ts"
-    ]
-  },
-  "test:p2p-peers": {
-    "test": [
-      "test-p2p-peers-local-relay.ts"
-    ]
-  },
-  "test:p2p-sync": {
-    "test": [
-      "test-p2p-sync.ts"
-    ]
-  },
-  "test:p2p-automatic-host": {
-    "test": [
-      "test-p2p-automatic-host.ts"
-    ]
-  },
-  "test:p2p-replacement": {
-    "test": [
-      "test-p2p-replicator-replacement.ts"
-    ]
-  },
-  "test:p2p-relay-disconnect": {
-    "test": [
-      "test-p2p-relay-disconnect.ts"
-    ]
-  },
-  "test:p2p:ci": {
-    "test": [
-      "test-p2p-sync.ts",
-      "test-p2p-automatic-host.ts",
-      "test-p2p-replicator-replacement.ts",
-      "test-p2p-relay-disconnect.ts"
-    ]
-  },
-  "test:p2p-three-nodes": {
-    "test": [
-      "test-p2p-three-nodes-conflict.ts"
-    ]
-  },
-  "test:p2p-upload-download": {
-    "test": [
-      "test-p2p-upload-download-repro.ts"
-    ]
-  },
-  "test:benchmark-contract": {
-    "test": [
-      "test-benchmark-contract.ts"
-    ]
-  },
-  "bench:p2p": {
-    "script": "bench-p2p.ts"
-  },
-  "bench:couchdb": {
-    "script": "bench-couchdb.ts"
-  },
-  "bench:compression": {
-    "script": "bench-compression.ts"
-  },
-  "bench:cases": {
-    "script": "bench-network-cases.ts"
-  },
-  "bench:latency-sweep": {
-    "script": "bench-latency-sweep.ts"
-  },
-  "bench:p2p-split-node": {
-    "script": "bench-p2p-split-node.ts"
-  },
-  "bench:item1": {
-    "command": "bash",
-    "args": [
-      "./bench-run-item1.sh"
-    ]
-  },
-  "bench:item1:full": {
-    "command": "bash",
-    "args": [
-      "-lc",
-      "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh"
-    ]
-  },
-  "test:e2e-couchdb": {
-    "test": [
-      "test-e2e-two-vaults-couchdb.ts"
-    ]
-  },
-  "test:journal-chunk-recovery": {
-    "test": [
-      "test-journal-chunk-recovery.ts"
-    ]
-  },
-  "test:e2e-matrix": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ]
-  },
-  "test:e2e-matrix:couchdb-enc0": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: COUCHDB-enc0"
-  },
-  "test:e2e-matrix:couchdb-enc1": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: COUCHDB-enc1"
-  },
-  "test:e2e-matrix:minio-enc0": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: MINIO-enc0"
-  },
-  "test:e2e-matrix:minio-enc1": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: MINIO-enc1"
-  }
+    "test:helpers": {
+        "test": ["helpers/process.test.ts", "test-ci-task-contract.ts"]
+    },
+    "test": {
+        "tasks": ["test:ci"]
+    },
+    "test:ci": {
+        "script": "run-ci-suite.ts"
+    },
+    "test:p2p:compose": {
+        "script": "run-compose-p2p.ts"
+    },
+    "test:local": {
+        "tasks": ["test:setup-put-cat", "test:mirror", "test:daemon"]
+    },
+    "test:daemon": {
+        "test": ["test-daemon.ts"]
+    },
+    "test:daemon-startup": {
+        "test": ["test-daemon-startup.ts"]
+    },
+    "test:decoupled-vault": {
+        "test": ["test-decoupled-vault.ts"]
+    },
+    "test:remote-commands": {
+        "test": ["test-remote-commands.ts"]
+    },
+    "test:settings-writeback": {
+        "test": ["test-settings-writeback.ts"]
+    },
+    "test:fetch-database-continuity": {
+        "test": ["test-fetch-database-continuity.ts"]
+    },
+    "test:remote-administration-exit-codes": {
+        "test": ["test-remote-administration-exit-codes.ts"]
+    },
+    "test:push-pull": {
+        "test": ["test-push-pull.ts"]
+    },
+    "test:setup-put-cat": {
+        "test": ["test-setup-put-cat.ts"]
+    },
+    "test:mirror": {
+        "test": ["test-mirror.ts"]
+    },
+    "test:sync-two-local": {
+        "test": ["test-sync-two-local-databases.ts"]
+    },
+    "test:sync-locked-remote": {
+        "test": ["test-sync-locked-remote.ts"]
+    },
+    "test:p2p-host": {
+        "test": ["test-p2p-host.ts"]
+    },
+    "test:p2p-peers": {
+        "test": ["test-p2p-peers-local-relay.ts"]
+    },
+    "test:p2p-sync": {
+        "test": ["test-p2p-sync.ts"]
+    },
+    "test:p2p-automatic-host": {
+        "test": ["test-p2p-automatic-host.ts"]
+    },
+    "test:p2p-replacement": {
+        "test": ["test-p2p-replicator-replacement.ts"]
+    },
+    "test:p2p-relay-disconnect": {
+        "test": ["test-p2p-relay-disconnect.ts"]
+    },
+    "test:p2p:ci": {
+        "test": [
+            "test-p2p-sync.ts",
+            "test-p2p-automatic-host.ts",
+            "test-p2p-replicator-replacement.ts",
+            "test-p2p-relay-disconnect.ts"
+        ]
+    },
+    "test:p2p-three-nodes": {
+        "test": ["test-p2p-three-nodes-conflict.ts"]
+    },
+    "test:p2p-upload-download": {
+        "test": ["test-p2p-upload-download-repro.ts"]
+    },
+    "test:benchmark-contract": {
+        "test": ["test-benchmark-contract.ts"]
+    },
+    "bench:p2p": {
+        "script": "bench-p2p.ts"
+    },
+    "bench:couchdb": {
+        "script": "bench-couchdb.ts"
+    },
+    "bench:compression": {
+        "script": "bench-compression.ts"
+    },
+    "bench:cases": {
+        "script": "bench-network-cases.ts"
+    },
+    "bench:latency-sweep": {
+        "script": "bench-latency-sweep.ts"
+    },
+    "bench:p2p-split-node": {
+        "script": "bench-p2p-split-node.ts"
+    },
+    "bench:item1": {
+        "command": "bash",
+        "args": ["./bench-run-item1.sh"]
+    },
+    "bench:item1:full": {
+        "command": "bash",
+        "args": [
+            "-lc",
+            "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh"
+        ]
+    },
+    "test:e2e-couchdb": {
+        "test": ["test-e2e-two-vaults-couchdb.ts"]
+    },
+    "test:journal-chunk-recovery": {
+        "test": ["test-journal-chunk-recovery.ts"]
+    },
+    "test:journal-receive-failure": {
+        "test": ["test-journal-receive-failure.ts"]
+    },
+    "test:e2e-matrix": {
+        "test": ["test-e2e-two-vaults-matrix.ts"]
+    },
+    "test:e2e-matrix:couchdb-enc0": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: COUCHDB-enc0"
+    },
+    "test:e2e-matrix:couchdb-enc1": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: COUCHDB-enc1"
+    },
+    "test:e2e-matrix:minio-enc0": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: MINIO-enc0"
+    },
+    "test:e2e-matrix:minio-enc1": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: MINIO-enc1"
+    }
 }
@@ -0,0 +1,369 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { GetObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { runCli, runCliOrFail, sanitiseCatStdout } from "./helpers/cli.ts";
+import { startMinio, stopMinio } from "./helpers/docker.ts";
+import { TempDir } from "./helpers/temp.ts";
+import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
+
+type JsonObject = Record<string, unknown>;
+
+function requireEnv(...keys: string[]): string {
+    for (const key of keys) {
+        const value = process.env[key]?.trim();
+        if (value) return value;
+    }
+    throw new Error(`Required env var is missing: ${keys.join(" or ")}`);
+}
+
+function createJournalReceiveBucketName(baseBucket: string, suffix: string): string {
+    const bucketSuffix = `journal-receive-${suffix}`;
+    const maxBaseLength = 63 - bucketSuffix.length - 1;
+    const truncatedBase = baseBucket.slice(0, maxBaseLength).replace(/-+$/, "");
+    return truncatedBase ? `${truncatedBase}-${bucketSuffix}` : bucketSuffix;
+}
+
+function isJsonObject(value: unknown): value is JsonObject {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+async function listJournalKeys(client: S3Client, bucket: string): Promise<string[]> {
+    const result = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
+    assert.equal(result.IsTruncated, false, "The disposable test bucket should fit in one S3 listing page");
+    return (result.Contents ?? [])
+        .flatMap(({ Key }) => (Key && /^\d+-docs\.jsonl\.gz$/u.test(Key) ? [Key] : []))
+        .sort((left, right) => left.localeCompare(right, undefined, { numeric: true }));
+}
+
+async function addedJournalKey(client: S3Client, bucket: string, before: Set<string>): Promise<string> {
+    const added = (await listJournalKeys(client, bucket)).filter((key) => !before.has(key));
+    assert.equal(added.length, 1, `Expected one new Journal object, got: ${added.join(", ")}`);
+    const [key] = added;
+    assert.ok(key);
+    return key;
+}
+
+async function withCorruptedJournals(
+    client: S3Client,
+    bucket: string,
+    keys: string[],
+    action: () => Promise<void>
+): Promise<void> {
+    const originals = await Promise.all(
+        keys.map(async (key) => {
+            const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+            const body = response.Body;
+            assert.ok(body, `Could not read the original Journal bytes for ${key}`);
+            const bytes = await body.transformToByteArray();
+            assert.ok(bytes.length > 0, `The owned Journal object ${key} was empty`);
+            return { key, bytes };
+        })
+    );
+
+    try {
+        await Promise.all(
+            originals.map(({ key }) =>
+                client.send(
+                    new PutObjectCommand({
+                        Bucket: bucket,
+                        Key: key,
+                        Body: Uint8Array.of(0xff, 0xff, 0xff),
+                    })
+                )
+            )
+        );
+        await action();
+    } finally {
+        await Promise.all(
+            originals.map(({ key, bytes }) =>
+                client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: bytes }))
+            )
+        );
+    }
+}
+
+async function readReceivedJournalKeys(databasePath: string): Promise<Set<string>> {
+    const scopeRoot = join(databasePath, ".livesync", "runtime", "scopes");
+    const files = await readdir(scopeRoot, { recursive: true });
+    for (const relativePath of files.filter((file) => file.endsWith("keyvalue-db.json"))) {
+        const stored: unknown = JSON.parse(await readFile(join(scopeRoot, relativePath), "utf8"));
+        if (!isJsonObject(stored)) continue;
+        const entry = Object.entries(stored).find(([key]) => key.includes("bucketsync-checkpoint-"));
+        if (!entry || !isJsonObject(entry[1])) continue;
+        const receivedFiles = entry[1]["receivedFiles"];
+        if (!isJsonObject(receivedFiles)) continue;
+        const values = receivedFiles["values"];
+        if (receivedFiles["__nodeKvType"] !== "Set" || !Array.isArray(values)) continue;
+        return new Set(values.filter((value): value is string => typeof value === "string"));
+    }
+    throw new Error(`Journal receive checkpoint was not found under ${scopeRoot}`);
+}
+
+async function runScenario(encrypt: boolean): Promise<void> {
+    const suffix = `${encrypt ? "enc1" : "enc0"}-${Date.now()}-${randomUUID().replaceAll("-", "").slice(0, 8)}`;
+    const minioEndpoint = requireEnv("MINIO_ENDPOINT", "minioEndpoint").replace(/\/$/, "");
+    const minioAccessKey = requireEnv("MINIO_ACCESS_KEY", "accessKey");
+    const minioSecretKey = requireEnv("MINIO_SECRET_KEY", "secretKey");
+    const minioBucket = createJournalReceiveBucketName(requireEnv("MINIO_BUCKET_NAME", "bucketName"), suffix);
+    const passphrase = "journal-receive-e2e-passphrase";
+    const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1";
+    const stablePath = "e2e/stable.md";
+    const middlePath = "e2e/middle.md";
+    const secondBrokenPath = "e2e/second-broken.md";
+    const laterPath = "e2e/later.md";
+    const stableV1 = "Stable file revision one.\n";
+    const middleContent = "Content in the intermediate Journal.\n";
+    const secondBrokenContent = "Content in the next intermediate Journal.\n";
+    const stableV2 = "Stable file revision two.\n";
+    const laterContent = "Later Journal content.\n";
+
+    await using workDir = await TempDir.create(`livesync-cli-journal-receive-${suffix}`);
+    const sourceDatabase = workDir.join("source-db");
+    const sourceVault = workDir.join("source-vault");
+    const sourceSettings = workDir.join("source-settings.json");
+    const defaultDatabase = workDir.join("default-receiver-db");
+    const defaultVault = workDir.join("default-receiver-vault");
+    const defaultSettings = workDir.join("default-receiver-settings.json");
+    const skipDatabase = workDir.join("skip-receiver-db");
+    const skipVault = workDir.join("skip-receiver-vault");
+    const skipSettings = workDir.join("skip-receiver-settings.json");
+    const sourceStableFile = join(sourceVault, stablePath);
+    const sourceMiddleFile = join(sourceVault, middlePath);
+    const sourceSecondBrokenFile = join(sourceVault, secondBrokenPath);
+    const sourceLaterFile = join(sourceVault, laterPath);
+    const commonSettings = {
+        remoteType: "MINIO" as const,
+        minioBucket,
+        minioEndpoint,
+        minioAccessKey,
+        minioSecretKey,
+        encrypt,
+        passphrase,
+        enableCompression: false,
+    };
+    const client = new S3Client({
+        endpoint: minioEndpoint,
+        region: "us-east-1",
+        forcePathStyle: true,
+        credentials: { accessKeyId: minioAccessKey, secretAccessKey: minioSecretKey },
+    });
+
+    await Promise.all([
+        mkdir(sourceDatabase, { recursive: true }),
+        mkdir(sourceVault, { recursive: true }),
+        mkdir(defaultDatabase, { recursive: true }),
+        mkdir(defaultVault, { recursive: true }),
+        mkdir(skipDatabase, { recursive: true }),
+        mkdir(skipVault, { recursive: true }),
+    ]);
+    await mkdir(dirname(sourceStableFile), { recursive: true });
+    await writeFile(sourceStableFile, stableV1);
+
+    try {
+        await startMinio(minioEndpoint, minioAccessKey, minioSecretKey, minioBucket);
+        await Promise.all([
+            initSettingsFile(sourceSettings),
+            initSettingsFile(defaultSettings),
+            initSettingsFile(skipSettings),
+        ]);
+        await Promise.all([
+            applyRemoteSyncSettings(sourceSettings, commonSettings),
+            applyRemoteSyncSettings(defaultSettings, commonSettings),
+            applyRemoteSyncSettings(skipSettings, commonSettings),
+        ]);
+        const runSource = (...args: string[]) =>
+            runCliOrFail(sourceDatabase, "--settings", sourceSettings, "--vault", sourceVault, ...args);
+        const runDefault = (...args: string[]) =>
+            runCliOrFail(defaultDatabase, "--settings", defaultSettings, "--vault", defaultVault, ...args);
+        const tryDefault = (...args: string[]) =>
+            runCli(defaultDatabase, "--settings", defaultSettings, "--vault", defaultVault, ...args);
+        const runSkip = (...args: string[]) =>
+            runCliOrFail(skipDatabase, "--settings", skipSettings, "--vault", skipVault, ...args);
+        const trySkip = (...args: string[]) =>
+            runCli(skipDatabase, "--settings", skipSettings, "--vault", skipVault, ...args);
+
+        const beforeInitial = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("rebuild", "--force", "--write-settings");
+        const firstJournalKey = await addedJournalKey(client, minioBucket, beforeInitial);
+        await runDefault("fetch", "--force", "--write-settings");
+        await runSkip("fetch", "--force", "--write-settings");
+        const firstInfo = JSON.parse(await runSource("info", stablePath)) as JsonObject;
+        assert.equal(typeof firstInfo["revision"], "string");
+        const firstRevision = firstInfo["revision"] as string;
+        for (const receiver of [
+            { name: "default receiver", database: defaultDatabase, run: runDefault },
+            { name: "skip receiver", database: skipDatabase, run: runSkip },
+        ]) {
+            assert.equal(
+                sanitiseCatStdout(await receiver.run("cat", stablePath)),
+                stableV1,
+                `${receiver.name} should fetch the initial healthy Journal content`
+            );
+            const checkpoint = await readReceivedJournalKeys(receiver.database);
+            assert.ok(
+                checkpoint.has(firstJournalKey),
+                `${receiver.name} should checkpoint the initial healthy Journal`
+            );
+            const info = JSON.parse(await receiver.run("info", stablePath)) as JsonObject;
+            assert.equal(info["revision"], firstRevision, `${receiver.name} should retain the first source revision`);
+            assert.equal(
+                sanitiseCatStdout(await receiver.run("cat-rev", stablePath, firstRevision)),
+                stableV1,
+                `${receiver.name} should retain the first revision contents`
+            );
+        }
+
+        await mkdir(dirname(sourceMiddleFile), { recursive: true });
+        await writeFile(sourceMiddleFile, middleContent);
+        await runSource("push", sourceMiddleFile, middlePath);
+        const beforeMiddle = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("sync");
+        const brokenJournalKey = await addedJournalKey(client, minioBucket, beforeMiddle);
+
+        await mkdir(dirname(sourceSecondBrokenFile), { recursive: true });
+        await writeFile(sourceSecondBrokenFile, secondBrokenContent);
+        await runSource("push", sourceSecondBrokenFile, secondBrokenPath);
+        const beforeSecondBroken = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("sync");
+        const secondBrokenJournalKey = await addedJournalKey(client, minioBucket, beforeSecondBroken);
+
+        await writeFile(sourceStableFile, stableV2);
+        await writeFile(sourceLaterFile, laterContent);
+        await runSource("push", sourceStableFile, stablePath);
+        await runSource("push", sourceLaterFile, laterPath);
+        const beforeLater = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("sync");
+        const laterJournalKey = await addedJournalKey(client, minioBucket, beforeLater);
+        assert.ok(firstJournalKey.localeCompare(brokenJournalKey, undefined, { numeric: true }) < 0);
+        assert.ok(brokenJournalKey.localeCompare(secondBrokenJournalKey, undefined, { numeric: true }) < 0);
+        assert.ok(secondBrokenJournalKey.localeCompare(laterJournalKey, undefined, { numeric: true }) < 0);
+
+        const finalInfo = JSON.parse(await runSource("info", stablePath)) as JsonObject;
+        assert.equal(typeof finalInfo["revision"], "string");
+        const finalRevision = finalInfo["revision"] as string;
+        assert.notEqual(finalRevision, firstRevision);
+
+        await withCorruptedJournals(client, minioBucket, [brokenJournalKey, secondBrokenJournalKey], async () => {
+            const failedSync = await tryDefault("--debug", "sync");
+            assert.notEqual(failedSync.code, 0, "Default receive must report failure for a corrupt Journal");
+            assert.match(failedSync.combined, /receive journal error|journal.*failed|replication.*failed/i);
+
+            const checkpoint = await readReceivedJournalKeys(defaultDatabase);
+            assert.ok(checkpoint.has(firstJournalKey), "Completed earlier Journal must remain checkpointed");
+            assert.equal(checkpoint.has(brokenJournalKey), false, "Failed Journal must not be checkpointed");
+            assert.equal(
+                checkpoint.has(secondBrokenJournalKey),
+                false,
+                "A later failed Journal must not be checkpointed"
+            );
+            assert.equal(checkpoint.has(laterJournalKey), false, "Failure must prevent checkpointing a later Journal");
+            assert.equal(
+                sanitiseCatStdout(await runDefault("cat", stablePath)),
+                stableV1,
+                "The earlier Journal content must remain available after the later failure"
+            );
+            assert.notEqual(
+                (await tryDefault("cat", middlePath)).code,
+                0,
+                "The broken Journal content must not be applied"
+            );
+            assert.notEqual(
+                (await tryDefault("cat", secondBrokenPath)).code,
+                0,
+                "The next broken Journal content must not be applied"
+            );
+            assert.notEqual(
+                (await tryDefault("cat", laterPath)).code,
+                0,
+                "A later Journal must not pass the failed Journal"
+            );
+        });
+
+        await runDefault("sync");
+        assert.equal(sanitiseCatStdout(await runDefault("cat", middlePath)), middleContent);
+        assert.equal(sanitiseCatStdout(await runDefault("cat", secondBrokenPath)), secondBrokenContent);
+        assert.equal(sanitiseCatStdout(await runDefault("cat", stablePath)), stableV2);
+        assert.equal(sanitiseCatStdout(await runDefault("cat", laterPath)), laterContent);
+        const retriedCheckpoint = await readReceivedJournalKeys(defaultDatabase);
+        for (const key of [firstJournalKey, brokenJournalKey, secondBrokenJournalKey, laterJournalKey]) {
+            assert.ok(retriedCheckpoint.has(key), `Retry should checkpoint Journal ${key}`);
+        }
+        const retriedInfo = JSON.parse(await runDefault("info", stablePath)) as JsonObject;
+        assert.equal(retriedInfo["revision"], finalRevision, "Retry should preserve the source stable-file revision");
+        assert.equal(
+            sanitiseCatStdout(await runDefault("cat-rev", stablePath, firstRevision)),
+            stableV1,
+            "Retry should retain the earlier stable-file revision"
+        );
+
+        await withCorruptedJournals(client, minioBucket, [brokenJournalKey, secondBrokenJournalKey], async () => {
+            const firstSkip = await trySkip("journal-skip", brokenJournalKey);
+            assert.notEqual(
+                firstSkip.code,
+                0,
+                `Skipping the first Journal should stop at the next broken Journal: ${firstSkip.combined}`
+            );
+            assert.ok(firstSkip.combined.includes(brokenJournalKey), "The first skip must identify its Journal key");
+            assert.ok(
+                firstSkip.combined.includes(secondBrokenJournalKey),
+                "Resumed replication must stop and identify the next broken Journal"
+            );
+            assert.match(
+                firstSkip.combined,
+                /decrypt|inflate|decompress|decode|parse|persist|apply/i,
+                "The first skip attempt must report the failed Journal processing stage"
+            );
+
+            const firstSkipCheckpoint = await readReceivedJournalKeys(skipDatabase);
+            assert.ok(firstSkipCheckpoint.has(firstJournalKey));
+            assert.ok(firstSkipCheckpoint.has(brokenJournalKey), "The requested Journal should be skipped");
+            assert.equal(
+                firstSkipCheckpoint.has(secondBrokenJournalKey),
+                false,
+                "The first skip must not acknowledge the next failed Journal"
+            );
+            assert.equal(firstSkipCheckpoint.has(laterJournalKey), false);
+            assert.equal(sanitiseCatStdout(await runSkip("cat", stablePath)), stableV1);
+            assert.notEqual((await trySkip("cat", middlePath)).code, 0);
+            assert.notEqual((await trySkip("cat", secondBrokenPath)).code, 0);
+            assert.notEqual((await trySkip("cat", laterPath)).code, 0);
+
+            const secondSkip = await trySkip("journal-skip", secondBrokenJournalKey);
+            assert.equal(secondSkip.code, 0, `Explicit second skip should resume replication: ${secondSkip.combined}`);
+            assert.ok(secondSkip.combined.includes(secondBrokenJournalKey), "The second skip must identify its key");
+            assert.match(
+                secondSkip.combined,
+                /decrypt|inflate|decompress|decode|parse|persist|apply/i,
+                "The second skip attempt must report the failed Journal processing stage"
+            );
+            const checkpoint = await readReceivedJournalKeys(skipDatabase);
+            for (const key of [firstJournalKey, brokenJournalKey, secondBrokenJournalKey, laterJournalKey]) {
+                assert.ok(checkpoint.has(key), `Explicit skips should advance past Journal ${key}`);
+            }
+            assert.equal(sanitiseCatStdout(await runSkip("cat", stablePath)), stableV2);
+            assert.equal(sanitiseCatStdout(await runSkip("cat", laterPath)), laterContent);
+            assert.notEqual((await trySkip("cat", middlePath)).code, 0);
+            assert.notEqual((await trySkip("cat", secondBrokenPath)).code, 0);
+            const skippedInfo = JSON.parse(await runSkip("info", stablePath)) as JsonObject;
+            assert.equal(skippedInfo["revision"], finalRevision, "Later valid data should retain its source revision");
+            assert.equal(
+                sanitiseCatStdout(await runSkip("cat-rev", stablePath, firstRevision)),
+                stableV1,
+                "Later valid data should retain earlier revision history"
+            );
+        });
+    } finally {
+        client.destroy();
+        if (!keepDocker) {
+            await stopMinio().catch(() => {});
+        }
+    }
+}
+
+test("journal receive failure, retry, and targeted skips through MINIO", async (t) => {
+    await t.test("without encryption", async () => runScenario(false));
+    await t.test("with encryption", async () => runScenario(true));
+});