Add explicit stopped-Journal recovery to Obsidian and CLI

Offer a confirmation in Maintenance and the journal-skip command to skip
only the current stopped Journal and resume receipt. Keep Commonlib notices
visible on stderr during ordinary CLI use.

Document the missing-content risk and replay procedure, and cover
cancellation, stale failures, subsequent failures, and receipt with unit
and plain and encrypted CLI and Obsidian end-to-end tests.

Requires the Commonlib stopped-Journal recovery API. Adopt its exact
published package version before integrating this consumer change.

test/e2e-cli/ci-tasks.json

@@ -16,5 +16,6 @@
     "test:e2e-matrix:minio-enc0",
     "test:e2e-matrix:minio-enc1",
     "test:fetch-database-continuity",
-    "test:journal-chunk-recovery"
+    "test:journal-chunk-recovery",
+    "test:journal-receive-failure"
 ]

test/e2e-cli/tasks.json

@@ -1,209 +1,145 @@
 {
-  "test:helpers": {
-    "test": [
-      "helpers/process.test.ts",
-      "test-ci-task-contract.ts"
-    ]
-  },
-  "test": {
-    "tasks": [
-      "test:ci"
-    ]
-  },
-  "test:ci": {
-    "script": "run-ci-suite.ts"
-  },
-  "test:p2p:compose": {
-    "script": "run-compose-p2p.ts"
-  },
-  "test:local": {
-    "tasks": [
-      "test:setup-put-cat",
-      "test:mirror",
-      "test:daemon"
-    ]
-  },
-  "test:daemon": {
-    "test": [
-      "test-daemon.ts"
-    ]
-  },
-  "test:daemon-startup": {
-    "test": [
-      "test-daemon-startup.ts"
-    ]
-  },
-  "test:decoupled-vault": {
-    "test": [
-      "test-decoupled-vault.ts"
-    ]
-  },
-  "test:remote-commands": {
-    "test": [
-      "test-remote-commands.ts"
-    ]
-  },
-  "test:settings-writeback": {
-    "test": [
-      "test-settings-writeback.ts"
-    ]
-  },
-  "test:fetch-database-continuity": {
-    "test": [
-      "test-fetch-database-continuity.ts"
-    ]
-  },
-  "test:remote-administration-exit-codes": {
-    "test": [
-      "test-remote-administration-exit-codes.ts"
-    ]
-  },
-  "test:push-pull": {
-    "test": [
-      "test-push-pull.ts"
-    ]
-  },
-  "test:setup-put-cat": {
-    "test": [
-      "test-setup-put-cat.ts"
-    ]
-  },
-  "test:mirror": {
-    "test": [
-      "test-mirror.ts"
-    ]
-  },
-  "test:sync-two-local": {
-    "test": [
-      "test-sync-two-local-databases.ts"
-    ]
-  },
-  "test:sync-locked-remote": {
-    "test": [
-      "test-sync-locked-remote.ts"
-    ]
-  },
-  "test:p2p-host": {
-    "test": [
-      "test-p2p-host.ts"
-    ]
-  },
-  "test:p2p-peers": {
-    "test": [
-      "test-p2p-peers-local-relay.ts"
-    ]
-  },
-  "test:p2p-sync": {
-    "test": [
-      "test-p2p-sync.ts"
-    ]
-  },
-  "test:p2p-automatic-host": {
-    "test": [
-      "test-p2p-automatic-host.ts"
-    ]
-  },
-  "test:p2p-replacement": {
-    "test": [
-      "test-p2p-replicator-replacement.ts"
-    ]
-  },
-  "test:p2p-relay-disconnect": {
-    "test": [
-      "test-p2p-relay-disconnect.ts"
-    ]
-  },
-  "test:p2p:ci": {
-    "test": [
-      "test-p2p-sync.ts",
-      "test-p2p-automatic-host.ts",
-      "test-p2p-replicator-replacement.ts",
-      "test-p2p-relay-disconnect.ts"
-    ]
-  },
-  "test:p2p-three-nodes": {
-    "test": [
-      "test-p2p-three-nodes-conflict.ts"
-    ]
-  },
-  "test:p2p-upload-download": {
-    "test": [
-      "test-p2p-upload-download-repro.ts"
-    ]
-  },
-  "test:benchmark-contract": {
-    "test": [
-      "test-benchmark-contract.ts"
-    ]
-  },
-  "bench:p2p": {
-    "script": "bench-p2p.ts"
-  },
-  "bench:couchdb": {
-    "script": "bench-couchdb.ts"
-  },
-  "bench:compression": {
-    "script": "bench-compression.ts"
-  },
-  "bench:cases": {
-    "script": "bench-network-cases.ts"
-  },
-  "bench:latency-sweep": {
-    "script": "bench-latency-sweep.ts"
-  },
-  "bench:p2p-split-node": {
-    "script": "bench-p2p-split-node.ts"
-  },
-  "bench:item1": {
-    "command": "bash",
-    "args": [
-      "./bench-run-item1.sh"
-    ]
-  },
-  "bench:item1:full": {
-    "command": "bash",
-    "args": [
-      "-lc",
-      "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh"
-    ]
-  },
-  "test:e2e-couchdb": {
-    "test": [
-      "test-e2e-two-vaults-couchdb.ts"
-    ]
-  },
-  "test:journal-chunk-recovery": {
-    "test": [
-      "test-journal-chunk-recovery.ts"
-    ]
-  },
-  "test:e2e-matrix": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ]
-  },
-  "test:e2e-matrix:couchdb-enc0": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: COUCHDB-enc0"
-  },
-  "test:e2e-matrix:couchdb-enc1": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: COUCHDB-enc1"
-  },
-  "test:e2e-matrix:minio-enc0": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: MINIO-enc0"
-  },
-  "test:e2e-matrix:minio-enc1": {
-    "test": [
-      "test-e2e-two-vaults-matrix.ts"
-    ],
-    "filter": "e2e matrix: MINIO-enc1"
-  }
+    "test:helpers": {
+        "test": ["helpers/process.test.ts", "test-ci-task-contract.ts"]
+    },
+    "test": {
+        "tasks": ["test:ci"]
+    },
+    "test:ci": {
+        "script": "run-ci-suite.ts"
+    },
+    "test:p2p:compose": {
+        "script": "run-compose-p2p.ts"
+    },
+    "test:local": {
+        "tasks": ["test:setup-put-cat", "test:mirror", "test:daemon"]
+    },
+    "test:daemon": {
+        "test": ["test-daemon.ts"]
+    },
+    "test:daemon-startup": {
+        "test": ["test-daemon-startup.ts"]
+    },
+    "test:decoupled-vault": {
+        "test": ["test-decoupled-vault.ts"]
+    },
+    "test:remote-commands": {
+        "test": ["test-remote-commands.ts"]
+    },
+    "test:settings-writeback": {
+        "test": ["test-settings-writeback.ts"]
+    },
+    "test:fetch-database-continuity": {
+        "test": ["test-fetch-database-continuity.ts"]
+    },
+    "test:remote-administration-exit-codes": {
+        "test": ["test-remote-administration-exit-codes.ts"]
+    },
+    "test:push-pull": {
+        "test": ["test-push-pull.ts"]
+    },
+    "test:setup-put-cat": {
+        "test": ["test-setup-put-cat.ts"]
+    },
+    "test:mirror": {
+        "test": ["test-mirror.ts"]
+    },
+    "test:sync-two-local": {
+        "test": ["test-sync-two-local-databases.ts"]
+    },
+    "test:sync-locked-remote": {
+        "test": ["test-sync-locked-remote.ts"]
+    },
+    "test:p2p-host": {
+        "test": ["test-p2p-host.ts"]
+    },
+    "test:p2p-peers": {
+        "test": ["test-p2p-peers-local-relay.ts"]
+    },
+    "test:p2p-sync": {
+        "test": ["test-p2p-sync.ts"]
+    },
+    "test:p2p-automatic-host": {
+        "test": ["test-p2p-automatic-host.ts"]
+    },
+    "test:p2p-replacement": {
+        "test": ["test-p2p-replicator-replacement.ts"]
+    },
+    "test:p2p-relay-disconnect": {
+        "test": ["test-p2p-relay-disconnect.ts"]
+    },
+    "test:p2p:ci": {
+        "test": [
+            "test-p2p-sync.ts",
+            "test-p2p-automatic-host.ts",
+            "test-p2p-replicator-replacement.ts",
+            "test-p2p-relay-disconnect.ts"
+        ]
+    },
+    "test:p2p-three-nodes": {
+        "test": ["test-p2p-three-nodes-conflict.ts"]
+    },
+    "test:p2p-upload-download": {
+        "test": ["test-p2p-upload-download-repro.ts"]
+    },
+    "test:benchmark-contract": {
+        "test": ["test-benchmark-contract.ts"]
+    },
+    "bench:p2p": {
+        "script": "bench-p2p.ts"
+    },
+    "bench:couchdb": {
+        "script": "bench-couchdb.ts"
+    },
+    "bench:compression": {
+        "script": "bench-compression.ts"
+    },
+    "bench:cases": {
+        "script": "bench-network-cases.ts"
+    },
+    "bench:latency-sweep": {
+        "script": "bench-latency-sweep.ts"
+    },
+    "bench:p2p-split-node": {
+        "script": "bench-p2p-split-node.ts"
+    },
+    "bench:item1": {
+        "command": "bash",
+        "args": ["./bench-run-item1.sh"]
+    },
+    "bench:item1:full": {
+        "command": "bash",
+        "args": [
+            "-lc",
+            "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh"
+        ]
+    },
+    "test:e2e-couchdb": {
+        "test": ["test-e2e-two-vaults-couchdb.ts"]
+    },
+    "test:journal-chunk-recovery": {
+        "test": ["test-journal-chunk-recovery.ts"]
+    },
+    "test:journal-receive-failure": {
+        "test": ["test-journal-receive-failure.ts"]
+    },
+    "test:e2e-matrix": {
+        "test": ["test-e2e-two-vaults-matrix.ts"]
+    },
+    "test:e2e-matrix:couchdb-enc0": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: COUCHDB-enc0"
+    },
+    "test:e2e-matrix:couchdb-enc1": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: COUCHDB-enc1"
+    },
+    "test:e2e-matrix:minio-enc0": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: MINIO-enc0"
+    },
+    "test:e2e-matrix:minio-enc1": {
+        "test": ["test-e2e-two-vaults-matrix.ts"],
+        "filter": "e2e matrix: MINIO-enc1"
+    }
 }

test/e2e-cli/test-journal-receive-failure.ts

@@ -0,0 +1,369 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { GetObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { runCli, runCliOrFail, sanitiseCatStdout } from "./helpers/cli.ts";
+import { startMinio, stopMinio } from "./helpers/docker.ts";
+import { TempDir } from "./helpers/temp.ts";
+import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
+
+type JsonObject = Record<string, unknown>;
+
+function requireEnv(...keys: string[]): string {
+    for (const key of keys) {
+        const value = process.env[key]?.trim();
+        if (value) return value;
+    }
+    throw new Error(`Required env var is missing: ${keys.join(" or ")}`);
+}
+
+function createJournalReceiveBucketName(baseBucket: string, suffix: string): string {
+    const bucketSuffix = `journal-receive-${suffix}`;
+    const maxBaseLength = 63 - bucketSuffix.length - 1;
+    const truncatedBase = baseBucket.slice(0, maxBaseLength).replace(/-+$/, "");
+    return truncatedBase ? `${truncatedBase}-${bucketSuffix}` : bucketSuffix;
+}
+
+function isJsonObject(value: unknown): value is JsonObject {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+async function listJournalKeys(client: S3Client, bucket: string): Promise<string[]> {
+    const result = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
+    assert.equal(result.IsTruncated, false, "The disposable test bucket should fit in one S3 listing page");
+    return (result.Contents ?? [])
+        .flatMap(({ Key }) => (Key && /^\d+-docs\.jsonl\.gz$/u.test(Key) ? [Key] : []))
+        .sort((left, right) => left.localeCompare(right, undefined, { numeric: true }));
+}
+
+async function addedJournalKey(client: S3Client, bucket: string, before: Set<string>): Promise<string> {
+    const added = (await listJournalKeys(client, bucket)).filter((key) => !before.has(key));
+    assert.equal(added.length, 1, `Expected one new Journal object, got: ${added.join(", ")}`);
+    const [key] = added;
+    assert.ok(key);
+    return key;
+}
+
+async function withCorruptedJournals(
+    client: S3Client,
+    bucket: string,
+    keys: string[],
+    action: () => Promise<void>
+): Promise<void> {
+    const originals = await Promise.all(
+        keys.map(async (key) => {
+            const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+            const body = response.Body;
+            assert.ok(body, `Could not read the original Journal bytes for ${key}`);
+            const bytes = await body.transformToByteArray();
+            assert.ok(bytes.length > 0, `The owned Journal object ${key} was empty`);
+            return { key, bytes };
+        })
+    );
+
+    try {
+        await Promise.all(
+            originals.map(({ key }) =>
+                client.send(
+                    new PutObjectCommand({
+                        Bucket: bucket,
+                        Key: key,
+                        Body: Uint8Array.of(0xff, 0xff, 0xff),
+                    })
+                )
+            )
+        );
+        await action();
+    } finally {
+        await Promise.all(
+            originals.map(({ key, bytes }) =>
+                client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: bytes }))
+            )
+        );
+    }
+}
+
+async function readReceivedJournalKeys(databasePath: string): Promise<Set<string>> {
+    const scopeRoot = join(databasePath, ".livesync", "runtime", "scopes");
+    const files = await readdir(scopeRoot, { recursive: true });
+    for (const relativePath of files.filter((file) => file.endsWith("keyvalue-db.json"))) {
+        const stored: unknown = JSON.parse(await readFile(join(scopeRoot, relativePath), "utf8"));
+        if (!isJsonObject(stored)) continue;
+        const entry = Object.entries(stored).find(([key]) => key.includes("bucketsync-checkpoint-"));
+        if (!entry || !isJsonObject(entry[1])) continue;
+        const receivedFiles = entry[1]["receivedFiles"];
+        if (!isJsonObject(receivedFiles)) continue;
+        const values = receivedFiles["values"];
+        if (receivedFiles["__nodeKvType"] !== "Set" || !Array.isArray(values)) continue;
+        return new Set(values.filter((value): value is string => typeof value === "string"));
+    }
+    throw new Error(`Journal receive checkpoint was not found under ${scopeRoot}`);
+}
+
+async function runScenario(encrypt: boolean): Promise<void> {
+    const suffix = `${encrypt ? "enc1" : "enc0"}-${Date.now()}-${randomUUID().replaceAll("-", "").slice(0, 8)}`;
+    const minioEndpoint = requireEnv("MINIO_ENDPOINT", "minioEndpoint").replace(/\/$/, "");
+    const minioAccessKey = requireEnv("MINIO_ACCESS_KEY", "accessKey");
+    const minioSecretKey = requireEnv("MINIO_SECRET_KEY", "secretKey");
+    const minioBucket = createJournalReceiveBucketName(requireEnv("MINIO_BUCKET_NAME", "bucketName"), suffix);
+    const passphrase = "journal-receive-e2e-passphrase";
+    const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1";
+    const stablePath = "e2e/stable.md";
+    const middlePath = "e2e/middle.md";
+    const secondBrokenPath = "e2e/second-broken.md";
+    const laterPath = "e2e/later.md";
+    const stableV1 = "Stable file revision one.\n";
+    const middleContent = "Content in the intermediate Journal.\n";
+    const secondBrokenContent = "Content in the next intermediate Journal.\n";
+    const stableV2 = "Stable file revision two.\n";
+    const laterContent = "Later Journal content.\n";
+
+    await using workDir = await TempDir.create(`livesync-cli-journal-receive-${suffix}`);
+    const sourceDatabase = workDir.join("source-db");
+    const sourceVault = workDir.join("source-vault");
+    const sourceSettings = workDir.join("source-settings.json");
+    const defaultDatabase = workDir.join("default-receiver-db");
+    const defaultVault = workDir.join("default-receiver-vault");
+    const defaultSettings = workDir.join("default-receiver-settings.json");
+    const skipDatabase = workDir.join("skip-receiver-db");
+    const skipVault = workDir.join("skip-receiver-vault");
+    const skipSettings = workDir.join("skip-receiver-settings.json");
+    const sourceStableFile = join(sourceVault, stablePath);
+    const sourceMiddleFile = join(sourceVault, middlePath);
+    const sourceSecondBrokenFile = join(sourceVault, secondBrokenPath);
+    const sourceLaterFile = join(sourceVault, laterPath);
+    const commonSettings = {
+        remoteType: "MINIO" as const,
+        minioBucket,
+        minioEndpoint,
+        minioAccessKey,
+        minioSecretKey,
+        encrypt,
+        passphrase,
+        enableCompression: false,
+    };
+    const client = new S3Client({
+        endpoint: minioEndpoint,
+        region: "us-east-1",
+        forcePathStyle: true,
+        credentials: { accessKeyId: minioAccessKey, secretAccessKey: minioSecretKey },
+    });
+
+    await Promise.all([
+        mkdir(sourceDatabase, { recursive: true }),
+        mkdir(sourceVault, { recursive: true }),
+        mkdir(defaultDatabase, { recursive: true }),
+        mkdir(defaultVault, { recursive: true }),
+        mkdir(skipDatabase, { recursive: true }),
+        mkdir(skipVault, { recursive: true }),
+    ]);
+    await mkdir(dirname(sourceStableFile), { recursive: true });
+    await writeFile(sourceStableFile, stableV1);
+
+    try {
+        await startMinio(minioEndpoint, minioAccessKey, minioSecretKey, minioBucket);
+        await Promise.all([
+            initSettingsFile(sourceSettings),
+            initSettingsFile(defaultSettings),
+            initSettingsFile(skipSettings),
+        ]);
+        await Promise.all([
+            applyRemoteSyncSettings(sourceSettings, commonSettings),
+            applyRemoteSyncSettings(defaultSettings, commonSettings),
+            applyRemoteSyncSettings(skipSettings, commonSettings),
+        ]);
+        const runSource = (...args: string[]) =>
+            runCliOrFail(sourceDatabase, "--settings", sourceSettings, "--vault", sourceVault, ...args);
+        const runDefault = (...args: string[]) =>
+            runCliOrFail(defaultDatabase, "--settings", defaultSettings, "--vault", defaultVault, ...args);
+        const tryDefault = (...args: string[]) =>
+            runCli(defaultDatabase, "--settings", defaultSettings, "--vault", defaultVault, ...args);
+        const runSkip = (...args: string[]) =>
+            runCliOrFail(skipDatabase, "--settings", skipSettings, "--vault", skipVault, ...args);
+        const trySkip = (...args: string[]) =>
+            runCli(skipDatabase, "--settings", skipSettings, "--vault", skipVault, ...args);
+
+        const beforeInitial = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("rebuild", "--force", "--write-settings");
+        const firstJournalKey = await addedJournalKey(client, minioBucket, beforeInitial);
+        await runDefault("fetch", "--force", "--write-settings");
+        await runSkip("fetch", "--force", "--write-settings");
+        const firstInfo = JSON.parse(await runSource("info", stablePath)) as JsonObject;
+        assert.equal(typeof firstInfo["revision"], "string");
+        const firstRevision = firstInfo["revision"] as string;
+        for (const receiver of [
+            { name: "default receiver", database: defaultDatabase, run: runDefault },
+            { name: "skip receiver", database: skipDatabase, run: runSkip },
+        ]) {
+            assert.equal(
+                sanitiseCatStdout(await receiver.run("cat", stablePath)),
+                stableV1,
+                `${receiver.name} should fetch the initial healthy Journal content`
+            );
+            const checkpoint = await readReceivedJournalKeys(receiver.database);
+            assert.ok(
+                checkpoint.has(firstJournalKey),
+                `${receiver.name} should checkpoint the initial healthy Journal`
+            );
+            const info = JSON.parse(await receiver.run("info", stablePath)) as JsonObject;
+            assert.equal(info["revision"], firstRevision, `${receiver.name} should retain the first source revision`);
+            assert.equal(
+                sanitiseCatStdout(await receiver.run("cat-rev", stablePath, firstRevision)),
+                stableV1,
+                `${receiver.name} should retain the first revision contents`
+            );
+        }
+
+        await mkdir(dirname(sourceMiddleFile), { recursive: true });
+        await writeFile(sourceMiddleFile, middleContent);
+        await runSource("push", sourceMiddleFile, middlePath);
+        const beforeMiddle = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("sync");
+        const brokenJournalKey = await addedJournalKey(client, minioBucket, beforeMiddle);
+
+        await mkdir(dirname(sourceSecondBrokenFile), { recursive: true });
+        await writeFile(sourceSecondBrokenFile, secondBrokenContent);
+        await runSource("push", sourceSecondBrokenFile, secondBrokenPath);
+        const beforeSecondBroken = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("sync");
+        const secondBrokenJournalKey = await addedJournalKey(client, minioBucket, beforeSecondBroken);
+
+        await writeFile(sourceStableFile, stableV2);
+        await writeFile(sourceLaterFile, laterContent);
+        await runSource("push", sourceStableFile, stablePath);
+        await runSource("push", sourceLaterFile, laterPath);
+        const beforeLater = new Set(await listJournalKeys(client, minioBucket));
+        await runSource("sync");
+        const laterJournalKey = await addedJournalKey(client, minioBucket, beforeLater);
+        assert.ok(firstJournalKey.localeCompare(brokenJournalKey, undefined, { numeric: true }) < 0);
+        assert.ok(brokenJournalKey.localeCompare(secondBrokenJournalKey, undefined, { numeric: true }) < 0);
+        assert.ok(secondBrokenJournalKey.localeCompare(laterJournalKey, undefined, { numeric: true }) < 0);
+
+        const finalInfo = JSON.parse(await runSource("info", stablePath)) as JsonObject;
+        assert.equal(typeof finalInfo["revision"], "string");
+        const finalRevision = finalInfo["revision"] as string;
+        assert.notEqual(finalRevision, firstRevision);
+
+        await withCorruptedJournals(client, minioBucket, [brokenJournalKey, secondBrokenJournalKey], async () => {
+            const failedSync = await tryDefault("--debug", "sync");
+            assert.notEqual(failedSync.code, 0, "Default receive must report failure for a corrupt Journal");
+            assert.match(failedSync.combined, /receive journal error|journal.*failed|replication.*failed/i);
+
+            const checkpoint = await readReceivedJournalKeys(defaultDatabase);
+            assert.ok(checkpoint.has(firstJournalKey), "Completed earlier Journal must remain checkpointed");
+            assert.equal(checkpoint.has(brokenJournalKey), false, "Failed Journal must not be checkpointed");
+            assert.equal(
+                checkpoint.has(secondBrokenJournalKey),
+                false,
+                "A later failed Journal must not be checkpointed"
+            );
+            assert.equal(checkpoint.has(laterJournalKey), false, "Failure must prevent checkpointing a later Journal");
+            assert.equal(
+                sanitiseCatStdout(await runDefault("cat", stablePath)),
+                stableV1,
+                "The earlier Journal content must remain available after the later failure"
+            );
+            assert.notEqual(
+                (await tryDefault("cat", middlePath)).code,
+                0,
+                "The broken Journal content must not be applied"
+            );
+            assert.notEqual(
+                (await tryDefault("cat", secondBrokenPath)).code,
+                0,
+                "The next broken Journal content must not be applied"
+            );
+            assert.notEqual(
+                (await tryDefault("cat", laterPath)).code,
+                0,
+                "A later Journal must not pass the failed Journal"
+            );
+        });
+
+        await runDefault("sync");
+        assert.equal(sanitiseCatStdout(await runDefault("cat", middlePath)), middleContent);
+        assert.equal(sanitiseCatStdout(await runDefault("cat", secondBrokenPath)), secondBrokenContent);
+        assert.equal(sanitiseCatStdout(await runDefault("cat", stablePath)), stableV2);
+        assert.equal(sanitiseCatStdout(await runDefault("cat", laterPath)), laterContent);
+        const retriedCheckpoint = await readReceivedJournalKeys(defaultDatabase);
+        for (const key of [firstJournalKey, brokenJournalKey, secondBrokenJournalKey, laterJournalKey]) {
+            assert.ok(retriedCheckpoint.has(key), `Retry should checkpoint Journal ${key}`);
+        }
+        const retriedInfo = JSON.parse(await runDefault("info", stablePath)) as JsonObject;
+        assert.equal(retriedInfo["revision"], finalRevision, "Retry should preserve the source stable-file revision");
+        assert.equal(
+            sanitiseCatStdout(await runDefault("cat-rev", stablePath, firstRevision)),
+            stableV1,
+            "Retry should retain the earlier stable-file revision"
+        );
+
+        await withCorruptedJournals(client, minioBucket, [brokenJournalKey, secondBrokenJournalKey], async () => {
+            const firstSkip = await trySkip("journal-skip", brokenJournalKey);
+            assert.notEqual(
+                firstSkip.code,
+                0,
+                `Skipping the first Journal should stop at the next broken Journal: ${firstSkip.combined}`
+            );
+            assert.ok(firstSkip.combined.includes(brokenJournalKey), "The first skip must identify its Journal key");
+            assert.ok(
+                firstSkip.combined.includes(secondBrokenJournalKey),
+                "Resumed replication must stop and identify the next broken Journal"
+            );
+            assert.match(
+                firstSkip.combined,
+                /decrypt|inflate|decompress|decode|parse|persist|apply/i,
+                "The first skip attempt must report the failed Journal processing stage"
+            );
+
+            const firstSkipCheckpoint = await readReceivedJournalKeys(skipDatabase);
+            assert.ok(firstSkipCheckpoint.has(firstJournalKey));
+            assert.ok(firstSkipCheckpoint.has(brokenJournalKey), "The requested Journal should be skipped");
+            assert.equal(
+                firstSkipCheckpoint.has(secondBrokenJournalKey),
+                false,
+                "The first skip must not acknowledge the next failed Journal"
+            );
+            assert.equal(firstSkipCheckpoint.has(laterJournalKey), false);
+            assert.equal(sanitiseCatStdout(await runSkip("cat", stablePath)), stableV1);
+            assert.notEqual((await trySkip("cat", middlePath)).code, 0);
+            assert.notEqual((await trySkip("cat", secondBrokenPath)).code, 0);
+            assert.notEqual((await trySkip("cat", laterPath)).code, 0);
+
+            const secondSkip = await trySkip("journal-skip", secondBrokenJournalKey);
+            assert.equal(secondSkip.code, 0, `Explicit second skip should resume replication: ${secondSkip.combined}`);
+            assert.ok(secondSkip.combined.includes(secondBrokenJournalKey), "The second skip must identify its key");
+            assert.match(
+                secondSkip.combined,
+                /decrypt|inflate|decompress|decode|parse|persist|apply/i,
+                "The second skip attempt must report the failed Journal processing stage"
+            );
+            const checkpoint = await readReceivedJournalKeys(skipDatabase);
+            for (const key of [firstJournalKey, brokenJournalKey, secondBrokenJournalKey, laterJournalKey]) {
+                assert.ok(checkpoint.has(key), `Explicit skips should advance past Journal ${key}`);
+            }
+            assert.equal(sanitiseCatStdout(await runSkip("cat", stablePath)), stableV2);
+            assert.equal(sanitiseCatStdout(await runSkip("cat", laterPath)), laterContent);
+            assert.notEqual((await trySkip("cat", middlePath)).code, 0);
+            assert.notEqual((await trySkip("cat", secondBrokenPath)).code, 0);
+            const skippedInfo = JSON.parse(await runSkip("info", stablePath)) as JsonObject;
+            assert.equal(skippedInfo["revision"], finalRevision, "Later valid data should retain its source revision");
+            assert.equal(
+                sanitiseCatStdout(await runSkip("cat-rev", stablePath, firstRevision)),
+                stableV1,
+                "Later valid data should retain earlier revision history"
+            );
+        });
+    } finally {
+        client.destroy();
+        if (!keepDocker) {
+            await stopMinio().catch(() => {});
+        }
+    }
+}
+
+test("journal receive failure, retry, and targeted skips through MINIO", async (t) => {
+    await t.test("without encryption", async () => runScenario(false));
+    await t.test("with encryption", async () => runScenario(true));
+});

test/e2e-obsidian/scripts/journal-nfc-resend.ts

@@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto";
 import { createServer } from "node:net";
 import { readFile, readdir } from "node:fs/promises";
 import { join } from "node:path";
+import { PutObjectCommand } from "@aws-sdk/client-s3";
 import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings";
 import { evalObsidianJson } from "../runner/cli.ts";
 import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
@@ -17,6 +18,8 @@ import {
     type LocalDatabaseEntry,
 } from "../runner/liveSyncWorkflow.ts";
 import {
+    createObjectStorageClient,
+    readObjectStorageObject,
     deleteObjectStoragePrefix,
     ensureObjectStorageBucket,
     listObjectStorageObjects,
@@ -27,7 +30,7 @@ import {
 } from "../runner/objectStorage.ts";
 import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
 import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts";
-import { withObsidianPage } from "../runner/ui.ts";
+import { openLiveSyncSettings, waitForVisibleObsidianDialogue, withObsidianPage } from "../runner/ui.ts";
 
 process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000";
 
@@ -557,6 +560,141 @@ async function waitForJournalObjects(
     );
 }
 
+async function verifySingleJournalRecovery(
+    cliBinary: string,
+    session: ObsidianLiveSyncSession,
+    vaultPath: string,
+    storage: ObjectStorageConfig,
+    prefix: string,
+    validJournalKey: string
+): Promise<void> {
+    const objects = await listObjectStorageObjects(storage, prefix);
+    const timestamp =
+        Math.max(Date.now(), ...objects.map(({ Key }) => Number(Key?.slice(prefix.length).split("-")[0]) || 0)) + 1;
+    const firstKey = `${timestamp}-docs.jsonl.gz`;
+    const secondKey = `${timestamp + 1}-docs.jsonl.gz`;
+    const laterKey = `${timestamp + 2}-docs.jsonl.gz`;
+    const validBytes = await readObjectStorageObject(storage, validJournalKey);
+    const client = createObjectStorageClient(storage);
+    try {
+        for (const [key, bytes] of [
+            [firstKey, new Uint8Array([1])],
+            [secondKey, new Uint8Array([1])],
+            [laterKey, validBytes],
+        ] as const) {
+            await client.send(new PutObjectCommand({ Bucket: storage.bucket, Key: prefix + key, Body: bytes }));
+        }
+    } finally {
+        client.destroy();
+    }
+
+    type RecoveryState = { key: string | null; stage: string | null; received: string[] };
+    const readState = async (): Promise<RecoveryState> =>
+        await evalObsidianJson<RecoveryState>(
+            cliBinary,
+            [
+                "(async()=>{",
+                "const core=app.plugins.plugins['obsidian-livesync'].core;",
+                "const replicator=core.services.replicator.getActiveReplicator();",
+                "const stopped=replicator.getStoppedJournal();",
+                "const checkpoint=await replicator.client.getCheckpointInfo();",
+                "return JSON.stringify({key:stopped?.key??null,stage:stopped?.stage??null,received:[...checkpoint.receivedFiles]});",
+                "})()",
+            ].join(""),
+            session.cliEnv,
+            timeoutMs
+        );
+    const waitForState = async (expectedKey: string | null, expectedReceived: string[]): Promise<void> => {
+        const deadline = Date.now() + timeoutMs;
+        let state: RecoveryState | undefined;
+        while (Date.now() < deadline) {
+            state = await readState();
+            if (state.key === expectedKey && expectedReceived.every((key) => state?.received.includes(key))) return;
+            await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+        throw new Error(`Journal recovery did not reach the expected state: ${JSON.stringify(state)}`);
+    };
+    const result = await evalObsidianJson<boolean>(
+        cliBinary,
+        [
+            "(async()=>{",
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            "const replicator=core.services.replicator.getActiveReplicator();",
+            "return JSON.stringify(await replicator.replicateAllFromServer(core.settings,true));",
+            "})()",
+        ].join(""),
+        session.cliEnv,
+        timeoutMs
+    );
+    assertEqual(result, false, "Receipt must stop at the first malformed Journal.");
+    const initial = await readState();
+    assertEqual(initial.key, firstKey, "The first failed Journal must be offered for recovery.");
+    for (const key of [firstKey, secondKey, laterKey]) {
+        assertEqual(initial.received.includes(key), false, "Failed and later Journals must remain unacknowledged.");
+    }
+
+    await withObsidianPage(session.remoteDebuggingPort, async (page) => {
+        const navigator = await openLiveSyncSettings(page, uiTimeoutMs);
+        try {
+            const maintenance = await navigator.openPage("Maintenance");
+            assertEqual(
+                await maintenance.getByText("Ignore broken journals (risk of data loss)", { exact: true }).count(),
+                0,
+                "The persistent skip toggle must not be offered."
+            );
+            const setting = maintenance.locator(".setting-item").filter({
+                has: navigator.page.getByText("Skip stopped Journal and resume", { exact: true }),
+            });
+            await setting.waitFor({ state: "visible", timeout: uiTimeoutMs });
+            const choose = async (key: string, accept: boolean): Promise<void> => {
+                await setting
+                    .getByRole("button", { name: "Skip and resume", exact: true })
+                    .click({ timeout: uiTimeoutMs });
+                const dialogue = await waitForVisibleObsidianDialogue(page, key, uiTimeoutMs);
+                const text = await dialogue.innerText();
+                if (!text.includes(key) || !/missing/i.test(text)) {
+                    throw new Error(
+                        "The skip confirmation must identify the exact Journal and its missing-content risk."
+                    );
+                }
+                await dialogue
+                    .getByRole("button", { name: accept ? "Yes" : "No", exact: true })
+                    .click({ timeout: uiTimeoutMs });
+            };
+            await choose(firstKey, false);
+            assertEqual(
+                JSON.stringify(await readState()),
+                JSON.stringify(initial),
+                "Cancelling must preserve the failure and checkpoint."
+            );
+            await choose(firstKey, true);
+            await waitForState(secondKey, [firstKey]);
+            const afterFirst = await readState();
+            assertEqual(
+                afterFirst.received.includes(secondKey),
+                false,
+                "One action must not skip the next failed Journal."
+            );
+            assertEqual(
+                afterFirst.received.includes(laterKey),
+                false,
+                "Receipt must stop before the later valid Journal."
+            );
+            await choose(secondKey, true);
+            await waitForState(null, [firstKey, secondKey, laterKey]);
+        } finally {
+            await navigator.close();
+        }
+    });
+    const saved = JSON.parse(
+        await readFile(join(vaultPath, ".obsidian/plugins/obsidian-livesync/data.json"), "utf8")
+    ) as Record<string, unknown>;
+    assertEqual("ignoreBrokenJournals" in saved, false, "Recovery must not persist a skip toggle.");
+    console.log(
+        "Journal recovery UI: cancellation preserved the checkpoint, each action skipped one named Journal, and the next failure stopped receipt."
+    );
+}
+
 async function postponeConflictIfOpen(session: ObsidianLiveSyncSession): Promise<void> {
     await withObsidianPage(session.remoteDebuggingPort, async (page) => {
         const dialogue = page.locator(".modal-container:visible").filter({ hasText: "Conflicting changes" });
@@ -794,6 +932,24 @@ async function runCase(
         );
         await waitForSinglePhysicalUnicodeFile(receiverVault.path, controlPathNfd, controlContent);
 
+        const validJournalKey = rebuiltKeys.find((key) => key.endsWith("-docs.jsonl.gz"));
+        if (!validJournalKey) throw new Error("A valid Journal fixture is required for recovery testing.");
+        await verifySingleJournalRecovery(
+            cliBinary,
+            receiverSession,
+            receiverVault.path,
+            objectStorage,
+            bucketPrefix,
+            validJournalKey
+        );
+        await postponeConflictIfOpen(receiverSession);
+        assertSameConflictTree(
+            await readConflictTree(cliBinary, receiverSession.cliEnv, conflictPathNfc, initialEntry.rev, false),
+            sourceTree,
+            "Receiver after explicit Journal skips"
+        );
+        await waitForSinglePhysicalUnicodeFile(receiverVault.path, controlPathNfd, controlContent);
+
         const initialJournalObjects = initialKeys.filter((key) => key.endsWith("-docs.jsonl.gz")).length;
         const rebuiltJournalObjects = rebuiltKeys.filter((key) => key.endsWith("-docs.jsonl.gz")).length;
         console.log(