Add Journal Chunk recovery regression coverage and guidance

test/README.md

@@ -77,6 +77,21 @@ npm run test:e2e:docker:mirror --workspace self-hosted-livesync-cli
 
 The expected result is `PASS=7 FAIL=0`. These mirror suites use temporary local databases and need no CouchDB service. The complete `test:e2e:docker:all` command also runs the other Docker CLI suites and manages a disposable CouchDB fixture; use it to check the full Docker CI gate. Keep the ordinary-edit and unknown-provenance scenarios aligned between the Node.js and Bash suites.
 
+## Object Storage Chunk recovery regression tests
+
+After building the CLI, run the focused recovery test with Node.js 26 or later:
+
+```bash
+NODE_OPTIONS=--max-old-space-size=3072 npm run build --workspace self-hosted-livesync-cli
+NODE_OPTIONS=--max-old-space-size=3072 MINIO_ENDPOINT=http://127.0.0.1:9000 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_BUCKET_NAME=livesync-test npm run test:e2e:task --workspace self-hosted-livesync-cli -- test:journal-chunk-recovery
+```
+
+The test manages the disposable `minio-test` RustFS container on ports 9000 and 9001. Run it separately from other tests using that container or those ports. It creates a distinct bucket and temporary Vaults for each case and removes its test container afterwards.
+
+Both the unencrypted and encrypted cases rebuild from a complete file containing raw unit separators (`U+001F`), Fetch into another Vault, and compare the exact bytes. A fresh CLI process reads the persisted content. The test then damages the receiving Vault file and verifies that Fetch retains that newer local file and preserves both contents as independent conflict branches. Each revision is read explicitly so the assertions do not depend on which branch PouchDB selects as the winner. After moving the damaged file to a separate backup, another Fetch into the empty Vault restores the remote contents. It uses `--write-settings` so later commands retain the database selection made during recovery. This test is included in the standard CLI CI task list.
+
+This establishes the CLI and shared Rebuild/Fetch behaviour. It does not exercise Obsidian's Hidden File Sync or Customisation Sync initialisation, nor inject a damaged Chunk under an existing content-addressed ID.
+
 ## Multiple-device conflict regression tests
 
 ### Preparation and execution

test/e2e-cli/ci-tasks.json

@@ -14,5 +14,6 @@
     "test:e2e-matrix:couchdb-enc0",
     "test:e2e-matrix:couchdb-enc1",
     "test:e2e-matrix:minio-enc0",
-    "test:e2e-matrix:minio-enc1"
+    "test:e2e-matrix:minio-enc1",
+    "test:journal-chunk-recovery"
 ]

test/e2e-cli/tasks.json

@@ -166,6 +166,11 @@
       "test-e2e-two-vaults-couchdb.ts"
     ]
   },
+  "test:journal-chunk-recovery": {
+    "test": [
+      "test-journal-chunk-recovery.ts"
+    ]
+  },
   "test:e2e-matrix": {
     "test": [
       "test-e2e-two-vaults-matrix.ts"

test/e2e-cli/test-journal-chunk-recovery.ts

@@ -0,0 +1,190 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { TempDir } from "./helpers/temp.ts";
+import { assertFilesEqual, runCliOrFail, sanitiseCatStdout } from "./helpers/cli.ts";
+import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
+import { startMinio, stopMinio } from "./helpers/docker.ts";
+
+function requireEnv(...keys: string[]): string {
+    for (const key of keys) {
+        const value = process.env[key]?.trim();
+        if (value) return value;
+    }
+    throw new Error(`Required env var is missing: ${keys.join(" or ")}`);
+}
+
+function createJournalFixture(): string {
+    return [
+        "Journal chunk recovery fixture: UTF-8, newlines, and backslashes.\n",
+        ...Array.from(
+            { length: 128 },
+            (_, index) =>
+                `record ${String(index).padStart(3, "0")}: prefix\\path café 日本語 \u001f middle-${index} \u001f suffix-${index} remains after the separator.\n`
+        ),
+        "Final suffix after every control separator remains present.\n",
+    ].join("");
+}
+
+async function runScenario(encrypt: boolean): Promise<void> {
+    const suffix = `${encrypt ? "enc1" : "enc0"}-${Date.now()}-${Math.floor(Math.random() * 100000)}`;
+    const minioEndpoint = requireEnv("MINIO_ENDPOINT", "minioEndpoint").replace(/\/$/, "");
+    const minioAccessKey = requireEnv("MINIO_ACCESS_KEY", "accessKey");
+    const minioSecretKey = requireEnv("MINIO_SECRET_KEY", "secretKey");
+    const minioBucket = `${requireEnv("MINIO_BUCKET_NAME", "bucketName")}-journal-recovery-${suffix}`;
+    const passphrase = "e2e-passphrase";
+    const fileContent = createJournalFixture();
+    const expectedBytes = Buffer.from(fileContent, "utf8");
+    const targetPath = "e2e/journal-chunk-recovery.md";
+    const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1";
+
+    await using workDir = await TempDir.create(`livesync-cli-journal-recovery-${suffix}`);
+    const sourceDatabase = workDir.join("source-db");
+    const receiverDatabase = workDir.join("receiver-db");
+    const sourceVault = workDir.join("source-vault");
+    const receiverVault = workDir.join("receiver-vault");
+    const sourceSettings = workDir.join("source-settings.json");
+    const receiverSettings = workDir.join("receiver-settings.json");
+    const sourceFile = join(sourceVault, targetPath);
+    const receiverFile = join(receiverVault, targetPath);
+    await Promise.all([
+        mkdir(sourceDatabase, { recursive: true }),
+        mkdir(receiverDatabase, { recursive: true }),
+        mkdir(sourceVault, { recursive: true }),
+        mkdir(receiverVault, { recursive: true }),
+    ]);
+    await mkdir(dirname(sourceFile), { recursive: true });
+    await writeFile(sourceFile, expectedBytes);
+
+    try {
+        await startMinio(minioEndpoint, minioAccessKey, minioSecretKey, minioBucket);
+        await initSettingsFile(sourceSettings);
+        await initSettingsFile(receiverSettings);
+        await applyRemoteSyncSettings(sourceSettings, {
+            remoteType: "MINIO",
+            minioBucket,
+            minioEndpoint,
+            minioAccessKey,
+            minioSecretKey,
+            encrypt,
+            passphrase,
+            enableCompression: false,
+        });
+        await applyRemoteSyncSettings(receiverSettings, {
+            remoteType: "MINIO",
+            minioBucket,
+            minioEndpoint,
+            minioAccessKey,
+            minioSecretKey,
+            encrypt,
+            passphrase,
+            enableCompression: false,
+        });
+
+        await runCliOrFail(
+            sourceDatabase,
+            "--settings",
+            sourceSettings,
+            "--vault",
+            sourceVault,
+            "rebuild",
+            "--force",
+            "--write-settings"
+        );
+        await runCliOrFail(
+            receiverDatabase,
+            "--settings",
+            receiverSettings,
+            "--vault",
+            receiverVault,
+            "fetch",
+            "--force",
+            "--write-settings"
+        );
+
+        await assertFilesEqual(sourceFile, receiverFile, "Fetched Vault file does not match the rebuild source");
+        const fetchedContent = sanitiseCatStdout(
+            await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "cat", targetPath)
+        );
+        assert.equal(fetchedContent, fileContent, "A fresh CLI process should read the exact fetched file content");
+
+        const damagedContent = "Damaged receiver Vault copy; the remote still has the complete source.\n";
+        await writeFile(receiverFile, damagedContent);
+        await runCliOrFail(
+            receiverDatabase,
+            "--settings",
+            receiverSettings,
+            "--vault",
+            receiverVault,
+            "fetch",
+            "--force",
+            "--write-settings"
+        );
+
+        assert.equal(
+            await readFile(receiverFile, "utf8"),
+            damagedContent,
+            "Fetch does not forcibly overwrite a newer local file"
+        );
+        const info: unknown = JSON.parse(
+            await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "info", targetPath)
+        );
+        assert.ok(info && typeof info === "object" && "revision" in info && "conflicts" in info);
+        assert.ok(typeof info.revision === "string" && typeof info.conflicts === "string");
+        assert.match(info.revision, /^1-[\da-f]+$/, "The fetched file should retain an independent root");
+        assert.match(info.conflicts, /^1-[\da-f]+$/, "The differing local file should be preserved as a conflict");
+        assert.notEqual(info.revision, info.conflicts);
+        const retainedContents: string[] = [];
+        for (const revision of [info.revision, info.conflicts]) {
+            retainedContents.push(
+                sanitiseCatStdout(
+                    await runCliOrFail(
+                        receiverDatabase,
+                        "--settings",
+                        receiverSettings,
+                        "cat-rev",
+                        targetPath,
+                        revision
+                    )
+                )
+            );
+        }
+        assert.deepEqual(
+            retainedContents.sort(),
+            [fileContent, damagedContent].sort(),
+            "Fetch should retain both the intact remote and differing local contents without inferring their ancestry"
+        );
+        await rename(receiverFile, workDir.join("damaged-vault-backup.md"));
+        await runCliOrFail(
+            receiverDatabase,
+            "--settings",
+            receiverSettings,
+            "--vault",
+            receiverVault,
+            "fetch",
+            "--force",
+            "--write-settings"
+        );
+        await assertFilesEqual(sourceFile, receiverFile, "Fetching into an empty Vault should restore the remote file");
+        assert.equal(await readFile(workDir.join("damaged-vault-backup.md"), "utf8"), damagedContent);
+        const restoredContent = sanitiseCatStdout(
+            await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "cat", targetPath)
+        );
+        assert.equal(restoredContent, fileContent, "A repeated fetch should retain the remote file content");
+        assert.deepEqual(
+            await readFile(sourceFile),
+            expectedBytes,
+            "The complete source Vault fixture should remain intact"
+        );
+    } finally {
+        if (!keepDocker) {
+            await stopMinio().catch(() => {});
+        }
+    }
+}
+
+test("journal chunk recovery through MINIO", async (t) => {
+    await t.test("without encryption", async () => runScenario(false));
+    await t.test("with encryption", async () => runScenario(true));
+});