Add explicit stopped-Journal recovery to Obsidian and CLI

Offer a confirmation in Maintenance and the journal-skip command to skip
only the current stopped Journal and resume receipt. Keep Commonlib notices
visible on stderr during ordinary CLI use.

Document the missing-content risk and replay procedure, and cover
cancellation, stale failures, subsequent failures, and receipt with unit
and plain and encrypted CLI and Obsidian end-to-end tests.

Requires the Commonlib stopped-Journal recovery API. Adopt its exact
published package version before integrating this consumer change.

src/apps/cli/README.md

@@ -80,6 +80,7 @@ livesync-cli [database-path] [command] [args...]
 ### Commands
 
 - `sync`: Run one replication cycle with the remote CouchDB.
+- `journal-skip <journal-key>`: Retry replication, skip only the exact currently stopped Object Storage Journal, and resume.
 - `mirror [vault-path]`: Bidirectional sync between the local database and a local directory (**the actual vault**).
     - If `vault-path` is provided, the CLI will synchronise the database with files in the vault directory.
     - If `vault-path` is omitted, it defaults to `database-path` (compatibility mode).
@@ -108,6 +109,27 @@ livesync-cli [database-path] [command] [args...]
 
 Remote-administration commands verify the resulting milestone state through the selected provider. The existing `[Verification]` lines remain suitable for scripts which inspect command output, while the default exit code now reflects whether that verification succeeded.
 
+### Object Storage receive failures
+
+`sync` stops at a Journal which cannot be processed or whose records cannot be
+saved. After correcting the cause, run `sync` again with the same database and
+settings to retry it. Existing partial writes are retained. `fetch --force`
+replaces the local database instead and follows the reconstruction rules below.
+
+If the Journal cannot be recovered and missing content or history is
+acceptable, run `journal-skip <journal-key>`. The CLI first performs normal
+replication with the retained local database. It skips only when that attempt
+stops at the exact supplied key; if replication succeeds or stops at another
+Journal, the command refuses the explicit skip. A successful normal attempt
+can still acknowledge Journals which it processed successfully.
+After the matching Journal is skipped, replication resumes. Any later failed
+Journal still stops, so each additional skip requires a separate command with
+that Journal's key. Normal identity, connectivity, authentication, and
+readiness checks still apply. This action does not persist a setting for future
+replication. Review the resulting file contents and retained history; a
+completed receive does not guarantee that later file reflection or reads will
+succeed.
+
 ### Interrupted reconstruction and older runtime state
 
 An interrupted `fetch` or `rebuild` leaves a record in
@@ -146,6 +168,7 @@ scopes do not make concurrent writers to shared device settings supported.
 ```bash
 # Basic sync with remote
 livesync-cli ./my-db sync
+livesync-cli ./my-db journal-skip 7-docs.jsonl.gz
 
 # Mirroring to your actual Obsidian vault
 livesync-cli ./my-db mirror /path/to/obsidian-vault
@@ -475,6 +498,8 @@ Commands:
   daemon                  (default) Run mirror scan then continuously sync CouchDB <-> local filesystem
   init-settings [path]    Create unconfigured settings JSON with the new-Vault recommendations
   sync                    Run one replication cycle and exit
+  journal-skip <journal-key>
+                          Retry replication, skip only the matching stopped Journal, and resume
   p2p-peers <timeout>     Show discovered peers as [peer]<TAB><peer-id><TAB><peer-name>
   p2p-sync <peer> <timeout>   Synchronise with specified peer-id or peer-name
   p2p-host                Start P2P host mode and wait until interrupted (Ctrl+C)

src/apps/cli/commands/runCommand.ts

@@ -28,6 +28,7 @@ import {
     USER_INITIATED_REPLICATION_AUTHORITY,
 } from "@vrtmrz/livesync-commonlib/replication";
 import { withOwnedRemoteResource } from "@/common/ownedRemoteResource";
+import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator";
 import {
     isCentralRemoteAdministrationCommand,
     runCentralRemoteAdministrationCommand,
@@ -45,10 +46,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
         writeStderrLine(standardIo, `[Error] ${forceRequirementError}`);
         return false;
     }
-    if (
-        options.command === "fetch" &&
-        isRemediationModeActive(core.services.setting.currentSettings())
-    ) {
+    if (options.command === "fetch" && isRemediationModeActive(core.services.setting.currentSettings())) {
         writeStderrLine(
             standardIo,
             `[Error] The fetch command cannot run while remediation mode is active. Clear the modification-time limit before fetching.`
@@ -202,6 +200,64 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
         return isReplicationCompleted(result);
     }
 
+    if (options.command === "journal-skip") {
+        const [journalKey] = options.commandArgs;
+        if (options.commandArgs.length !== 1 || journalKey === undefined || journalKey.length === 0) {
+            writeStderrLine(standardIo, `[Error] journal-skip requires one argument: <journal-key>`);
+            return false;
+        }
+
+        writeStdoutLine(standardIo, `[Command] journal-skip ${journalKey}`);
+        const replicatorBeforeReplication = core.services.replicator.getActiveReplicator();
+        const result = await core.services.replication.replicateUserInitiated({
+            trigger: "manual",
+            progressPresentation: REPLICATION_PROGRESS_PRESENTATIONS.NOTICE,
+            interaction: USER_INITIATED_REPLICATION_AUTHORITY,
+        });
+        const activeReplicator = core.services.replicator.getActiveReplicator();
+
+        if (activeReplicator !== replicatorBeforeReplication) {
+            writeStderrLine(
+                standardIo,
+                `[Error] The active replicator changed during replication; no Journal was skipped.`
+            );
+            return false;
+        }
+        if (isReplicationCompleted(result)) {
+            writeStderrLine(
+                standardIo,
+                `[Error] Replication completed without a stopped Journal; no Journal was skipped.`
+            );
+            return false;
+        }
+        if (!(activeReplicator instanceof LiveSyncJournalReplicator)) {
+            writeStderrLine(
+                standardIo,
+                `[Error] The active replicator is not a Journal replicator; no Journal was skipped.`
+            );
+            return false;
+        }
+
+        const stoppedJournal = activeReplicator.getStoppedJournal();
+        if (!stoppedJournal) {
+            writeStderrLine(standardIo, `[Error] Replication did not stop at a Journal; no Journal was skipped.`);
+            return false;
+        }
+        if (stoppedJournal.key !== journalKey) {
+            writeStderrLine(
+                standardIo,
+                `[Error] Replication stopped at Journal '${stoppedJournal.key}' (${stoppedJournal.stage}), not '${journalKey}'; no Journal was skipped.`
+            );
+            return false;
+        }
+
+        return await activeReplicator.skipStoppedJournalAndResume(
+            core.services.setting.currentSettings(),
+            stoppedJournal,
+            true
+        );
+    }
+
     if (options.command === "p2p-peers") {
         if (options.commandArgs.length < 1) {
             throw new Error("p2p-peers requires one argument: <timeout>");

src/apps/cli/commands/runCommand.unit.spec.ts

@@ -11,6 +11,7 @@ import {
 import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
 import { runCommand } from "./runCommand";
 import type { CLIOptions } from "./types";
+import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator";
 import {
     CENTRAL_REMOTE_ADMINISTRATION_ACTIONS,
     CENTRAL_REMOTE_ADMINISTRATION_FAILURE_REASONS,
@@ -20,6 +21,7 @@ import {
     CENTRAL_COMPATIBILITY_REJECTION_REASONS,
     REPLICATION_COMPLETED,
     REPLICATION_PROGRESS_PRESENTATIONS,
+    USER_INITIATED_REPLICATION_AUTHORITY,
     replicationFailed,
 } from "@vrtmrz/livesync-commonlib/replication";
 
@@ -137,6 +139,17 @@ function makeOptions(command: CLIOptions["command"], commandArgs: string[]): CLI
     };
 }
 
+function makeJournalSkipOptions(journalKey: string): CLIOptions {
+    return makeOptions("journal-skip", [journalKey]);
+}
+
+function createJournalReplicatorMock(stoppedJournal: unknown) {
+    const replicator = Object.create(LiveSyncJournalReplicator.prototype) as any;
+    replicator.getStoppedJournal = vi.fn(() => stoppedJournal);
+    replicator.skipStoppedJournalAndResume = vi.fn(async () => true);
+    return replicator;
+}
+
 async function createSetupURI(passphrase: string): Promise<string> {
     const settings = {
         ...DEFAULT_SETTINGS,
@@ -354,6 +367,108 @@ describe("runCommand abnormal cases", () => {
         );
     });
 
+    describe("journal-skip command", () => {
+        const snapshot = { key: "7-docs.jsonl.gz", stage: "parse" } as const;
+
+        it("skips only the exact stopped Journal after a normal user-initiated replication", async () => {
+            const core = createCoreMock();
+            const replicator = createJournalReplicatorMock(snapshot);
+            core.services.replicator.getActiveReplicator.mockReturnValue(replicator);
+            core.services.replication.replicateUserInitiated.mockResolvedValue(
+                replicationFailed(new Error("stopped Journal"))
+            );
+
+            await expect(runCommand(makeJournalSkipOptions(snapshot.key), { ...context, core })).resolves.toBe(true);
+
+            expect(core.services.replication.replicateUserInitiated).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    trigger: "manual",
+                    progressPresentation: REPLICATION_PROGRESS_PRESENTATIONS.NOTICE,
+                    interaction: USER_INITIATED_REPLICATION_AUTHORITY,
+                })
+            );
+            expect(replicator.getStoppedJournal).toHaveBeenCalledOnce();
+            expect(replicator.skipStoppedJournalAndResume).toHaveBeenCalledWith(
+                core.services.setting.currentSettings(),
+                snapshot,
+                true
+            );
+        });
+
+        it("refuses a key which does not match the stopped Journal", async () => {
+            const core = createCoreMock();
+            const replicator = createJournalReplicatorMock(snapshot);
+            core.services.replicator.getActiveReplicator.mockReturnValue(replicator);
+            core.services.replication.replicateUserInitiated.mockResolvedValue(
+                replicationFailed(new Error("stopped Journal"))
+            );
+
+            await expect(runCommand(makeJournalSkipOptions("8-docs.jsonl.gz"), { ...context, core })).resolves.toBe(
+                false
+            );
+
+            expect(replicator.skipStoppedJournalAndResume).not.toHaveBeenCalled();
+            expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith(
+                expect.stringContaining("7-docs.jsonl.gz")
+            );
+        });
+
+        it("refuses when normal replication does not establish a stopped Journal", async () => {
+            const core = createCoreMock();
+            const replicator = createJournalReplicatorMock(undefined);
+            core.services.replicator.getActiveReplicator.mockReturnValue(replicator);
+
+            await expect(runCommand(makeJournalSkipOptions(snapshot.key), { ...context, core })).resolves.toBe(false);
+
+            expect(replicator.skipStoppedJournalAndResume).not.toHaveBeenCalled();
+        });
+
+        it("refuses when the active replicator is not a Journal replicator", async () => {
+            const core = createCoreMock();
+            core.services.replicator.getActiveReplicator.mockReturnValue({ getStoppedJournal: vi.fn(() => snapshot) });
+            core.services.replication.replicateUserInitiated.mockResolvedValue(
+                replicationFailed(new Error("stopped Journal"))
+            );
+
+            await expect(runCommand(makeJournalSkipOptions(snapshot.key), { ...context, core })).resolves.toBe(false);
+
+            expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith(
+                expect.stringContaining("Journal replicator")
+            );
+        });
+
+        it("propagates a failure while resuming after the requested Journal was skipped", async () => {
+            const core = createCoreMock();
+            const replicator = createJournalReplicatorMock(snapshot);
+            replicator.skipStoppedJournalAndResume.mockResolvedValue(false);
+            core.services.replicator.getActiveReplicator.mockReturnValue(replicator);
+            core.services.replication.replicateUserInitiated.mockResolvedValue(
+                replicationFailed(new Error("stopped Journal"))
+            );
+
+            await expect(runCommand(makeJournalSkipOptions(snapshot.key), { ...context, core })).resolves.toBe(false);
+
+            expect(replicator.skipStoppedJournalAndResume).toHaveBeenCalledOnce();
+        });
+
+        it("refuses if the active replicator changes during the awaited replication", async () => {
+            const core = createCoreMock();
+            const replicator = createJournalReplicatorMock(snapshot);
+            const replacement = createJournalReplicatorMock(snapshot);
+            core.services.replicator.getActiveReplicator
+                .mockReturnValueOnce(replicator)
+                .mockReturnValueOnce(replacement);
+            core.services.replication.replicateUserInitiated.mockResolvedValue(
+                replicationFailed(new Error("stopped Journal"))
+            );
+
+            await expect(runCommand(makeJournalSkipOptions(snapshot.key), { ...context, core })).resolves.toBe(false);
+
+            expect(replicator.skipStoppedJournalAndResume).not.toHaveBeenCalled();
+            expect(replacement.skipStoppedJournalAndResume).not.toHaveBeenCalled();
+        });
+    });
+
     it("reports a lock from the exact sync outcome without inspecting a replacement Replicator", async () => {
         const core = createCoreMock();
         core.services.replication.replicateUserInitiated.mockResolvedValue(
@@ -546,9 +661,9 @@ describe("runCommand abnormal cases", () => {
             clock.mockReturnValue(end);
             const outOfWindow = createCoreMock();
             outOfWindow.services.context.standardIo.prompt.mockResolvedValue(passphrase);
-            await expect(runCommand(makeOptions("setup", [uri.trim()]), { ...context, core: outOfWindow })).rejects.toThrow(
-                "Cannot open Setup URI"
-            );
+            await expect(
+                runCommand(makeOptions("setup", [uri.trim()]), { ...context, core: outOfWindow })
+            ).rejects.toThrow("Cannot open Setup URI");
             expect(outOfWindow.services.setting.applyExternalSettings).not.toHaveBeenCalled();
             expect(outOfWindow.services.control.applySettings).not.toHaveBeenCalled();
         } finally {
@@ -559,8 +674,12 @@ describe("runCommand abnormal cases", () => {
     it("setup imports managed TURN through the existing encrypted URI", async () => {
         const core = createCoreMock();
         const profiles = {
-            turn: { id: "turn", name: "TURN", isEncrypted: false,
-                uri: "sls+p2p://room?managedType=CF&managedId=turn-key&token=private-token" },
+            turn: {
+                id: "turn",
+                name: "TURN",
+                isEncrypted: false,
+                uri: "sls+p2p://room?managedType=CF&managedId=turn-key&token=private-token",
+            },
         };
         const passphrase = "setup-passphrase";
         const setupURI = await processSetting.encodeSettingsToSetupURI(

src/apps/cli/commands/types.ts

@@ -7,6 +7,7 @@ import type { ReplicationSchedulingControl } from "@/serviceFeatures/replication
 export type CLICommand =
     | "daemon"
     | "sync"
+    | "journal-skip"
     | "p2p-peers"
     | "p2p-sync"
     | "p2p-host"
@@ -87,6 +88,7 @@ export interface CLICommandContext {
 export const VALID_COMMANDS = new Set([
     "daemon",
     "sync",
+    "journal-skip",
     "p2p-peers",
     "p2p-sync",
     "p2p-host",

src/apps/cli/main.log-handler.unit.spec.ts

@@ -2,6 +2,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import type { LOG_LEVEL } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import {
+    Logger,
+    LOG_LEVEL_DEBUG,
+    LOG_LEVEL_INFO,
+    LOG_LEVEL_NOTICE,
+    LOG_LEVEL_URGENT,
+    LOG_LEVEL_VERBOSE,
+    setGlobalLogFunction,
+} from "octagonal-wheels/common/logger";
 
 const mocks = vi.hoisted(() => ({
     setLogHandler: vi.fn(),
@@ -29,6 +39,18 @@ function createStandardIoMock() {
     };
 }
 
+async function invokeGlobalLogger(databasePath: string, message: string, level: LOG_LEVEL, debug = false) {
+    process.argv = ["node", "livesync-cli", databasePath, ...(debug ? ["--debug"] : []), "remote-ls"];
+    const standardIo = createStandardIoMock();
+    mocks.setLogHandler.mockImplementation(() => {
+        Logger(message, level);
+        throw new Error("__LOG_HANDLER_CONFIGURED__");
+    });
+
+    await expect(main(standardIo)).rejects.toThrow("__LOG_HANDLER_CONFIGURED__");
+    return standardIo;
+}
+
 describe("CLI log handler", () => {
     const originalArgv = process.argv.slice();
     let databasePath: string;
@@ -43,6 +65,7 @@ describe("CLI log handler", () => {
 
     afterEach(async () => {
         process.argv = originalArgv.slice();
+        setGlobalLogFunction(() => {});
         await rm(databasePath, { recursive: true, force: true });
     });
 
@@ -52,4 +75,50 @@ describe("CLI log handler", () => {
         await expect(main(createStandardIoMock())).rejects.toThrow("__LOG_HANDLER_CONFIGURED__");
         expect(mocks.setLogHandler).toHaveBeenCalledWith(expect.any(Function), true);
     });
+
+    it.each([
+        ["NOTICE", LOG_LEVEL_NOTICE],
+        ["URGENT", LOG_LEVEL_URGENT],
+    ] as const)("writes Commonlib %s diagnostics to stderr by default", async (name, level) => {
+        const message = `Commonlib ${name} diagnostic`;
+        const standardIo = await invokeGlobalLogger(databasePath, message, level);
+
+        expect(standardIo.writeStderr).toHaveBeenCalledWith(expect.stringContaining(`[${level}] ${message}\n`));
+        expect(standardIo.writeStdout).not.toHaveBeenCalledWith(expect.stringContaining(message));
+    });
+
+    it.each([
+        ["DEBUG", LOG_LEVEL_DEBUG],
+        ["INFO", LOG_LEVEL_INFO],
+        ["VERBOSE", LOG_LEVEL_VERBOSE],
+    ] as const)("keeps Commonlib %s diagnostics hidden by default", async (name, level) => {
+        const message = `Commonlib ${name} diagnostic`;
+        const standardIo = await invokeGlobalLogger(databasePath, message, level);
+
+        expect(standardIo.writeStderr).not.toHaveBeenCalledWith(expect.stringContaining(message));
+        expect(standardIo.writeStdout).not.toHaveBeenCalledWith(expect.stringContaining(message));
+    });
+
+    it("writes detailed Commonlib diagnostics to stderr in debug mode", async () => {
+        const messages = [
+            ["DEBUG", LOG_LEVEL_DEBUG],
+            ["INFO", LOG_LEVEL_INFO],
+            ["VERBOSE", LOG_LEVEL_VERBOSE],
+        ] as const;
+        process.argv = ["node", "livesync-cli", databasePath, "--debug", "remote-ls"];
+        const standardIo = createStandardIoMock();
+        mocks.setLogHandler.mockImplementation(() => {
+            for (const [name, level] of messages) Logger(`Commonlib ${name} diagnostic`, level);
+            throw new Error("__LOG_HANDLER_CONFIGURED__");
+        });
+
+        await expect(main(standardIo)).rejects.toThrow("__LOG_HANDLER_CONFIGURED__");
+
+        for (const [name, level] of messages) {
+            expect(standardIo.writeStderr).toHaveBeenCalledWith(
+                expect.stringContaining(`[${level}] Commonlib ${name} diagnostic\n`)
+            );
+        }
+        expect(standardIo.writeStdout).not.toHaveBeenCalledWith(expect.stringContaining("Commonlib"));
+    });
 });

src/apps/cli/main.ts

@@ -108,6 +108,8 @@ Arguments:
 Commands:
     daemon                  (default) Run mirror scan then continuously sync CouchDB <-> local filesystem
     sync                    Run one replication cycle and exit
+    journal-skip <journal-key>
+                            Retry replication, skip only the matching stopped Journal, and resume
     p2p-peers <timeout>     Show discovered peers as [peer]\t<peer-id>\t<peer-name>
     p2p-sync <peer> <timeout>
                             Sync with the specified peer-id or peer-name
@@ -376,15 +378,10 @@ export async function main(
         const writeLine = avoidStdoutNoise ? writeStderrLine : writeStdoutLine;
         writeLine(standardIo, ...values);
     };
-    if (options.debug) {
-        setGlobalLogFunction((msg, level) => {
-            writeStderrLine(standardIo, `[${level}]`, msg);
-        });
-    } else {
-        setGlobalLogFunction((msg, level) => {
-            // NO OP, leave it to logFunction
-        });
-    }
+    setGlobalLogFunction((msg, level) => {
+        if (!options.debug && level !== LOG_LEVEL_NOTICE && level !== LOG_LEVEL_URGENT) return;
+        writeStderrLine(standardIo, `[${level}]`, msg);
+    });
     if (options.command === "init-settings") {
         await createDefaultSettingsFile(options, standardIo);
         return;

src/apps/cli/main.unit.spec.ts

@@ -72,6 +72,7 @@ describe("CLI parseArgs", () => {
         expect(combined).toContain("Usage:");
         expect(combined).toContain("livesync-cli <database-path> [options] <command> [command-args]");
         expect(combined).toContain("--compat-remote-admin-exit-zero");
+        expect(combined).toContain("journal-skip <journal-key>");
     });
 
     it("parses p2p-peers command and timeout", () => {
@@ -101,6 +102,15 @@ describe("CLI parseArgs", () => {
         expect(parsed.commandArgs).toEqual([]);
     });
 
+    it("parses journal-skip and its exact Journal key", () => {
+        process.argv = ["node", "livesync-cli", "./databasePath", "journal-skip", "7-docs.jsonl.gz"];
+        const parsed = parseArgs();
+
+        expect(parsed.databasePath).toBe("./databasePath");
+        expect(parsed.command).toBe("journal-skip");
+        expect(parsed.commandArgs).toEqual(["7-docs.jsonl.gz"]);
+    });
+
     it.each(["rebuild", "fetch"] as const)("parses %s and its force flag", (command) => {
         process.argv = ["node", "livesync-cli", "./databasePath", "--force", command];
         const parsed = parseArgs();