ci: publish CLI images for AMD64 and ARM64

.github/workflows/cli-docker.yml

@@ -57,7 +57,7 @@ jobs:
                   MAJOR_MINOR=$(echo "${VERSION}" | cut -d. -f1,2)
                   SHORT_SHA=$(git rev-parse --short HEAD)
                   IMAGE="ghcr.io/${{ github.repository_owner }}/livesync-cli"
-                  
+
                   # Build tag list based on the event and git ref
                   TAGS=""
                   if [[ "${{ github.ref }}" == refs/tags/* ]]; then
@@ -70,7 +70,7 @@ jobs:
                       # Other branches / manual run fallback
                       TAGS="${IMAGE}:${VERSION}-dev-sha-${SHORT_SHA}-cli"
                   fi
-                  
+
                   # Determine if the image should be pushed
                   PUSH="true"
                   if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
@@ -78,7 +78,7 @@ jobs:
                           PUSH="false"
                       fi
                   fi
-                  
+
                   echo "tags=${TAGS}" >> $GITHUB_OUTPUT
                   echo "push=${PUSH}" >> $GITHUB_OUTPUT
 
@@ -89,6 +89,11 @@ jobs:
                   username: ${{ github.actor }}
                   password: ${{ secrets.GITHUB_TOKEN }}
 
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+              with:
+                  platforms: arm64
+
             - name: Set up Docker Buildx
               uses: docker/setup-buildx-action@v3
 
@@ -126,5 +131,6 @@ jobs:
                   file: src/apps/cli/Dockerfile
                   push: ${{ steps.meta.outputs.push }}
                   tags: ${{ steps.meta.outputs.tags }}
+                  platforms: linux/amd64,linux/arm64
                   cache-from: type=gha
                   cache-to: type=gha,mode=max

package.json

@@ -45,6 +45,7 @@
         "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts",
         "test:e2e:obsidian:vault-reflection": "tsx test/e2e-obsidian/scripts/vault-reflection.ts",
         "test:e2e:obsidian:couchdb-upload": "tsx test/e2e-obsidian/scripts/couchdb-upload.ts",
+        "test:e2e:obsidian:cli-to-obsidian-sync": "tsx test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts",
         "test:e2e:obsidian:minio-upload": "tsx test/e2e-obsidian/scripts/minio-upload.ts",
         "test:e2e:obsidian:startup-scan": "tsx test/e2e-obsidian/scripts/startup-scan.ts",
         "test:e2e:obsidian:two-vault-sync": "tsx test/e2e-obsidian/scripts/two-vault-sync.ts",

test/e2e-obsidian/README.md

@@ -51,6 +51,7 @@ npm run test:e2e:obsidian:cli-help -- vaults verbose
 npm run test:e2e:obsidian:smoke
 npm run test:e2e:obsidian:vault-reflection
 npm run test:e2e:obsidian:couchdb-upload
+npm run test:e2e:obsidian:cli-to-obsidian-sync
 npm run test:e2e:obsidian:minio-upload
 npm run test:e2e:obsidian:startup-scan
 npm run test:e2e:obsidian:two-vault-sync
@@ -61,10 +62,27 @@ npm run test:e2e:obsidian:local-suite
 npm run test:e2e:obsidian:local-suite:services
 ```
 
-`test:e2e:obsidian:local-suite` runs `npm run build`, discovery, smoke, vault reflection, CouchDB upload, Object Storage upload, startup scan, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB and MinIO fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run.
+`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, vault reflection, CouchDB upload, CLI-to-Obsidian synchronisation, Object Storage upload, startup scan, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB and MinIO fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run.
 
 `test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, configures Self-hosted LiveSync through `obsidian-cli eval`, creates a note in real Obsidian, commits the note into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the metadata document and its chunk documents.
 
+`test:e2e:obsidian:cli-to-obsidian-sync` is the cross-runtime compatibility check for the official LiveSync CLI and the real Obsidian plug-in. Build the plug-in first, and build the local CLI too when no external CLI command is selected. The script uses E2EE, Path Obfuscation, and the current preferred chunk settings to create and synchronise a note through the CLI, starts real Obsidian with an isolated Vault and profile, synchronises the same CouchDB database, and verifies that the plug-in materialises identical note content. This covers the boundary that CLI-only and plug-in-only round trips do not exercise.
+
+By default, the compatibility check runs `node src/apps/cli/dist/index.cjs`. Set `LIVESYNC_CLI_COMMAND` to test another CLI build or distribution. The value may be a quoted command line or a JSON array of executable and prefix arguments; the scenario arguments are appended without going through a shell.
+
+For example, to test an executable on `PATH`:
+
+```bash
+LIVESYNC_CLI_COMMAND='livesync-cli' npm run test:e2e:obsidian:cli-to-obsidian-sync
+```
+
+On Linux, a multi-architecture published Docker image can run against the local CouchDB fixture by sharing the temporary directory, using host networking, preserving the host user's file ownership, and overriding the image entrypoint so that the runner can supply its explicit database path. Images published before ARM64 support remain AMD64-only and require configured Docker emulation on an ARM host.
+
+```bash
+LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) --volume /tmp:/tmp --entrypoint node ghcr.io/vrtmrz/livesync-cli:edge /app/dist/index.cjs" \
+  npm run test:e2e:obsidian:cli-to-obsidian-sync
+```
+
 `test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service, configures Self-hosted LiveSync for Object Storage through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix.
 
 `test:e2e:obsidian:startup-scan` configures a temporary CouchDB database, stops Obsidian, writes a note directly into the vault, restarts Obsidian, and verifies from CouchDB that the boot-time scan picked up the offline file.
@@ -104,6 +122,8 @@ Useful environment variables:
 - `E2E_OBSIDIAN_READY_TIMEOUT_MS`: plug-in readiness timeout in milliseconds.
 - `E2E_OBSIDIAN_CLI_READY_TIMEOUT_MS`: timeout for waiting until the vault-side Obsidian CLI exposes the plug-in catalogue.
 - `E2E_OBSIDIAN_CLI_TIMEOUT_MS`: timeout for each `obsidian-cli` invocation.
+- `E2E_LIVESYNC_CLI_TIMEOUT_MS`: timeout for each official LiveSync CLI invocation in the CLI-to-Obsidian compatibility check; default is 60 seconds.
+- `LIVESYNC_CLI_COMMAND`: optional LiveSync CLI executable and prefix arguments used by the CLI-to-Obsidian compatibility check. The default is the locally built CLI.
 - `E2E_OBSIDIAN_FILE_TIMEOUT_MS`: timeout for waiting until a note created through Obsidian's vault API is reflected to disk.
 - `E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS`: timeout for waiting until Self-hosted LiveSync reports that its core lifecycle and local database are ready.
 - `E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS`: timeout for waiting until a file appears in Self-hosted LiveSync's local database.

test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts

@@ -0,0 +1,356 @@
+import { spawn } from "node:child_process";
+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import {
+    assertCouchDbReachable,
+    createCouchDbDatabase,
+    deleteCouchDbDatabase,
+    loadCouchDbConfig,
+    makeUniqueDatabaseName,
+    waitForCouchDbDocs,
+} from "../runner/couchdb.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertEqual,
+    configureCouchDb,
+    prepareRemote,
+    pushLocalChanges,
+    waitForLiveSyncCoreReady,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
+process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "30000";
+process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ??= "30000";
+
+const liveSyncCli = resolve("src/apps/cli/dist/index.cjs");
+const notePath = "E2E/cli-to-obsidian.md";
+const noteContent = [
+    "# CLI to real Obsidian",
+    "",
+    "This note was created by the Self-hosted LiveSync CLI.",
+    "The real Obsidian plug-in must retrieve the same content from CouchDB.",
+    "0123456789 abcdefghijklmnopqrstuvwxyz 0123456789 abcdefghijklmnopqrstuvwxyz",
+    "",
+].join("\n");
+const e2eePassphrase = "real-obsidian-cli-compatibility-e2e";
+
+type LiveSyncCliCommand = {
+    executable: string;
+    prefixArgs: string[];
+};
+
+type CliResult = {
+    stdout: string;
+    stderr: string;
+};
+
+type CliFileInfo = {
+    id: string;
+    children: string[];
+};
+
+function parseCommandLine(value: string): string[] {
+    const trimmed = value.trim();
+    if (trimmed.startsWith("[")) {
+        const parsed = JSON.parse(trimmed) as unknown;
+        if (!Array.isArray(parsed) || parsed.length === 0 || parsed.some((part) => typeof part !== "string")) {
+            throw new Error("LIVESYNC_CLI_COMMAND JSON form must be a non-empty array of strings.");
+        }
+        return parsed;
+    }
+
+    const parts: string[] = [];
+    let current = "";
+    let quote: "'" | '"' | undefined;
+    let tokenStarted = false;
+    for (let index = 0; index < trimmed.length; index++) {
+        const character = trimmed[index];
+        if (quote) {
+            if (character === quote) {
+                quote = undefined;
+                continue;
+            }
+            if (character === "\\" && quote === '"' && ['"', "\\"].includes(trimmed[index + 1] ?? "")) {
+                current += trimmed[++index];
+                continue;
+            }
+            current += character;
+            continue;
+        }
+        if (character === "'" || character === '"') {
+            quote = character;
+            tokenStarted = true;
+            continue;
+        }
+        if (character === "\\" && ["'", '"', "\\", " ", "\t"].includes(trimmed[index + 1] ?? "")) {
+            current += trimmed[++index];
+            tokenStarted = true;
+            continue;
+        }
+        if (/\s/u.test(character)) {
+            if (tokenStarted) {
+                parts.push(current);
+                current = "";
+                tokenStarted = false;
+            }
+            continue;
+        }
+        current += character;
+        tokenStarted = true;
+    }
+    if (quote) {
+        throw new Error("LIVESYNC_CLI_COMMAND contains an unterminated quoted value.");
+    }
+    if (tokenStarted) {
+        parts.push(current);
+    }
+    if (parts.length === 0) {
+        throw new Error("LIVESYNC_CLI_COMMAND must not be empty.");
+    }
+    return parts;
+}
+
+function resolveLiveSyncCliCommand(): LiveSyncCliCommand {
+    const override = process.env.LIVESYNC_CLI_COMMAND;
+    if (override !== undefined) {
+        const [executable, ...prefixArgs] = parseCommandLine(override);
+        return { executable, prefixArgs };
+    }
+    return { executable: process.execPath, prefixArgs: [liveSyncCli] };
+}
+
+async function runLiveSyncCli(command: LiveSyncCliCommand, args: string[]): Promise<CliResult> {
+    return await new Promise((resolvePromise, reject) => {
+        const timeoutMs = Number(process.env.E2E_LIVESYNC_CLI_TIMEOUT_MS ?? 60000);
+        const child = spawn(command.executable, [...command.prefixArgs, ...args], {
+            cwd: process.cwd(),
+            env: process.env,
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        let stdout = "";
+        let stderr = "";
+        child.stdout.on("data", (chunk: Buffer) => {
+            stdout += chunk.toString("utf-8");
+        });
+        child.stderr.on("data", (chunk: Buffer) => {
+            stderr += chunk.toString("utf-8");
+        });
+        let timedOut = false;
+        const timeout = setTimeout(() => {
+            timedOut = true;
+            child.kill("SIGTERM");
+        }, timeoutMs);
+        child.on("error", (error) => {
+            clearTimeout(timeout);
+            reject(error);
+        });
+        child.on("exit", (code, signal) => {
+            clearTimeout(timeout);
+            const result = {
+                stdout,
+                stderr,
+            };
+            if (timedOut) {
+                reject(
+                    new Error(
+                        `LiveSync CLI timed out after ${timeoutMs} ms\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`
+                    )
+                );
+                return;
+            }
+            if (code === 0) {
+                resolvePromise(result);
+                return;
+            }
+            reject(
+                new Error(
+                    `LiveSync CLI failed with ${signal ? `signal ${signal}` : `exit code ${String(code)}`}\n` +
+                        `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`
+                )
+            );
+        });
+    });
+}
+
+async function configureLiveSyncCli(
+    command: LiveSyncCliCommand,
+    settingsPath: string,
+    couchDb: Awaited<ReturnType<typeof loadCouchDbConfig>>,
+    dbName: string
+): Promise<void> {
+    await runLiveSyncCli(command, ["init-settings", "--force", settingsPath]);
+    const settings = JSON.parse(await readFile(settingsPath, "utf-8")) as Record<string, unknown>;
+    Object.assign(settings, {
+        couchDB_URI: couchDb.uri,
+        couchDB_USER: couchDb.username,
+        couchDB_PASSWORD: couchDb.password,
+        couchDB_DBNAME: dbName,
+        remoteType: "",
+        liveSync: false,
+        syncOnStart: false,
+        syncOnSave: false,
+        usePluginSync: false,
+        usePluginSyncV2: true,
+        useEden: false,
+        customChunkSize: 60,
+        sendChunksBulk: false,
+        sendChunksBulkMaxSize: 1,
+        chunkSplitterVersion: "v3-rabin-karp",
+        readChunksOnline: true,
+        disableCheckingConfigMismatch: false,
+        enableCompression: false,
+        hashAlg: "xxhash64",
+        handleFilenameCaseSensitive: false,
+        doNotUseFixedRevisionForChunks: true,
+        E2EEAlgorithm: "v2",
+        encrypt: true,
+        passphrase: e2eePassphrase,
+        usePathObfuscation: true,
+        doctorProcessedVersion: "0.25.27",
+        isConfigured: true,
+    });
+    await writeFile(settingsPath, `${JSON.stringify(settings, null, 2)}\n`, "utf-8");
+}
+
+async function writeCliNote(
+    command: LiveSyncCliCommand,
+    databasePath: string,
+    settingsPath: string,
+    sourcePath: string
+): Promise<CliFileInfo> {
+    await mkdir(dirname(sourcePath), { recursive: true });
+    await writeFile(sourcePath, noteContent, "utf-8");
+    await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "push", sourcePath, notePath]);
+    const info = await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "info", notePath]);
+    const fileInfo = JSON.parse(info.stdout) as CliFileInfo;
+    if (!fileInfo.id || !Array.isArray(fileInfo.children) || fileInfo.children.length === 0) {
+        throw new Error(`LiveSync CLI did not create complete metadata for ${notePath}: ${info.stdout}`);
+    }
+    await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "sync"]);
+    return fileInfo;
+}
+
+async function waitForVaultContent(
+    vaultPath: string,
+    path: string,
+    timeoutMs = Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS)
+): Promise<string> {
+    const fullPath = join(vaultPath, path);
+    const deadline = Date.now() + timeoutMs;
+    let lastContent = "";
+    while (Date.now() < deadline) {
+        try {
+            lastContent = await readFile(fullPath, "utf-8");
+            if (lastContent === noteContent) {
+                return lastContent;
+            }
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+                throw error;
+            }
+        }
+        await new Promise((resolvePromise) => setTimeout(resolvePromise, 250));
+    }
+    throw new Error(`Timed out waiting for CLI-created note at ${fullPath}. Last content:\n${lastContent}`);
+}
+
+async function main(): Promise<void> {
+    const liveSyncCliCommand = resolveLiveSyncCliCommand();
+    if (process.env.LIVESYNC_CLI_COMMAND === undefined) {
+        await access(liveSyncCli).catch(() => {
+            throw new Error(
+                `Built LiveSync CLI was not found at ${liveSyncCli}. Run 'npm run build -w self-hosted-livesync-cli' first, or set LIVESYNC_CLI_COMMAND.`
+            );
+        });
+    }
+
+    const binary = requireObsidianBinary();
+    const obsidianCli = discoverObsidianCli();
+    if (!obsidianCli.binary) {
+        throw new Error(`Could not find obsidian-cli. Checked paths: ${obsidianCli.checked.join(", ")}`);
+    }
+
+    const couchDb = await loadCouchDbConfig();
+    const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "cli-to-obsidian");
+    const cliState = await mkdtemp(join(tmpdir(), "livesync-cli-to-obsidian-e2e-"));
+    const cliDatabasePath = join(cliState, "database");
+    const cliSettingsPath = join(cliState, "settings.json");
+    const cliSourcePath = join(cliState, "source", "cli-to-obsidian.md");
+    const vault = await createTemporaryVault();
+    let session: ObsidianLiveSyncSession | undefined;
+
+    try {
+        await assertCouchDbReachable(couchDb);
+        await createCouchDbDatabase(couchDb, dbName);
+        await mkdir(cliDatabasePath, { recursive: true });
+        await configureLiveSyncCli(liveSyncCliCommand, cliSettingsPath, couchDb, dbName);
+
+        if (process.env.LIVESYNC_CLI_COMMAND === undefined) {
+            console.log(`Using locally built LiveSync CLI: ${liveSyncCli}`);
+        } else {
+            console.log(
+                `Using LiveSync CLI command override: ${JSON.stringify(liveSyncCliCommand.executable)} ` +
+                    `with ${liveSyncCliCommand.prefixArgs.length} prefix argument(s)`
+            );
+        }
+        console.log(`Using Obsidian executable: ${binary}`);
+        console.log(`Temporary Obsidian vault: ${vault.path}`);
+        console.log(`Temporary CouchDB database: ${dbName}`);
+
+        const cliFileInfo = await writeCliNote(liveSyncCliCommand, cliDatabasePath, cliSettingsPath, cliSourcePath);
+        await waitForCouchDbDocs(couchDb, dbName, (docs) => {
+            const ids = new Set(docs.map((doc) => doc._id));
+            return ids.has(cliFileInfo.id) && cliFileInfo.children.every((childId) => ids.has(childId));
+        });
+
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: obsidianCli.binary,
+            vault,
+            startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
+        });
+        await waitForLiveSyncCoreReady(obsidianCli.binary, session.cliEnv);
+        await configureCouchDb(
+            obsidianCli.binary,
+            session.cliEnv,
+            {
+                uri: couchDb.uri,
+                username: couchDb.username,
+                password: couchDb.password,
+                dbName,
+            },
+            {
+                encrypt: true,
+                passphrase: e2eePassphrase,
+                usePathObfuscation: true,
+                E2EEAlgorithm: "v2",
+            }
+        );
+        await waitForLiveSyncCoreReady(obsidianCli.binary, session.cliEnv);
+        await prepareRemote(obsidianCli.binary, session.cliEnv);
+        await pushLocalChanges(obsidianCli.binary, session.cliEnv);
+
+        const received = await waitForVaultContent(vault.path, notePath);
+        assertEqual(received, noteContent, "The real Obsidian plug-in did not materialise the CLI-created note.");
+        console.log("CLI-created encrypted note was retrieved by the real Obsidian plug-in with identical content.");
+    } finally {
+        if (session) {
+            await session.app.stop();
+        }
+        await vault.dispose();
+        await rm(cliState, { recursive: true, force: true });
+        if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
+            await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => {
+                console.warn(error instanceof Error ? error.message : error);
+            });
+        }
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exit(1);
+});

test/e2e-obsidian/scripts/local-suite.ts

@@ -8,10 +8,17 @@ type Step = {
 
 const testSteps: Step[] = [
     { name: "build", args: ["run", "build"] },
+    ...(process.env.LIVESYNC_CLI_COMMAND === undefined
+        ? [{ name: "CLI build", args: ["run", "build", "-w", "self-hosted-livesync-cli"] }]
+        : []),
     { name: "discover", args: ["run", "test:e2e:obsidian:discover"] },
     { name: "smoke", args: ["run", "test:e2e:obsidian:smoke"] },
     { name: "vault reflection", args: ["run", "test:e2e:obsidian:vault-reflection"] },
     { name: "CouchDB upload", args: ["run", "test:e2e:obsidian:couchdb-upload"] },
+    {
+        name: "CLI to real Obsidian synchronisation",
+        args: ["run", "test:e2e:obsidian:cli-to-obsidian-sync"],
+    },
     { name: "Object Storage upload", args: ["run", "test:e2e:obsidian:minio-upload"] },
     { name: "startup scan", args: ["run", "test:e2e:obsidian:startup-scan"] },
     { name: "two-vault synchronisation", args: ["run", "test:e2e:obsidian:two-vault-sync"] },