Expose NFC path identity across supported transports

test/e2e-cli/test-p2p-sync.ts

@@ -19,152 +19,271 @@ import {
 import { runCli, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts";
 import { getOptimalLoopbackIp } from "./helpers/net.ts";
 
-test("p2p-sync: transfers with the same ID key and rejects a different document ID key", async () => {
-    const loopbackIp = await getOptimalLoopbackIp();
-    const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp;
+for (const [pathNormalizationVersion, encrypted] of [
+    [0, true],
+    [1, true],
+    [1, false],
+] as const) {
+    test(`p2p-sync: identity mode ${pathNormalizationVersion}, encryption ${encrypted} transfers matching IDs and rejects incompatible peers`, async () => {
+        const loopbackIp = await getOptimalLoopbackIp();
+        const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp;
 
-    const relay = process.env["RELAY"] ?? `ws://${loopbackHost}:4000/`;
-    const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`;
-    const passphrase = process.env["PASSPHRASE"] ?? "test";
-    const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "12");
-    const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "15");
-    const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`;
-    const hostPeerName = process.env["HOST_PEER_NAME"] ?? `p2p-host-${nonce}`;
-    const clientPeerName = process.env["CLIENT_PEER_NAME"] ?? `p2p-client-${nonce}`;
-    const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0";
-    const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none");
+        const relay = process.env["RELAY"] ?? `ws://${loopbackHost}:4000/`;
+        const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`;
+        const passphrase = process.env["PASSPHRASE"] ?? "test";
+        const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "12");
+        const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "15");
+        const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`;
+        const hostPeerName = process.env["HOST_PEER_NAME"] ?? `p2p-host-${nonce}`;
+        const clientPeerName = process.env["CLIENT_PEER_NAME"] ?? `p2p-client-${nonce}`;
+        const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0";
+        const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none");
 
-    await using workDir = await TempDir.create("livesync-cli-p2p-sync");
-    const hostVault = workDir.join("vault-host");
-    const hostSettings = workDir.join("settings-host.json");
-    const clientVault = workDir.join("vault-sync");
-    const clientSettings = workDir.join("settings-sync.json");
-    const rejectedVault = workDir.join("vault-rejected");
-    const rejectedSettings = workDir.join("settings-rejected.json");
-    await mkdir(hostVault, { recursive: true });
-    await mkdir(clientVault, { recursive: true });
-    await mkdir(rejectedVault, { recursive: true });
+        await using workDir = await TempDir.create("livesync-cli-p2p-sync");
+        const hostVault = workDir.join("vault-host");
+        const hostSettings = workDir.join("settings-host.json");
+        const clientVault = workDir.join("vault-sync");
+        const clientSettings = workDir.join("settings-sync.json");
+        const rejectedVault = workDir.join("vault-rejected");
+        const rejectedSettings = workDir.join("settings-rejected.json");
+        await mkdir(hostVault, { recursive: true });
+        await mkdir(clientVault, { recursive: true });
+        await mkdir(rejectedVault, { recursive: true });
 
-    const relayStarted = await maybeStartLocalRelay(relay);
-    const coturnStarted = await maybeStartCoturn(turnServers);
-    try {
-        await initSettingsFile(hostSettings);
-        await initSettingsFile(clientSettings);
-        await initSettingsFile(rejectedSettings);
-        await applyP2pSettings(
-            hostSettings,
-            roomId,
-            passphrase,
-            "self-hosted-livesync-cli-tests",
-            relay,
-            "~.*",
-            turnServers
-        );
-        await applyP2pSettings(
-            clientSettings,
-            roomId,
-            passphrase,
-            "self-hosted-livesync-cli-tests",
-            relay,
-            "~.*",
-            turnServers
-        );
-        await applyP2pSettings(
-            rejectedSettings,
-            roomId,
-            passphrase,
-            "self-hosted-livesync-cli-tests",
-            relay,
-            "~.*",
-            turnServers
-        );
-        await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase);
-        await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase);
-        await applyP2pTestTweaks(rejectedSettings, "p2p-rejected-" + nonce, passphrase);
-        for (const [vault, path, key, label] of [
-            [hostVault, hostSettings, "ab".repeat(32), "host"],
-            [clientVault, clientSettings, "ab".repeat(32), "client"],
-            [rejectedVault, rejectedSettings, "cd".repeat(32), "rejected"],
-        ]) {
-            const settings = JSON.parse(await readFile(path, "utf8"));
-            settings.idDerivationVersion = 1;
-            settings.idDerivationKey = key;
-            const sourcePath = workDir.join("setup-source-" + label + ".json");
-            await writeFile(sourcePath, JSON.stringify(settings));
-            const setupPassphrase = "independent-id-setup-passphrase";
-            const setupUri = await generateSetupUriFromSettings(sourcePath, setupPassphrase, true);
-            await runCliWithInputOrFail(setupPassphrase + "\n", vault, "--settings", path, "setup", setupUri);
-            const persisted = JSON.parse(await readFile(path, "utf8"));
-            assert(persisted.idDerivationVersion === 1, "The Setup URI lost the ID derivation version.");
-            assert(persisted.idDerivationKey === "", "The CLI stored the ID key in plain text.");
-            assert(
-                typeof persisted.encryptedIdDerivationKey === "string" && persisted.encryptedIdDerivationKey.length > 0,
-                "The CLI did not encrypt the saved ID key."
-            );
-            assert(persisted.P2P_Enabled === true, "The Setup URI disabled P2P.");
-            assert(persisted.P2P_roomID === roomId, "The Setup URI changed the P2P room.");
-            assert(persisted.P2P_relays === relay, "The Setup URI changed the P2P relay.");
-            assert(persisted.remoteType === "ONLY_P2P", "The Setup URI changed the remote type.");
-        }
-        const notePath = "p2p/independent-id-note.md";
-        await runCliWithInputOrFail(
-            "A note transferred with the saved ID key.\n",
-            clientVault,
-            "--settings",
-            clientSettings,
-            "put",
-            notePath
-        );
-
-        const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
+        const relayStarted = await maybeStartLocalRelay(relay);
+        const coturnStarted = await maybeStartCoturn(turnServers);
         try {
-            await host.waitUntilContains("P2P host is running", 20000);
-            const peer = await discoverPeer(
-                clientVault,
+            await initSettingsFile(hostSettings);
+            await initSettingsFile(clientSettings);
+            await initSettingsFile(rejectedSettings);
+            await applyP2pSettings(
+                hostSettings,
+                roomId,
+                passphrase,
+                "self-hosted-livesync-cli-tests",
+                relay,
+                "~.*",
+                turnServers
+            );
+            await applyP2pSettings(
                 clientSettings,
-                peersTimeout,
-                process.env["TARGET_PEER"] ?? hostPeerName
+                roomId,
+                passphrase,
+                "self-hosted-livesync-cli-tests",
+                relay,
+                "~.*",
+                turnServers
             );
-            const syncResult = await runCli(
-                clientVault,
-                "--settings",
-                clientSettings,
-                "p2p-sync",
-                peer.id,
-                String(syncTimeout)
-            );
-            assert(
-                syncResult.code === 0,
-                `p2p-sync failed\nstdout: ${syncResult.stdout}\nstderr: ${syncResult.stderr}`
-            );
-            const rejectedPeer = await discoverPeer(rejectedVault, rejectedSettings, peersTimeout, hostPeerName);
-            const rejectedSync = await runCli(
-                rejectedVault,
-                "--settings",
+            await applyP2pSettings(
                 rejectedSettings,
-                "p2p-sync",
-                rejectedPeer.id,
-                String(syncTimeout)
+                roomId,
+                passphrase,
+                "self-hosted-livesync-cli-tests",
+                relay,
+                "~.*",
+                turnServers
             );
+            await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase);
+            await applyP2pTestTweaks(clientSettings, clientPeerName, passphrase);
+            await applyP2pTestTweaks(rejectedSettings, "p2p-rejected-" + nonce, passphrase);
+            for (const [vault, path, key, label] of [
+                [hostVault, hostSettings, "ab".repeat(32), "host"],
+                [clientVault, clientSettings, "ab".repeat(32), "client"],
+                [
+                    rejectedVault,
+                    rejectedSettings,
+                    (pathNormalizationVersion === 1 ? "ab" : "cd").repeat(32),
+                    "rejected",
+                ],
+            ]) {
+                const settings = JSON.parse(await readFile(path, "utf8"));
+                settings.pathNormalizationVersion =
+                    pathNormalizationVersion === 1 && label === "rejected" ? 0 : pathNormalizationVersion;
+                settings.encrypt = encrypted;
+                settings.usePathObfuscation = encrypted;
+                settings.idDerivationVersion = 1;
+                settings.idDerivationKey = key;
+                const sourcePath = workDir.join("setup-source-" + label + ".json");
+                await writeFile(sourcePath, JSON.stringify(settings));
+                const setupPassphrase = "independent-id-setup-passphrase";
+                const setupUri = await generateSetupUriFromSettings(sourcePath, setupPassphrase, true);
+                await runCliWithInputOrFail(setupPassphrase + "\n", vault, "--settings", path, "setup", setupUri);
+                const persisted = JSON.parse(await readFile(path, "utf8"));
+                assert(persisted.idDerivationVersion === 1, "The Setup URI lost the ID derivation version.");
+                assert(
+                    persisted.pathNormalizationVersion === settings.pathNormalizationVersion,
+                    "The Setup URI lost the path identity version."
+                );
+                assert(persisted.idDerivationKey === "", "The CLI stored the ID key in plain text.");
+                assert(
+                    typeof persisted.encryptedIdDerivationKey === "string" &&
+                        persisted.encryptedIdDerivationKey.length > 0,
+                    "The CLI did not encrypt the saved ID key."
+                );
+                assert(persisted.P2P_Enabled === true, "The Setup URI disabled P2P.");
+                assert(persisted.P2P_roomID === roomId, "The Setup URI changed the P2P room.");
+                assert(persisted.P2P_relays === relay, "The Setup URI changed the P2P relay.");
+                assert(persisted.remoteType === "ONLY_P2P", "The Setup URI changed the remote type.");
+            }
+            const notePath =
+                pathNormalizationVersion === 1 ? "Cafe\u0301/re\u0301sume\u0301.md" : "p2p/independent-id-note.md";
+            const receivedPath = notePath.normalize("NFC");
+            await runCliWithInputOrFail(
+                "A note transferred with the saved ID key.\n",
+                clientVault,
+                "--settings",
+                clientSettings,
+                "put",
+                notePath
+            );
+
+            const originalInfo = JSON.parse(
+                await runCliOrFail(clientVault, "--settings", clientSettings, "info", notePath)
+            ) as { id: string; revision: string };
+            if (pathNormalizationVersion === 1) {
+                assert.equal(
+                    sanitiseCatStdout(
+                        await runCliOrFail(clientVault, "--settings", clientSettings, "cat", receivedPath)
+                    ),
+                    "A note transferred with the saved ID key.\n",
+                    "The NFC spelling did not address the NFD Metadata."
+                );
+            }
+            const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
+            try {
+                await host.waitUntilContains("P2P host is running", 20000);
+                const peer = await discoverPeer(
+                    clientVault,
+                    clientSettings,
+                    peersTimeout,
+                    process.env["TARGET_PEER"] ?? hostPeerName
+                );
+                const syncResult = await runCli(
+                    clientVault,
+                    "--settings",
+                    clientSettings,
+                    "p2p-sync",
+                    peer.id,
+                    String(syncTimeout)
+                );
+                assert(
+                    syncResult.code === 0,
+                    `p2p-sync failed\nstdout: ${syncResult.stdout}\nstderr: ${syncResult.stderr}`
+                );
+                const rejectedPeer = await discoverPeer(rejectedVault, rejectedSettings, peersTimeout, hostPeerName);
+                const rejectedSync = await runCli(
+                    rejectedVault,
+                    "--settings",
+                    rejectedSettings,
+                    "p2p-sync",
+                    rejectedPeer.id,
+                    String(syncTimeout)
+                );
+                assert(
+                    rejectedSync.code !== 0,
+                    `P2P accepted incompatible peer settings.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}`
+                );
+
+                assert(
+                    rejectedSync.combined.includes("Tweak values are not matched"),
+                    `P2P failed before checking peer settings.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}`
+                );
+            } finally {
+                await host.stop();
+            }
+            const received = sanitiseCatStdout(
+                await runCliOrFail(hostVault, "--settings", hostSettings, "cat", receivedPath)
+            ).trimEnd();
             assert(
-                rejectedSync.code !== 0,
-                `P2P accepted a different key for obfuscated document IDs.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}`
-            );
-            assert(
-                rejectedSync.combined.includes("Tweak values are not matched"),
-                `P2P failed before checking peer settings.\nstdout: ${rejectedSync.stdout}\nstderr: ${rejectedSync.stderr}`
+                received === "A note transferred with the saved ID key.",
+                "The host did not receive the keyed note."
             );
+            const rejectedRead = await runCli(rejectedVault, "--settings", rejectedSettings, "cat", notePath);
+            assert(rejectedRead.code !== 0, "The rejected device received the keyed note.");
+            if (pathNormalizationVersion === 1) {
+                const canonicalRead = await runCli(rejectedVault, "--settings", rejectedSettings, "cat", receivedPath);
+                assert(
+                    canonicalRead.code !== 0,
+                    "The rejected device received canonical Metadata despite its different identity mode."
+                );
+            }
+            if (pathNormalizationVersion === 1) {
+                const exchange = async (
+                    senderVault: string,
+                    senderSettings: string,
+                    receiverVault: string,
+                    receiverSettings: string,
+                    receiverName: string
+                ) => {
+                    const restartedHost = startCliInBackground(
+                        receiverVault,
+                        "--settings",
+                        receiverSettings,
+                        "p2p-host"
+                    );
+                    try {
+                        await restartedHost.waitUntilContains("P2P host is running", 20000);
+                        const peer = await discoverPeer(senderVault, senderSettings, peersTimeout, receiverName);
+                        await runCliOrFail(
+                            senderVault,
+                            "--settings",
+                            senderSettings,
+                            "p2p-sync",
+                            peer.id,
+                            String(syncTimeout)
+                        );
+                    } finally {
+                        await restartedHost.stop();
+                    }
+                };
+                const editedBody = "Edited through the NFC spelling after process restart.\n";
+                await runCliWithInputOrFail(editedBody, hostVault, "--settings", hostSettings, "put", receivedPath);
+                await exchange(hostVault, hostSettings, clientVault, clientSettings, clientPeerName);
+                assert.equal(
+                    sanitiseCatStdout(await runCliOrFail(clientVault, "--settings", clientSettings, "cat", notePath)),
+                    editedBody
+                );
+                const storedPath = (await runCliOrFail(clientVault, "--settings", clientSettings, "ls"))
+                    .trim()
+                    .split("\t")[0];
+                assert.equal(storedPath.normalize("NFC"), receivedPath);
+                const editedInfo = JSON.parse(
+                    await runCliOrFail(clientVault, "--settings", clientSettings, "info", storedPath)
+                ) as { id: string };
+                assert.equal(editedInfo.id, originalInfo.id, "The reverse edit changed the canonical Metadata ID.");
+                assert.equal(
+                    sanitiseCatStdout(
+                        await runCliOrFail(
+                            clientVault,
+                            "--settings",
+                            clientSettings,
+                            "cat-rev",
+                            notePath,
+                            originalInfo.revision
+                        )
+                    ),
+                    "A note transferred with the saved ID key.\n",
+                    "P2P discarded the original revision body."
+                );
+
+                const renamedPath = "Cafe\u0301/renamed.md";
+                await runCliWithInputOrFail(editedBody, clientVault, "--settings", clientSettings, "put", renamedPath);
+                await runCliOrFail(clientVault, "--settings", clientSettings, "rm", receivedPath);
+                await exchange(clientVault, clientSettings, hostVault, hostSettings, hostPeerName);
+                assert.notEqual((await runCli(hostVault, "--settings", hostSettings, "cat", notePath)).code, 0);
+                assert.equal(
+                    sanitiseCatStdout(
+                        await runCliOrFail(hostVault, "--settings", hostSettings, "cat", renamedPath.normalize("NFC"))
+                    ),
+                    editedBody
+                );
+                await runCliOrFail(clientVault, "--settings", clientSettings, "rm", renamedPath.normalize("NFC"));
+                await exchange(clientVault, clientSettings, hostVault, hostSettings, hostPeerName);
+                assert.notEqual((await runCli(hostVault, "--settings", hostSettings, "cat", renamedPath)).code, 0);
+            }
         } finally {
-            await host.stop();
+            await stopLocalRelayIfStarted(relayStarted);
+            await stopCoturnIfStarted(coturnStarted);
         }
-        const received = sanitiseCatStdout(
-            await runCliOrFail(hostVault, "--settings", hostSettings, "cat", notePath)
-        ).trimEnd();
-        assert(received === "A note transferred with the saved ID key.", "The host did not receive the keyed note.");
-        const rejectedRead = await runCli(rejectedVault, "--settings", rejectedSettings, "cat", notePath);
-        assert(rejectedRead.code !== 0, "The rejected device received the keyed note.");
-    } finally {
-        await stopLocalRelayIfStarted(relayStarted);
-        await stopCoturnIfStarted(coturnStarted);
-    }
-});
+    });
+}

test/e2e-obsidian/runner/cliJournalMigration.ts

@@ -0,0 +1,243 @@
+import { deepStrictEqual } from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { promisify } from "node:util";
+import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { assertEqual } from "./liveSyncWorkflow.ts";
+import { deleteObjectStoragePrefix, readObjectStorageJson, type ObjectStorageConfig } from "./objectStorage.ts";
+
+const notePath = "Cafe\u0301/re\u0301sume\u0301.md";
+const noteContent = "# CLI journal migration\n\nThe NFD physical path must keep its identity during migration.\n";
+
+type CliFileInfo = {
+    id: string;
+    revision: string;
+    path: string;
+};
+
+type JournalMilestone = {
+    node_chunk_info?: Record<string, { current?: number }>;
+};
+
+type CliResult = {
+    stdout: string;
+    stderr: string;
+};
+
+type CliRunner = (...args: string[]) => Promise<CliResult>;
+
+function cliErrorOutput(error: unknown): string {
+    if (error === null || typeof error !== "object") {
+        return String(error);
+    }
+    const details = error as { message?: unknown; stdout?: unknown; stderr?: unknown };
+    return [details.message, details.stdout, details.stderr]
+        .filter((value): value is string => typeof value === "string")
+        .join("\n");
+}
+
+async function readSingleMetadata(run: CliRunner, label: string): Promise<CliFileInfo> {
+    const output = (await run("ls")).stdout.trimEnd();
+    const paths = output === "" ? [] : output.split(/\r?\n/u).map((line) => line.split("\t", 1)[0] ?? "");
+    assertEqual(paths.length, 1, `${label} did not contain exactly one Metadata document.`);
+    const storedPath = paths[0];
+    if (!storedPath) {
+        throw new Error(`${label} did not report a stored path.`);
+    }
+    return JSON.parse((await run("info", storedPath)).stdout) as CliFileInfo;
+}
+
+export async function runCliJournalMigration(
+    objectStorage: ObjectStorageConfig,
+    bucketPrefix: string,
+    sourceSettings: Record<string, unknown>
+): Promise<void> {
+    const migrationPrefix = `${bucketPrefix}migration/`;
+    const cliRoot = await mkdtemp(join(tmpdir(), "livesync-cli-journal-migration-"));
+    const sourceRoot = join(cliRoot, "source");
+    const targetRoot = join(cliRoot, "target");
+    const sourceDatabasePath = join(sourceRoot, "database");
+    const targetDatabasePath = join(targetRoot, "database");
+    const sourceVaultPath = join(sourceRoot, "vault");
+    const targetVaultPath = join(targetRoot, "vault");
+    const sourceSettingsPath = join(sourceRoot, "settings.json");
+    const targetSettingsPath = join(targetRoot, "settings.json");
+    const physicalParent = notePath.split("/")[0] ?? "";
+    const physicalFilename = notePath.split("/").slice(-1)[0] ?? "";
+
+    const writeCliSettings = async (settingsPath: string, pathNormalizationVersion: number) => {
+        const previous = await readFile(settingsPath, "utf8").then(
+            (text) => JSON.parse(text) as Record<string, unknown>,
+            (error: NodeJS.ErrnoException) => {
+                if (error.code !== "ENOENT") throw error;
+                return undefined;
+            }
+        );
+        await writeFile(
+            settingsPath,
+            JSON.stringify({
+                ...(previous ?? {
+                    ...DEFAULT_SETTINGS,
+                    ...objectStorage,
+                    ...sourceSettings,
+                    remoteType: "MINIO",
+                    bucketPrefix: migrationPrefix,
+                    isConfigured: true,
+                }),
+                pathNormalizationVersion,
+                liveSync: false,
+                syncOnStart: false,
+                syncOnSave: false,
+                syncOnEditorSave: false,
+                syncOnFileOpen: false,
+                syncAfterMerge: false,
+                periodicReplication: false,
+            })
+        );
+    };
+
+    try {
+        await mkdir(sourceDatabasePath, { recursive: true });
+        await mkdir(targetDatabasePath, { recursive: true });
+        await mkdir(dirname(join(sourceVaultPath, notePath)), { recursive: true });
+        await mkdir(dirname(join(targetVaultPath, notePath)), { recursive: true });
+        await writeFile(join(sourceVaultPath, notePath), noteContent);
+        await writeFile(join(targetVaultPath, notePath), noteContent);
+        await writeCliSettings(sourceSettingsPath, 0);
+        await writeCliSettings(targetSettingsPath, 0);
+
+        const execute = promisify(execFile);
+        const createRunner =
+            (databasePath: string, vaultPath: string, settingsPath: string): CliRunner =>
+            async (...args: string[]) =>
+                await execute(
+                    process.execPath,
+                    [
+                        resolve("src/apps/cli/dist/index.cjs"),
+                        databasePath,
+                        "--vault",
+                        vaultPath,
+                        "--settings",
+                        settingsPath,
+                        ...args,
+                    ],
+                    { timeout: 60_000, maxBuffer: 4 * 1024 * 1024 }
+                );
+
+        const source = createRunner(sourceDatabasePath, sourceVaultPath, sourceSettingsPath);
+        const target = createRunner(targetDatabasePath, targetVaultPath, targetSettingsPath);
+
+        await source("--force", "--write-settings", "rebuild");
+        await target("mirror");
+
+        const legacySource = await readSingleMetadata(source, "Legacy CLI source");
+        const legacyTarget = await readSingleMetadata(target, "Legacy CLI target");
+        const targetVaultEntriesBeforeFetch = await readdir(targetVaultPath);
+        const targetParentEntriesBeforeFetch = await readdir(dirname(join(targetVaultPath, notePath)));
+        assertEqual(
+            targetVaultEntriesBeforeFetch.includes(physicalParent),
+            true,
+            "The legacy CLI target did not retain the NFD physical parent."
+        );
+        assertEqual(
+            targetParentEntriesBeforeFetch.includes(physicalFilename),
+            true,
+            "The legacy CLI target did not retain the NFD physical filename."
+        );
+
+        await writeCliSettings(sourceSettingsPath, 1);
+        await source("--force", "--write-settings", "rebuild");
+
+        const migratedSource = await readSingleMetadata(source, "Migrated CLI source");
+        if (migratedSource.id === legacySource.id) {
+            throw new Error("The NFC migration did not change the source Metadata ID.");
+        }
+        assertEqual(
+            (await source("cat", notePath.normalize("NFC"))).stdout,
+            noteContent,
+            "The migrated CLI source did not retain the canonical Chunk content."
+        );
+
+        const milestoneKey = `${migrationPrefix}_00000000-milestone.json`;
+        const milestoneBeforeFailedFetch = await readObjectStorageJson<JournalMilestone>(objectStorage, milestoneKey);
+        const migrationVersions = Object.values(milestoneBeforeFailedFetch.node_chunk_info ?? {}).map(
+            (node) => node.current
+        );
+        assertEqual(migrationVersions.includes(4), true, "The migrated Journal milestone did not declare version 4.");
+
+        let legacyFetchError: unknown;
+        try {
+            await target("--force", "--write-settings", "fetch");
+        } catch (error) {
+            legacyFetchError = error;
+        }
+        if (legacyFetchError === undefined) {
+            throw new Error("The legacy CLI target unexpectedly fetched a Journal with NFC path identity.");
+        }
+        const legacyFetchOutput = cliErrorOutput(legacyFetchError);
+        if (!/path identity/iu.test(legacyFetchOutput)) {
+            throw new Error(
+                `The legacy CLI Fetch failed without the expected path identity error. Output: ${legacyFetchOutput}`
+            );
+        }
+
+        const targetAfterFailedFetch = await readSingleMetadata(target, "Legacy CLI target after failed Fetch");
+        assertEqual(
+            targetAfterFailedFetch.id,
+            legacyTarget.id,
+            "The failed legacy Fetch changed the target Metadata ID."
+        );
+        assertEqual(
+            targetAfterFailedFetch.revision,
+            legacyTarget.revision,
+            "The failed legacy Fetch changed the target Metadata revision."
+        );
+        const milestoneAfterFailedFetch = await readObjectStorageJson<JournalMilestone>(objectStorage, milestoneKey);
+        deepStrictEqual(
+            milestoneAfterFailedFetch,
+            milestoneBeforeFailedFetch,
+            "The failed legacy Fetch changed the remote Journal milestone."
+        );
+
+        await writeCliSettings(targetSettingsPath, 1);
+        await target("--force", "--write-settings", "fetch");
+
+        const migratedTarget = await readSingleMetadata(target, "Migrated CLI target");
+        assertEqual(migratedTarget.id, migratedSource.id, "The CLI source and target Metadata IDs did not converge.");
+        assertEqual(
+            (await target("cat", notePath.normalize("NFC"))).stdout,
+            noteContent,
+            "The migrated CLI target did not retain the canonical Chunk content."
+        );
+        assertEqual(
+            await readFile(join(targetVaultPath, notePath), "utf8"),
+            noteContent,
+            "The migrated CLI Fetch changed the NFD physical file contents."
+        );
+
+        const targetVaultEntriesAfterFetch = await readdir(targetVaultPath);
+        const targetParentEntriesAfterFetch = await readdir(dirname(join(targetVaultPath, notePath)));
+        deepStrictEqual(
+            targetVaultEntriesAfterFetch,
+            targetVaultEntriesBeforeFetch,
+            "The migrated CLI Fetch changed the physical Vault parent names."
+        );
+        deepStrictEqual(
+            targetParentEntriesAfterFetch,
+            targetParentEntriesBeforeFetch,
+            "The migrated CLI Fetch changed the physical NFD filename."
+        );
+
+        console.log(
+            "CLI_JOURNAL_MIGRATION: legacy Fetch rejection, safe migration, and NFD physical path preservation verified."
+        );
+    } finally {
+        try {
+            await rm(cliRoot, { recursive: true, force: true });
+        } finally {
+            await deleteObjectStoragePrefix(objectStorage, migrationPrefix);
+        }
+    }
+}

test/e2e-obsidian/runner/remoteActivity.ts

@@ -34,6 +34,7 @@ export type RemoteActivitySnapshot = {
     responseCount: number;
     statusBarFound: boolean;
     statusBarText: string;
+    indicatorElements?: Array<{ text: string; inactive: boolean; ariaHidden: string | null; display: string }>;
 };
 
 export type ExpectedRemoteActivityState =
@@ -88,14 +89,26 @@ async function readRemoteActivitySnapshotFromPage(page: Page): Promise<RemoteAct
                 gateKind: gate?.kind,
                 requestCount: Number(core.services?.API?.requestCount?.value ?? -1),
                 remoteOperationIndicatorVisible: statusBars.some((element) =>
-                    (element.textContent ?? "").includes(operationIcon)
-                ),
-                remoteRequestIndicatorVisible: statusBars.some((element) =>
-                    (element.textContent ?? "").includes(requestIcon)
+                    element.innerText.includes(operationIcon)
                 ),
+                remoteRequestIndicatorVisible: statusBars.some((element) => element.innerText.includes(requestIcon)),
                 responseCount: Number(core.services?.API?.responseCount?.value ?? -1),
                 statusBarFound: statusBars.length > 0,
-                statusBarText: statusBars.map((element) => element.textContent ?? "").join("\n"),
+                statusBarText: statusBars.map((element) => element.innerText).join("\n"),
+                indicatorElements: Array.from(
+                    document.querySelectorAll<HTMLElement>(".syncstatusbar .livesync-stat-item")
+                )
+                    .filter(
+                        (element) =>
+                            (element.textContent ?? "").includes(operationIcon) ||
+                            (element.textContent ?? "").includes(requestIcon)
+                    )
+                    .map((element) => ({
+                        text: element.textContent ?? "",
+                        inactive: element.classList.contains("livesync-stat-inactive"),
+                        ariaHidden: element.getAttribute("aria-hidden"),
+                        display: getComputedStyle(element).display,
+                    })),
             } satisfies RemoteActivitySnapshot;
         },
         {
@@ -141,7 +154,7 @@ export async function waitForRemoteActivityState(
                     if (!core) return false;
                     const gate = (globalThis as unknown as Record<string, RuntimeGate | undefined>)[stateKey];
                     const statusBarText = Array.from(document.querySelectorAll<HTMLElement>(".syncstatusbar"))
-                        .map((element) => element.textContent ?? "")
+                        .map((element) => element.innerText)
                         .join("\n");
                     const bounded = Number(core.services?.replicator?.boundedRemoteActivityCount?.value ?? -1);
                     const finite = Number(core.services?.replicator?.finiteReplicationActivityCount?.value ?? -1);

test/e2e-obsidian/scripts/minio-upload.ts

@@ -16,8 +16,15 @@
  * same upload.
  */
 import { randomBytes } from "node:crypto";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { promisify } from "node:util";
+import { DEFAULT_SETTINGS, TweakValuesTemplate } from "@vrtmrz/livesync-commonlib/compat/common/types";
 import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings";
 import { evalObsidianJson } from "../runner/cli.ts";
+import { runCliJournalMigration } from "../runner/cliJournalMigration.ts";
 import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
 import {
     assertEqual,
@@ -43,7 +50,8 @@ import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../r
 
 process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
 
-const notePath = "E2E/minio-upload.md";
+const unicodePaths = process.env.E2E_OBSIDIAN_UNICODE_PATHS === "true";
+const notePath = unicodePaths ? "Cafe\u0301/re\u0301sume\u0301.md" : "E2E/minio-upload.md";
 const useIndependentIds = process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true";
 const useCustomRequestHandler = process.env.E2E_OBSIDIAN_CUSTOM_HTTP_HANDLER === "true";
 const noteContent = [
@@ -63,10 +71,11 @@ async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.Proces
             `const path=${JSON.stringify(notePath)};`,
             `const content=${JSON.stringify(noteContent)};`,
             "const core=app.plugins.plugins['obsidian-livesync'].core;",
-            "if(!(await app.vault.adapter.exists('E2E'))) await app.vault.createFolder('E2E');",
+            "const parent=path.slice(0,path.lastIndexOf('/'));",
+            "if(!(await app.vault.adapter.exists(parent))) await app.vault.createFolder(parent);",
             "const existing=app.vault.getAbstractFileByPath(path);",
             "if(existing) await app.vault.delete(existing);",
-            "await app.vault.create(path,content);",
+            "const createdFile=await app.vault.create(path,content);",
             "const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));",
             "let entry=false;",
             "for(let i=0;i<40;i++){",
@@ -76,7 +85,7 @@ async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.Proces
             "await sleep(250);",
             "}",
             "if(!entry||!entry._id) throw new Error('Timed out waiting for local database entry');",
-            "return JSON.stringify({id:entry._id,path:entry.path,type:entry.type,children:entry.children||[]});",
+            "return JSON.stringify({id:entry._id,path:createdFile.path,type:entry.type,children:entry.children||[]});",
             "})()",
         ].join(""),
         env
@@ -124,10 +133,13 @@ async function main(): Promise<void> {
             cliBinary: cli.binary,
             vault,
             startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
-            pluginData: createE2eObjectStoragePluginData({
-                ...objectStorage,
-                bucketPrefix,
-            }),
+            pluginData: createE2eObjectStoragePluginData(
+                {
+                    ...objectStorage,
+                    bucketPrefix,
+                },
+                { pathNormalizationVersion: unicodePaths ? 1 : 0 }
+            ),
             localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
         });
         await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
@@ -137,6 +149,7 @@ async function main(): Promise<void> {
             session.cliEnv,
             { ...objectStorage, bucketPrefix },
             {
+                pathNormalizationVersion: unicodePaths ? 1 : 0,
                 ...(useIndependentIds
                     ? {
                           encrypt: true,
@@ -173,6 +186,13 @@ async function main(): Promise<void> {
                 );
             }
         }
+        if (unicodePaths && !useIndependentIds) {
+            assertEqual(
+                localEntry.id,
+                notePath.normalize("NFC").toLowerCase(),
+                "Obsidian did not derive the canonical Metadata ID."
+            );
+        }
         await pushLocalChanges(cli.binary, session.cliEnv);
         const activityAfterUpload = await waitForRemoteActivityState(
             session.remoteDebuggingPort,
@@ -198,6 +218,131 @@ async function main(): Promise<void> {
             }
         }
 
+        if (unicodePaths) {
+            const cliRoot = await mkdtemp(join(tmpdir(), "livesync-nfc-journal-"));
+            try {
+                const databasePath = join(cliRoot, "db");
+                const vaultPath = join(cliRoot, "vault");
+                const settingsPath = join(cliRoot, "settings.json");
+                await mkdir(databasePath, { recursive: true });
+                await mkdir(dirname(join(vaultPath, notePath)), { recursive: true });
+                await writeFile(join(vaultPath, notePath), noteContent);
+                const sourceSettings = await evalObsidianJson<Record<string, unknown>>(
+                    cli.binary,
+                    [
+                        "(async()=>{",
+                        "const service=app.plugins.plugins['obsidian-livesync'].core.services.setting;",
+                        "const s=service.currentSettings();",
+                        `const keys=${JSON.stringify([...Object.keys(TweakValuesTemplate), "encrypt", "passphrase", "usePathObfuscation", "E2EEAlgorithm", "idDerivationKey"])};`,
+                        "const values=Object.fromEntries(keys.map(key=>[key,s[key]]));",
+                        "if(s.idDerivationVersion===1){",
+                        "values.encryptedIdDerivationKey=await service.encryptConfigurationItem(s.idDerivationKey,s,'*');",
+                        "values.idDerivationKey='';values.configPassphraseStore='';",
+                        "}",
+                        "return JSON.stringify(values);",
+                        "})()",
+                    ].join(""),
+                    session.cliEnv
+                );
+                await runCliJournalMigration(objectStorage, bucketPrefix, sourceSettings);
+                await writeFile(
+                    settingsPath,
+                    JSON.stringify({
+                        ...DEFAULT_SETTINGS,
+                        ...objectStorage,
+                        ...sourceSettings,
+                        remoteType: "MINIO",
+                        bucketPrefix,
+                        pathNormalizationVersion: 1,
+                        isConfigured: true,
+                    })
+                );
+                const execute = promisify(execFile);
+                const run = async (...args: string[]) =>
+                    await execute(
+                        process.execPath,
+                        [
+                            resolve("src/apps/cli/dist/index.cjs"),
+                            databasePath,
+                            "--vault",
+                            vaultPath,
+                            "--settings",
+                            settingsPath,
+                            ...args,
+                        ],
+                        { timeout: 60_000, maxBuffer: 4 * 1024 * 1024 }
+                    );
+                await run("--force", "--write-settings", "fetch");
+                assertEqual(
+                    await readFile(join(vaultPath, notePath), "utf8"),
+                    noteContent,
+                    "CLI Fetch changed the file contents."
+                );
+                assertEqual(
+                    (await run("cat", notePath.normalize("NFC"))).stdout,
+                    noteContent,
+                    "CLI Fetch did not retain the remote Chunk content."
+                );
+                const listedPaths = (await run("ls")).stdout
+                    .trim()
+                    .split("\n")
+                    .map((line) => line.split("\t")[0]);
+                assertEqual(listedPaths.length, 1, "CLI Fetch retained duplicate Metadata.");
+                assertEqual(
+                    listedPaths[0].normalize("NFC"),
+                    notePath.normalize("NFC"),
+                    "CLI Fetch changed the logical path."
+                );
+                const fileInfo = JSON.parse((await run("info", listedPaths[0])).stdout) as { id: string };
+                assertEqual(fileInfo.id, localEntry.id, "CLI Fetch used a different Metadata ID.");
+                const expectedNames = await readdir(dirname(join(vaultPath, notePath)));
+                assertEqual(expectedNames.length, 1, "CLI Fetch created a duplicate physical file.");
+                assertEqual(
+                    expectedNames[0],
+                    notePath.split("/").slice(-1)[0],
+                    "CLI Fetch renamed the NFD physical file."
+                );
+                const editedContent = noteContent + "\nEdited after CLI restart.\n";
+                await writeFile(join(vaultPath, notePath), editedContent);
+                const nextTime = new Date(Date.now() + 5000);
+                await utimes(join(vaultPath, notePath), nextTime, nextTime);
+                await run("mirror");
+                await run("sync");
+                await pushLocalChanges(cli.binary, session.cliEnv);
+                const sourceFile = join(vault.path, localEntry.path);
+                const deadline = Date.now() + 20_000;
+                while ((await readFile(sourceFile, "utf8")) !== editedContent && Date.now() < deadline) {
+                    await new Promise((resolveWait) => setTimeout(resolveWait, 200));
+                }
+                assertEqual(
+                    await readFile(sourceFile, "utf8"),
+                    editedContent,
+                    "Obsidian did not receive the CLI edit."
+                );
+                await run("rm", notePath.normalize("NFC"));
+                await run("sync");
+                await pushLocalChanges(cli.binary, session.cliEnv);
+                const deleteDeadline = Date.now() + 20_000;
+                let exists = true;
+                while (exists && Date.now() < deleteDeadline) {
+                    exists = await readFile(sourceFile).then(
+                        () => true,
+                        (error: NodeJS.ErrnoException) => {
+                            if (error.code === "ENOENT") return false;
+                            throw error;
+                        }
+                    );
+                    if (exists) await new Promise((resolveWait) => setTimeout(resolveWait, 200));
+                }
+                assertEqual(exists, false, "Obsidian did not reflect the CLI deletion.");
+                console.log(
+                    "NFC_JOURNAL_ROUND_TRIP: canonical ID, NFD physical name, CLI Fetch, edit after process restart, and deletion verified."
+                );
+            } finally {
+                await rm(cliRoot, { recursive: true, force: true });
+            }
+        }
+
         console.log(
             `Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}`
         );