Expose NFC path identity across supported transports

test/e2e-obsidian/runner/cliJournalMigration.ts

@@ -0,0 +1,243 @@
+import { deepStrictEqual } from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { promisify } from "node:util";
+import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { assertEqual } from "./liveSyncWorkflow.ts";
+import { deleteObjectStoragePrefix, readObjectStorageJson, type ObjectStorageConfig } from "./objectStorage.ts";
+
+const notePath = "Cafe\u0301/re\u0301sume\u0301.md";
+const noteContent = "# CLI journal migration\n\nThe NFD physical path must keep its identity during migration.\n";
+
+type CliFileInfo = {
+    id: string;
+    revision: string;
+    path: string;
+};
+
+type JournalMilestone = {
+    node_chunk_info?: Record<string, { current?: number }>;
+};
+
+type CliResult = {
+    stdout: string;
+    stderr: string;
+};
+
+type CliRunner = (...args: string[]) => Promise<CliResult>;
+
+function cliErrorOutput(error: unknown): string {
+    if (error === null || typeof error !== "object") {
+        return String(error);
+    }
+    const details = error as { message?: unknown; stdout?: unknown; stderr?: unknown };
+    return [details.message, details.stdout, details.stderr]
+        .filter((value): value is string => typeof value === "string")
+        .join("\n");
+}
+
+async function readSingleMetadata(run: CliRunner, label: string): Promise<CliFileInfo> {
+    const output = (await run("ls")).stdout.trimEnd();
+    const paths = output === "" ? [] : output.split(/\r?\n/u).map((line) => line.split("\t", 1)[0] ?? "");
+    assertEqual(paths.length, 1, `${label} did not contain exactly one Metadata document.`);
+    const storedPath = paths[0];
+    if (!storedPath) {
+        throw new Error(`${label} did not report a stored path.`);
+    }
+    return JSON.parse((await run("info", storedPath)).stdout) as CliFileInfo;
+}
+
+export async function runCliJournalMigration(
+    objectStorage: ObjectStorageConfig,
+    bucketPrefix: string,
+    sourceSettings: Record<string, unknown>
+): Promise<void> {
+    const migrationPrefix = `${bucketPrefix}migration/`;
+    const cliRoot = await mkdtemp(join(tmpdir(), "livesync-cli-journal-migration-"));
+    const sourceRoot = join(cliRoot, "source");
+    const targetRoot = join(cliRoot, "target");
+    const sourceDatabasePath = join(sourceRoot, "database");
+    const targetDatabasePath = join(targetRoot, "database");
+    const sourceVaultPath = join(sourceRoot, "vault");
+    const targetVaultPath = join(targetRoot, "vault");
+    const sourceSettingsPath = join(sourceRoot, "settings.json");
+    const targetSettingsPath = join(targetRoot, "settings.json");
+    const physicalParent = notePath.split("/")[0] ?? "";
+    const physicalFilename = notePath.split("/").slice(-1)[0] ?? "";
+
+    const writeCliSettings = async (settingsPath: string, pathNormalizationVersion: number) => {
+        const previous = await readFile(settingsPath, "utf8").then(
+            (text) => JSON.parse(text) as Record<string, unknown>,
+            (error: NodeJS.ErrnoException) => {
+                if (error.code !== "ENOENT") throw error;
+                return undefined;
+            }
+        );
+        await writeFile(
+            settingsPath,
+            JSON.stringify({
+                ...(previous ?? {
+                    ...DEFAULT_SETTINGS,
+                    ...objectStorage,
+                    ...sourceSettings,
+                    remoteType: "MINIO",
+                    bucketPrefix: migrationPrefix,
+                    isConfigured: true,
+                }),
+                pathNormalizationVersion,
+                liveSync: false,
+                syncOnStart: false,
+                syncOnSave: false,
+                syncOnEditorSave: false,
+                syncOnFileOpen: false,
+                syncAfterMerge: false,
+                periodicReplication: false,
+            })
+        );
+    };
+
+    try {
+        await mkdir(sourceDatabasePath, { recursive: true });
+        await mkdir(targetDatabasePath, { recursive: true });
+        await mkdir(dirname(join(sourceVaultPath, notePath)), { recursive: true });
+        await mkdir(dirname(join(targetVaultPath, notePath)), { recursive: true });
+        await writeFile(join(sourceVaultPath, notePath), noteContent);
+        await writeFile(join(targetVaultPath, notePath), noteContent);
+        await writeCliSettings(sourceSettingsPath, 0);
+        await writeCliSettings(targetSettingsPath, 0);
+
+        const execute = promisify(execFile);
+        const createRunner =
+            (databasePath: string, vaultPath: string, settingsPath: string): CliRunner =>
+            async (...args: string[]) =>
+                await execute(
+                    process.execPath,
+                    [
+                        resolve("src/apps/cli/dist/index.cjs"),
+                        databasePath,
+                        "--vault",
+                        vaultPath,
+                        "--settings",
+                        settingsPath,
+                        ...args,
+                    ],
+                    { timeout: 60_000, maxBuffer: 4 * 1024 * 1024 }
+                );
+
+        const source = createRunner(sourceDatabasePath, sourceVaultPath, sourceSettingsPath);
+        const target = createRunner(targetDatabasePath, targetVaultPath, targetSettingsPath);
+
+        await source("--force", "--write-settings", "rebuild");
+        await target("mirror");
+
+        const legacySource = await readSingleMetadata(source, "Legacy CLI source");
+        const legacyTarget = await readSingleMetadata(target, "Legacy CLI target");
+        const targetVaultEntriesBeforeFetch = await readdir(targetVaultPath);
+        const targetParentEntriesBeforeFetch = await readdir(dirname(join(targetVaultPath, notePath)));
+        assertEqual(
+            targetVaultEntriesBeforeFetch.includes(physicalParent),
+            true,
+            "The legacy CLI target did not retain the NFD physical parent."
+        );
+        assertEqual(
+            targetParentEntriesBeforeFetch.includes(physicalFilename),
+            true,
+            "The legacy CLI target did not retain the NFD physical filename."
+        );
+
+        await writeCliSettings(sourceSettingsPath, 1);
+        await source("--force", "--write-settings", "rebuild");
+
+        const migratedSource = await readSingleMetadata(source, "Migrated CLI source");
+        if (migratedSource.id === legacySource.id) {
+            throw new Error("The NFC migration did not change the source Metadata ID.");
+        }
+        assertEqual(
+            (await source("cat", notePath.normalize("NFC"))).stdout,
+            noteContent,
+            "The migrated CLI source did not retain the canonical Chunk content."
+        );
+
+        const milestoneKey = `${migrationPrefix}_00000000-milestone.json`;
+        const milestoneBeforeFailedFetch = await readObjectStorageJson<JournalMilestone>(objectStorage, milestoneKey);
+        const migrationVersions = Object.values(milestoneBeforeFailedFetch.node_chunk_info ?? {}).map(
+            (node) => node.current
+        );
+        assertEqual(migrationVersions.includes(4), true, "The migrated Journal milestone did not declare version 4.");
+
+        let legacyFetchError: unknown;
+        try {
+            await target("--force", "--write-settings", "fetch");
+        } catch (error) {
+            legacyFetchError = error;
+        }
+        if (legacyFetchError === undefined) {
+            throw new Error("The legacy CLI target unexpectedly fetched a Journal with NFC path identity.");
+        }
+        const legacyFetchOutput = cliErrorOutput(legacyFetchError);
+        if (!/path identity/iu.test(legacyFetchOutput)) {
+            throw new Error(
+                `The legacy CLI Fetch failed without the expected path identity error. Output: ${legacyFetchOutput}`
+            );
+        }
+
+        const targetAfterFailedFetch = await readSingleMetadata(target, "Legacy CLI target after failed Fetch");
+        assertEqual(
+            targetAfterFailedFetch.id,
+            legacyTarget.id,
+            "The failed legacy Fetch changed the target Metadata ID."
+        );
+        assertEqual(
+            targetAfterFailedFetch.revision,
+            legacyTarget.revision,
+            "The failed legacy Fetch changed the target Metadata revision."
+        );
+        const milestoneAfterFailedFetch = await readObjectStorageJson<JournalMilestone>(objectStorage, milestoneKey);
+        deepStrictEqual(
+            milestoneAfterFailedFetch,
+            milestoneBeforeFailedFetch,
+            "The failed legacy Fetch changed the remote Journal milestone."
+        );
+
+        await writeCliSettings(targetSettingsPath, 1);
+        await target("--force", "--write-settings", "fetch");
+
+        const migratedTarget = await readSingleMetadata(target, "Migrated CLI target");
+        assertEqual(migratedTarget.id, migratedSource.id, "The CLI source and target Metadata IDs did not converge.");
+        assertEqual(
+            (await target("cat", notePath.normalize("NFC"))).stdout,
+            noteContent,
+            "The migrated CLI target did not retain the canonical Chunk content."
+        );
+        assertEqual(
+            await readFile(join(targetVaultPath, notePath), "utf8"),
+            noteContent,
+            "The migrated CLI Fetch changed the NFD physical file contents."
+        );
+
+        const targetVaultEntriesAfterFetch = await readdir(targetVaultPath);
+        const targetParentEntriesAfterFetch = await readdir(dirname(join(targetVaultPath, notePath)));
+        deepStrictEqual(
+            targetVaultEntriesAfterFetch,
+            targetVaultEntriesBeforeFetch,
+            "The migrated CLI Fetch changed the physical Vault parent names."
+        );
+        deepStrictEqual(
+            targetParentEntriesAfterFetch,
+            targetParentEntriesBeforeFetch,
+            "The migrated CLI Fetch changed the physical NFD filename."
+        );
+
+        console.log(
+            "CLI_JOURNAL_MIGRATION: legacy Fetch rejection, safe migration, and NFD physical path preservation verified."
+        );
+    } finally {
+        try {
+            await rm(cliRoot, { recursive: true, force: true });
+        } finally {
+            await deleteObjectStoragePrefix(objectStorage, migrationPrefix);
+        }
+    }
+}

test/e2e-obsidian/runner/remoteActivity.ts

@@ -34,6 +34,7 @@ export type RemoteActivitySnapshot = {
     responseCount: number;
     statusBarFound: boolean;
     statusBarText: string;
+    indicatorElements?: Array<{ text: string; inactive: boolean; ariaHidden: string | null; display: string }>;
 };
 
 export type ExpectedRemoteActivityState =
@@ -88,14 +89,26 @@ async function readRemoteActivitySnapshotFromPage(page: Page): Promise<RemoteAct
                 gateKind: gate?.kind,
                 requestCount: Number(core.services?.API?.requestCount?.value ?? -1),
                 remoteOperationIndicatorVisible: statusBars.some((element) =>
-                    (element.textContent ?? "").includes(operationIcon)
-                ),
-                remoteRequestIndicatorVisible: statusBars.some((element) =>
-                    (element.textContent ?? "").includes(requestIcon)
+                    element.innerText.includes(operationIcon)
                 ),
+                remoteRequestIndicatorVisible: statusBars.some((element) => element.innerText.includes(requestIcon)),
                 responseCount: Number(core.services?.API?.responseCount?.value ?? -1),
                 statusBarFound: statusBars.length > 0,
-                statusBarText: statusBars.map((element) => element.textContent ?? "").join("\n"),
+                statusBarText: statusBars.map((element) => element.innerText).join("\n"),
+                indicatorElements: Array.from(
+                    document.querySelectorAll<HTMLElement>(".syncstatusbar .livesync-stat-item")
+                )
+                    .filter(
+                        (element) =>
+                            (element.textContent ?? "").includes(operationIcon) ||
+                            (element.textContent ?? "").includes(requestIcon)
+                    )
+                    .map((element) => ({
+                        text: element.textContent ?? "",
+                        inactive: element.classList.contains("livesync-stat-inactive"),
+                        ariaHidden: element.getAttribute("aria-hidden"),
+                        display: getComputedStyle(element).display,
+                    })),
             } satisfies RemoteActivitySnapshot;
         },
         {
@@ -141,7 +154,7 @@ export async function waitForRemoteActivityState(
                     if (!core) return false;
                     const gate = (globalThis as unknown as Record<string, RuntimeGate | undefined>)[stateKey];
                     const statusBarText = Array.from(document.querySelectorAll<HTMLElement>(".syncstatusbar"))
-                        .map((element) => element.textContent ?? "")
+                        .map((element) => element.innerText)
                         .join("\n");
                     const bounded = Number(core.services?.replicator?.boundedRemoteActivityCount?.value ?? -1);
                     const finite = Number(core.services?.replicator?.finiteReplicationActivityCount?.value ?? -1);

test/e2e-obsidian/scripts/minio-upload.ts

@@ -16,8 +16,15 @@
  * same upload.
  */
 import { randomBytes } from "node:crypto";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { promisify } from "node:util";
+import { DEFAULT_SETTINGS, TweakValuesTemplate } from "@vrtmrz/livesync-commonlib/compat/common/types";
 import { deriveIdKey } from "@vrtmrz/livesync-commonlib/settings";
 import { evalObsidianJson } from "../runner/cli.ts";
+import { runCliJournalMigration } from "../runner/cliJournalMigration.ts";
 import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
 import {
     assertEqual,
@@ -43,7 +50,8 @@ import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../r
 
 process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
 
-const notePath = "E2E/minio-upload.md";
+const unicodePaths = process.env.E2E_OBSIDIAN_UNICODE_PATHS === "true";
+const notePath = unicodePaths ? "Cafe\u0301/re\u0301sume\u0301.md" : "E2E/minio-upload.md";
 const useIndependentIds = process.env.E2E_OBSIDIAN_INDEPENDENT_IDS === "true";
 const useCustomRequestHandler = process.env.E2E_OBSIDIAN_CUSTOM_HTTP_HANDLER === "true";
 const noteContent = [
@@ -63,10 +71,11 @@ async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.Proces
             `const path=${JSON.stringify(notePath)};`,
             `const content=${JSON.stringify(noteContent)};`,
             "const core=app.plugins.plugins['obsidian-livesync'].core;",
-            "if(!(await app.vault.adapter.exists('E2E'))) await app.vault.createFolder('E2E');",
+            "const parent=path.slice(0,path.lastIndexOf('/'));",
+            "if(!(await app.vault.adapter.exists(parent))) await app.vault.createFolder(parent);",
             "const existing=app.vault.getAbstractFileByPath(path);",
             "if(existing) await app.vault.delete(existing);",
-            "await app.vault.create(path,content);",
+            "const createdFile=await app.vault.create(path,content);",
             "const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));",
             "let entry=false;",
             "for(let i=0;i<40;i++){",
@@ -76,7 +85,7 @@ async function createNoteAndWaitForLocalDb(cliBinary: string, env: NodeJS.Proces
             "await sleep(250);",
             "}",
             "if(!entry||!entry._id) throw new Error('Timed out waiting for local database entry');",
-            "return JSON.stringify({id:entry._id,path:entry.path,type:entry.type,children:entry.children||[]});",
+            "return JSON.stringify({id:entry._id,path:createdFile.path,type:entry.type,children:entry.children||[]});",
             "})()",
         ].join(""),
         env
@@ -124,10 +133,13 @@ async function main(): Promise<void> {
             cliBinary: cli.binary,
             vault,
             startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
-            pluginData: createE2eObjectStoragePluginData({
-                ...objectStorage,
-                bucketPrefix,
-            }),
+            pluginData: createE2eObjectStoragePluginData(
+                {
+                    ...objectStorage,
+                    bucketPrefix,
+                },
+                { pathNormalizationVersion: unicodePaths ? 1 : 0 }
+            ),
             localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
         });
         await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
@@ -137,6 +149,7 @@ async function main(): Promise<void> {
             session.cliEnv,
             { ...objectStorage, bucketPrefix },
             {
+                pathNormalizationVersion: unicodePaths ? 1 : 0,
                 ...(useIndependentIds
                     ? {
                           encrypt: true,
@@ -173,6 +186,13 @@ async function main(): Promise<void> {
                 );
             }
         }
+        if (unicodePaths && !useIndependentIds) {
+            assertEqual(
+                localEntry.id,
+                notePath.normalize("NFC").toLowerCase(),
+                "Obsidian did not derive the canonical Metadata ID."
+            );
+        }
         await pushLocalChanges(cli.binary, session.cliEnv);
         const activityAfterUpload = await waitForRemoteActivityState(
             session.remoteDebuggingPort,
@@ -198,6 +218,131 @@ async function main(): Promise<void> {
             }
         }
 
+        if (unicodePaths) {
+            const cliRoot = await mkdtemp(join(tmpdir(), "livesync-nfc-journal-"));
+            try {
+                const databasePath = join(cliRoot, "db");
+                const vaultPath = join(cliRoot, "vault");
+                const settingsPath = join(cliRoot, "settings.json");
+                await mkdir(databasePath, { recursive: true });
+                await mkdir(dirname(join(vaultPath, notePath)), { recursive: true });
+                await writeFile(join(vaultPath, notePath), noteContent);
+                const sourceSettings = await evalObsidianJson<Record<string, unknown>>(
+                    cli.binary,
+                    [
+                        "(async()=>{",
+                        "const service=app.plugins.plugins['obsidian-livesync'].core.services.setting;",
+                        "const s=service.currentSettings();",
+                        `const keys=${JSON.stringify([...Object.keys(TweakValuesTemplate), "encrypt", "passphrase", "usePathObfuscation", "E2EEAlgorithm", "idDerivationKey"])};`,
+                        "const values=Object.fromEntries(keys.map(key=>[key,s[key]]));",
+                        "if(s.idDerivationVersion===1){",
+                        "values.encryptedIdDerivationKey=await service.encryptConfigurationItem(s.idDerivationKey,s,'*');",
+                        "values.idDerivationKey='';values.configPassphraseStore='';",
+                        "}",
+                        "return JSON.stringify(values);",
+                        "})()",
+                    ].join(""),
+                    session.cliEnv
+                );
+                await runCliJournalMigration(objectStorage, bucketPrefix, sourceSettings);
+                await writeFile(
+                    settingsPath,
+                    JSON.stringify({
+                        ...DEFAULT_SETTINGS,
+                        ...objectStorage,
+                        ...sourceSettings,
+                        remoteType: "MINIO",
+                        bucketPrefix,
+                        pathNormalizationVersion: 1,
+                        isConfigured: true,
+                    })
+                );
+                const execute = promisify(execFile);
+                const run = async (...args: string[]) =>
+                    await execute(
+                        process.execPath,
+                        [
+                            resolve("src/apps/cli/dist/index.cjs"),
+                            databasePath,
+                            "--vault",
+                            vaultPath,
+                            "--settings",
+                            settingsPath,
+                            ...args,
+                        ],
+                        { timeout: 60_000, maxBuffer: 4 * 1024 * 1024 }
+                    );
+                await run("--force", "--write-settings", "fetch");
+                assertEqual(
+                    await readFile(join(vaultPath, notePath), "utf8"),
+                    noteContent,
+                    "CLI Fetch changed the file contents."
+                );
+                assertEqual(
+                    (await run("cat", notePath.normalize("NFC"))).stdout,
+                    noteContent,
+                    "CLI Fetch did not retain the remote Chunk content."
+                );
+                const listedPaths = (await run("ls")).stdout
+                    .trim()
+                    .split("\n")
+                    .map((line) => line.split("\t")[0]);
+                assertEqual(listedPaths.length, 1, "CLI Fetch retained duplicate Metadata.");
+                assertEqual(
+                    listedPaths[0].normalize("NFC"),
+                    notePath.normalize("NFC"),
+                    "CLI Fetch changed the logical path."
+                );
+                const fileInfo = JSON.parse((await run("info", listedPaths[0])).stdout) as { id: string };
+                assertEqual(fileInfo.id, localEntry.id, "CLI Fetch used a different Metadata ID.");
+                const expectedNames = await readdir(dirname(join(vaultPath, notePath)));
+                assertEqual(expectedNames.length, 1, "CLI Fetch created a duplicate physical file.");
+                assertEqual(
+                    expectedNames[0],
+                    notePath.split("/").slice(-1)[0],
+                    "CLI Fetch renamed the NFD physical file."
+                );
+                const editedContent = noteContent + "\nEdited after CLI restart.\n";
+                await writeFile(join(vaultPath, notePath), editedContent);
+                const nextTime = new Date(Date.now() + 5000);
+                await utimes(join(vaultPath, notePath), nextTime, nextTime);
+                await run("mirror");
+                await run("sync");
+                await pushLocalChanges(cli.binary, session.cliEnv);
+                const sourceFile = join(vault.path, localEntry.path);
+                const deadline = Date.now() + 20_000;
+                while ((await readFile(sourceFile, "utf8")) !== editedContent && Date.now() < deadline) {
+                    await new Promise((resolveWait) => setTimeout(resolveWait, 200));
+                }
+                assertEqual(
+                    await readFile(sourceFile, "utf8"),
+                    editedContent,
+                    "Obsidian did not receive the CLI edit."
+                );
+                await run("rm", notePath.normalize("NFC"));
+                await run("sync");
+                await pushLocalChanges(cli.binary, session.cliEnv);
+                const deleteDeadline = Date.now() + 20_000;
+                let exists = true;
+                while (exists && Date.now() < deleteDeadline) {
+                    exists = await readFile(sourceFile).then(
+                        () => true,
+                        (error: NodeJS.ErrnoException) => {
+                            if (error.code === "ENOENT") return false;
+                            throw error;
+                        }
+                    );
+                    if (exists) await new Promise((resolveWait) => setTimeout(resolveWait, 200));
+                }
+                assertEqual(exists, false, "Obsidian did not reflect the CLI deletion.");
+                console.log(
+                    "NFC_JOURNAL_ROUND_TRIP: canonical ID, NFD physical name, CLI Fetch, edit after process restart, and deletion verified."
+                );
+            } finally {
+                await rm(cliRoot, { recursive: true, force: true });
+            }
+        }
+
         console.log(
             `Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}`
         );