Merge pull request #1209 from vrtmrz/fix/issue-1206-colon-paths

Preserve colon-containing Vault paths with Commonlib 0.1.28

package-lock.json

@@ -23,7 +23,7 @@
                 "@smithy/types": "^4.14.3",
                 "@smithy/util-retry": "^4.4.5",
                 "@vrtmrz/browser-ui-kit": "0.1.0",
-                "@vrtmrz/livesync-commonlib": "0.1.27",
+                "@vrtmrz/livesync-commonlib": "0.1.28",
                 "@vrtmrz/obsidian-plugin-kit": "0.1.4",
                 "@vrtmrz/ui-interactions": "0.1.2",
                 "diff-match-patch": "^1.0.5",
@@ -4567,9 +4567,9 @@
             }
         },
         "node_modules/@vrtmrz/livesync-commonlib": {
-            "version": "0.1.27",
+            "version": "0.1.28",
-            "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.27.tgz",
+            "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.28.tgz",
-            "integrity": "sha512-n/AIqXPGN9Z7zmq0QSKnlE5uY5hTZgXW6VM6NJERR0/ggx3TTv1b3WsSbiLpCeU7Mu6KbPqC2BSjr4jCGvSdCA==",
+            "integrity": "sha512-3NXswTtOU+fE4KRxBWHfkMx9r2Eu9IFH9N/ibj75C5eLh1tFPH7n87bGib6La7Twz9yCDIJRBsBUc2tx81rxOQ==",
             "license": "MIT",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.808.0",

package.json

@@ -183,7 +183,7 @@
         "@smithy/types": "^4.14.3",
         "@smithy/util-retry": "^4.4.5",
         "@vrtmrz/browser-ui-kit": "0.1.0",
-        "@vrtmrz/livesync-commonlib": "0.1.27",
+        "@vrtmrz/livesync-commonlib": "0.1.28",
         "@vrtmrz/obsidian-plugin-kit": "0.1.4",
         "@vrtmrz/ui-interactions": "0.1.2",
         "diff-match-patch": "^1.0.5",

src/common/utils.path.unit.spec.ts

@@ -0,0 +1,64 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+    normalizePath: vi.fn((path: string) => `normalised(${path})`),
+    path2idBase: vi.fn(async (path: string) => path),
+    id2pathBase: vi.fn((path: string) => path),
+    expandFilePathPrefix: vi.fn((path: string): [string, string] => {
+        if (path.startsWith("i:")) return ["i:", path.substring(2)];
+        return ["", path];
+    }),
+}));
+
+vi.mock("@/deps.ts", () => ({
+    normalizePath: mocks.normalizePath,
+    Platform: {},
+    requestUrl: vi.fn(),
+}));
+
+vi.mock("@vrtmrz/livesync-commonlib/compat/string_and_binary/path", () => ({
+    path2id_base: mocks.path2idBase,
+    id2path_base: mocks.id2pathBase,
+    expandFilePathPrefix: mocks.expandFilePathPrefix,
+    isValidFilenameInLinux: vi.fn(),
+    isValidFilenameInDarwin: vi.fn(),
+    isValidFilenameInWidows: vi.fn(),
+    isValidFilenameInAndroid: vi.fn(),
+    stripAllPrefixes: vi.fn(),
+}));
+
+describe("path ID normalisation", () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+
+    it.each([
+        ["Folder/Note.md", "", "Folder/Note.md"],
+        ["Folder/Poem: Example.md", "", "Folder/Poem: Example.md"],
+        ["Folder/Poem: Example: Final Draft.md", "", "Folder/Poem: Example: Final Draft.md"],
+        ["i:Folder/Poem: Example.md", "i:", "Folder/Poem: Example.md"],
+    ])("normalises the complete path body for %s", async (filename, prefix, body) => {
+        const { path2id } = await import("./utils.ts");
+
+        const result = await path2id(filename as never, false, false);
+
+        expect(mocks.normalizePath).toHaveBeenCalledWith(body);
+        expect(mocks.path2idBase).toHaveBeenCalledWith(`${prefix}normalised(${body})`, false, false);
+        expect(result).toBe(`${prefix}normalised(${body})`);
+    });
+
+    it.each([
+        ["Folder/Note.md", "", "Folder/Note.md"],
+        ["Folder/Poem: Example.md", "", "Folder/Poem: Example.md"],
+        ["Folder/Poem: Example: Final Draft.md", "", "Folder/Poem: Example: Final Draft.md"],
+        ["i:Folder/Poem: Example.md", "i:", "Folder/Poem: Example.md"],
+    ])("preserves the path namespace while normalising %s", async (filename, prefix, body) => {
+        mocks.id2pathBase.mockReturnValue(filename);
+        const { id2path } = await import("./utils.ts");
+
+        const result = id2path(filename as never);
+
+        expect(mocks.normalizePath).toHaveBeenCalledWith(body);
+        expect(result).toBe(`${prefix}normalised(${body})`);
+    });
+});

src/common/utils.ts

@@ -7,6 +7,7 @@ import {
     isValidFilenameInWidows,
     isValidFilenameInAndroid,
     stripAllPrefixes,
+    expandFilePathPrefix,
 } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
 
 import { Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger";
@@ -41,22 +42,18 @@ export async function path2id(
     obfuscatePassphrase: string | false,
     caseInsensitive: boolean
 ): Promise<DocumentID> {
-    const temp = filename.split(":");
+    const [prefix, path] = expandFilePathPrefix(filename);
-    const path = temp.pop();
     const normalizedPath = normalizePath(path as FilePath);
-    temp.push(normalizedPath);
+    const fixedPath = `${prefix}${normalizedPath}` as FilePathWithPrefix;
-    const fixedPath = temp.join(":") as FilePathWithPrefix;
 
     const out = await path2id_base(fixedPath, obfuscatePassphrase, caseInsensitive);
     return out;
 }
 export function id2path(id: DocumentID, entry?: EntryHasPath): FilePathWithPrefix {
     const filename = id2path_base(id, entry);
-    const temp = filename.split(":");
+    const [prefix, path] = expandFilePathPrefix(filename);
-    const path = temp.pop();
     const normalizedPath = normalizePath(path as FilePath);
-    temp.push(normalizedPath);
+    const fixedPath = `${prefix}${normalizedPath}` as FilePathWithPrefix;
-    const fixedPath = temp.join(":") as FilePathWithPrefix;
     return fixedPath;
 }
 

src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts

@@ -1,7 +1,11 @@
 import { promiseWithResolvers } from "octagonal-wheels/promises";
 import { reactiveSource } from "octagonal-wheels/dataobject/reactive";
 import { describe, expect, it, vi } from "vitest";
-import { VER, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { VER, type EntryDoc, type FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import {
+    isValidFilenameInAndroid,
+    isValidFilenameInWidows,
+} from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
 import {
     defaultLogger,
     LOG_LEVEL_DEBUG,
@@ -28,6 +32,7 @@ function note(id: string): PouchDB.Core.ExistingDocument<EntryDoc> {
 
 type SetupOptions = {
     applicationReady?: boolean;
+    isValidPath?: (path: string) => boolean;
     processSynchroniseResult?: (entry: unknown) => Promise<boolean>;
     setSnapshot?: (key: string, value: unknown) => Promise<unknown>;
 };
@@ -38,6 +43,8 @@ function setup(options: SetupOptions = {}) {
     const runBoundedLocalApplicationActivity = vi.fn(async (task: () => Promise<void>) => await task());
     const onCloseActiveReplication = vi.fn(async () => true);
     const isReady = vi.fn(() => options.applicationReady ?? true);
+    const isValidPath = vi.fn(options.isValidPath ?? (() => true));
+    const getDBEntryFromMeta = vi.fn(async (entry: object) => ({ ...entry, data: "x" }));
     const core = {
         services: {
             appLifecycle: { isReady, isSuspended: () => false },
@@ -54,13 +61,13 @@ function setup(options: SetupOptions = {}) {
             vault: {
                 isTargetFile: vi.fn(async () => true),
                 isFileSizeTooLarge: vi.fn(() => false),
-                isValidPath: vi.fn(() => true),
+                isValidPath,
             },
         },
         kvDB: { set: setSnapshot },
         localDatabase: {
             getRaw: vi.fn(async (id: string) => ({ _id: id, _rev: "1-test" })),
-            getDBEntryFromMeta: vi.fn(async (entry: object) => ({ ...entry, data: "x" })),
+            getDBEntryFromMeta,
         },
     };
     const processor = new ReplicateResultProcessor({
@@ -74,7 +81,9 @@ function setup(options: SetupOptions = {}) {
         services: core.services,
     } as never);
     return {
+        getDBEntryFromMeta,
         isReady,
+        isValidPath,
         onCloseActiveReplication,
         processor,
         processSynchroniseResult,
@@ -83,6 +92,24 @@ function setup(options: SetupOptions = {}) {
 }
 
 describe("ReplicateResultProcessor", () => {
+    it.each([
+        ["Windows", isValidFilenameInWidows],
+        ["Android", isValidFilenameInAndroid],
+    ])("does not reflect a replicated colon path into the %s Vault", async (_platform, validatePath) => {
+        const path = "Folder/Poem: Example.md" as FilePathWithPrefix;
+        const document = { ...note("colon-path"), path };
+        const { getDBEntryFromMeta, isValidPath, processor, processSynchroniseResult } = setup({
+            isValidPath: validatePath,
+        });
+
+        processor.enqueueAll([document]);
+
+        await vi.waitFor(() => expect(isValidPath).toHaveBeenCalledWith(path));
+        await vi.waitFor(() => expect(processor["_processingChanges"]).toHaveLength(0));
+        expect(getDBEntryFromMeta).toHaveBeenCalledWith(expect.objectContaining({ path }), false, true);
+        expect(processSynchroniseResult).not.toHaveBeenCalled();
+    });
+
     it("resumes another document after in-flight updates to one document fill the application slots", async () => {
         const hotGate = promiseWithResolvers<boolean>();
         const { processor, processSynchroniseResult } = setup({

test/e2e-obsidian/README.md

@@ -82,7 +82,7 @@ npm run test:e2e:obsidian:focused -- security-seed-reconnect
 
 The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite.
 
-`folder-batch` needs no remote service. It creates 24 notes in nested folders, renames and deletes the parent through the Obsidian Vault API, and checks descendant events, content, Chunks, deletion markers, and provenance. A note outside the parent must remain writable.
+`folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems.
 
 `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file with exact provenance receives the newer database content without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. The database advance and pending snapshot are controlled fixtures; startup processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application.
 

test/e2e-obsidian/scripts/folder-batch.ts

@@ -8,11 +8,17 @@ process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000";
 const originalRoot = "batch/original";
 const renamedRoot = "batch/renamed";
 const outsidePath = "batch/outside.md";
+const missingColonPath = "batch/incoming/Poem: Example.md";
 const folders = ["alpha", "alpha/deep", "beta"];
 const notes = Array.from({ length: 24 }, (_, index) => ({
     relativePath: `${folders[index % folders.length]}/note-${index}.md`,
     body: `# Descendant ${index}\n\nThis body must survive a parent folder rename.\n`,
 }));
+notes.push(
+    { relativePath: "alpha/Poem: Example.md", body: "First poem\n" },
+    { relativePath: "beta/Poem: Example.md", body: "Second poem\n" },
+    { relativePath: "alpha/deep/Poem: Part: Example.md", body: "Poem with multiple colons\n" }
+);
 
 async function main(): Promise<void> {
     const binary = requireObsidianBinary();
@@ -45,7 +51,7 @@ async function main(): Promise<void> {
             localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
         });
         await waitForLiveSyncCoreReady(cliBinary, session.cliEnv);
-        const result = await evalObsidianJson<{ descendants: number; renamed: number; deleted: number }>(
+        const result = await evalObsidianJson<{ descendants: number; renamed: number; deleted: number; missingRejected: boolean }>(
             cliBinary,
             `(async()=>{
                 const core=app.plugins.plugins['obsidian-livesync'].core;
@@ -54,6 +60,7 @@ async function main(): Promise<void> {
                 const originalRoot=${JSON.stringify(originalRoot)};
                 const renamedRoot=${JSON.stringify(renamedRoot)};
                 const outsidePath=${JSON.stringify(outsidePath)};
+                const missingColonPath=${JSON.stringify(missingColonPath)};
                 const renamed=new Set(), deleted=new Set();
                 const refs=[
                     app.vault.on('rename',(file,oldPath)=>{
@@ -112,11 +119,23 @@ async function main(): Promise<void> {
                     await app.vault.createFolder(originalRoot);
                     for(const folder of ${JSON.stringify(folders)})
                         await app.vault.createFolder(originalRoot+'/'+folder);
-                    await Promise.all(notes.map(note=>app.vault.create(originalRoot+'/'+note.relativePath,note.body)));
+                    // Obsidian indexes imported colon names but rejects them in Vault.create.
+                    await Promise.all(notes.map(note=>note.relativePath.includes(':')
+                        ? app.vault.adapter.write(originalRoot+'/'+note.relativePath,note.body)
+                        : app.vault.create(originalRoot+'/'+note.relativePath,note.body)));
                     await app.vault.create(outsidePath,'Outside note');
                     await waitFor('Initial batch',async()=>[
                         ...await liveBatch(originalRoot), ...await liveErrors(outsidePath,'Outside note'),
                     ]);
+                    for(const note of notes){
+                        const path=originalRoot+'/'+note.relativePath;
+                        if(!await core.serviceModules.fileHandler.dbToStorage(await meta(path),null,true))
+                            throw new Error('Database reflection failed: '+path);
+                    }
+                    await waitFor('Reflected batch',()=>liveBatch(originalRoot));
+                    const expectedPaths=new Set([outsidePath,...notes.map(note=>originalRoot+'/'+note.relativePath)]);
+                    const unexpected=app.vault.getFiles().map(file=>file.path).filter(path=>!expectedPaths.has(path));
+                    if(unexpected.length) throw new Error('Unexpected reflected files: '+unexpected.join(', '));
                     const originalIds=await Promise.all(notes.map(async note=>(await meta(originalRoot+'/'+note.relativePath))._id));
 
                     // Rename the parent once: Obsidian must emit every descendant event.
@@ -144,7 +163,40 @@ async function main(): Promise<void> {
                     if(app.vault.getAbstractFileByPath(renamedRoot)) throw new Error('Deleted folder remains');
                     await app.vault.modify(app.vault.getAbstractFileByPath(outsidePath),'Outside note updated');
                     await waitFor('Outside update',()=>liveErrors(outsidePath,'Outside note updated'));
-                    return JSON.stringify({descendants:notes.length,renamed:renamed.size,deleted:deleted.size});
+
+                    // A received database entry must not create a different Vault file when Obsidian rejects its name.
+                    const filesBefore=new Set(app.vault.getFiles().map(file=>file.path));
+                    const incomingBody='Received colon note\\n';
+                    const incomingData=new Blob([incomingBody],{type:'text/plain'});
+                    const incomingId=await core.services.path.path2id(missingColonPath);
+                    const incomingTime=Date.now();
+                    const saved=await core.localDatabase.putDBEntry({
+                        _id:incomingId,path:missingColonPath,data:incomingData,
+                        ctime:incomingTime,mtime:incomingTime,size:incomingData.size,
+                        children:[],datatype:'plain',type:'plain',eden:{},
+                    });
+                    if(!saved?.ok) throw new Error('Could not seed received Metadata: '+missingColonPath);
+                    const incomingMeta=await meta(missingColonPath);
+                    if(!incomingMeta || incomingMeta._id!==incomingId || incomingMeta.path!==missingColonPath)
+                        throw new Error('Received Metadata has the wrong path: '+missingColonPath);
+                    const incomingEntry=await core.localDatabase.getDBEntry(missingColonPath,{rev:incomingMeta._rev},false,true,true);
+                    if(!incomingEntry || getContent(incomingEntry)!==incomingBody)
+                        throw new Error('Received content could not be read: '+missingColonPath);
+                    let creationFailure='';
+                    try{
+                        const reflected=await core.serviceModules.fileHandler.dbToStorage(incomingMeta,null,true);
+                        if(reflected) throw new Error('Obsidian unexpectedly created: '+missingColonPath);
+                    }catch(error){
+                        creationFailure=String(error);
+                        if(!creationFailure.includes('File name cannot contain')) throw error;
+                    }
+                    if(!creationFailure) throw new Error('Missing name rejection: '+missingColonPath);
+                    const filesAfter=app.vault.getFiles().map(file=>file.path);
+                    const newFiles=filesAfter.filter(path=>!filesBefore.has(path));
+                    if(newFiles.length) throw new Error('Received note was written under another name: '+newFiles.join(', '));
+                    if((await meta(missingColonPath))?.path!==missingColonPath)
+                        throw new Error('Received Metadata changed after rejection: '+missingColonPath);
+                    return JSON.stringify({descendants:notes.length,renamed:renamed.size,deleted:deleted.size,missingRejected:true});
                 }finally{
                     for(const ref of refs) app.vault.offref(ref);
                 }
@@ -153,7 +205,8 @@ async function main(): Promise<void> {
         );
         console.log(
             `Folder batch: ${result.descendants} descendants persisted, renamed, and deleted; ` +
-                `${result.renamed} rename and ${result.deleted} delete events observed; outside note remained writable.`
+                `${result.renamed} rename and ${result.deleted} delete events observed; outside note remained writable; ` +
+                `missing colon note rejected without an alternate file: ${result.missingRejected}.`
         );
     } finally {
         if (session) await session.app.stop();

updates.md

@@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
 
 ## Unreleased
 
+### Synchronisation and storage
+
+#### Fixed
+
+- Files with colons in their names now retain their full paths in synchronisation data instead of appearing as incorrectly named copies at the Vault root. (#1206)
+    - Obsidian may refuse to create a missing file with such a name. LiveSync also treats these names as invalid on Windows and Android, so the file may not appear in those devices' Vaults. Existing misplaced copies are left for you to review; this change does not remove them automatically.
+
 ## 1.0.30
 
 18th September, 2026