diff --git a/package-lock.json b/package-lock.json index acdb3c1d..01b9f1d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.27", + "@vrtmrz/livesync-commonlib": "0.1.28", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", @@ -4567,9 +4567,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.1.27", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.27.tgz", - "integrity": "sha512-n/AIqXPGN9Z7zmq0QSKnlE5uY5hTZgXW6VM6NJERR0/ggx3TTv1b3WsSbiLpCeU7Mu6KbPqC2BSjr4jCGvSdCA==", + "version": "0.1.28", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.1.28.tgz", + "integrity": "sha512-3NXswTtOU+fE4KRxBWHfkMx9r2Eu9IFH9N/ibj75C5eLh1tFPH7n87bGib6La7Twz9yCDIJRBsBUc2tx81rxOQ==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index 2557723b..f2e8dc1b 100644 --- a/package.json +++ b/package.json @@ -183,7 +183,7 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.1.27", + "@vrtmrz/livesync-commonlib": "0.1.28", "@vrtmrz/obsidian-plugin-kit": "0.1.4", "@vrtmrz/ui-interactions": "0.1.2", "diff-match-patch": "^1.0.5", diff --git a/src/common/utils.path.unit.spec.ts b/src/common/utils.path.unit.spec.ts new file mode 100644 index 00000000..2f008b86 --- /dev/null +++ b/src/common/utils.path.unit.spec.ts @@ -0,0 +1,64 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + normalizePath: vi.fn((path: string) => `normalised(${path})`), + path2idBase: vi.fn(async (path: string) => path), + id2pathBase: vi.fn((path: string) => path), + expandFilePathPrefix: vi.fn((path: string): [string, string] => { + if (path.startsWith("i:")) return ["i:", path.substring(2)]; + return ["", path]; + }), +})); + +vi.mock("@/deps.ts", () => ({ + normalizePath: mocks.normalizePath, + Platform: {}, + requestUrl: vi.fn(), +})); + +vi.mock("@vrtmrz/livesync-commonlib/compat/string_and_binary/path", () => ({ + path2id_base: mocks.path2idBase, + id2path_base: mocks.id2pathBase, + expandFilePathPrefix: mocks.expandFilePathPrefix, + isValidFilenameInLinux: vi.fn(), + isValidFilenameInDarwin: vi.fn(), + isValidFilenameInWidows: vi.fn(), + isValidFilenameInAndroid: vi.fn(), + stripAllPrefixes: vi.fn(), +})); + +describe("path ID normalisation", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it.each([ + ["Folder/Note.md", "", "Folder/Note.md"], + ["Folder/Poem: Example.md", "", "Folder/Poem: Example.md"], + ["Folder/Poem: Example: Final Draft.md", "", "Folder/Poem: Example: Final Draft.md"], + ["i:Folder/Poem: Example.md", "i:", "Folder/Poem: Example.md"], + ])("normalises the complete path body for %s", async (filename, prefix, body) => { + const { path2id } = await import("./utils.ts"); + + const result = await path2id(filename as never, false, false); + + expect(mocks.normalizePath).toHaveBeenCalledWith(body); + expect(mocks.path2idBase).toHaveBeenCalledWith(`${prefix}normalised(${body})`, false, false); + expect(result).toBe(`${prefix}normalised(${body})`); + }); + + it.each([ + ["Folder/Note.md", "", "Folder/Note.md"], + ["Folder/Poem: Example.md", "", "Folder/Poem: Example.md"], + ["Folder/Poem: Example: Final Draft.md", "", "Folder/Poem: Example: Final Draft.md"], + ["i:Folder/Poem: Example.md", "i:", "Folder/Poem: Example.md"], + ])("preserves the path namespace while normalising %s", async (filename, prefix, body) => { + mocks.id2pathBase.mockReturnValue(filename); + const { id2path } = await import("./utils.ts"); + + const result = id2path(filename as never); + + expect(mocks.normalizePath).toHaveBeenCalledWith(body); + expect(result).toBe(`${prefix}normalised(${body})`); + }); +}); diff --git a/src/common/utils.ts b/src/common/utils.ts index f9d60799..f4bc621d 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -7,6 +7,7 @@ import { isValidFilenameInWidows, isValidFilenameInAndroid, stripAllPrefixes, + expandFilePathPrefix, } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger"; @@ -41,22 +42,18 @@ export async function path2id( obfuscatePassphrase: string | false, caseInsensitive: boolean ): Promise { - const temp = filename.split(":"); - const path = temp.pop(); + const [prefix, path] = expandFilePathPrefix(filename); const normalizedPath = normalizePath(path as FilePath); - temp.push(normalizedPath); - const fixedPath = temp.join(":") as FilePathWithPrefix; + const fixedPath = `${prefix}${normalizedPath}` as FilePathWithPrefix; const out = await path2id_base(fixedPath, obfuscatePassphrase, caseInsensitive); return out; } export function id2path(id: DocumentID, entry?: EntryHasPath): FilePathWithPrefix { const filename = id2path_base(id, entry); - const temp = filename.split(":"); - const path = temp.pop(); + const [prefix, path] = expandFilePathPrefix(filename); const normalizedPath = normalizePath(path as FilePath); - temp.push(normalizedPath); - const fixedPath = temp.join(":") as FilePathWithPrefix; + const fixedPath = `${prefix}${normalizedPath}` as FilePathWithPrefix; return fixedPath; } diff --git a/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts b/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts index db5ec9d4..2fa97048 100644 --- a/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts +++ b/src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts @@ -1,7 +1,11 @@ import { promiseWithResolvers } from "octagonal-wheels/promises"; import { reactiveSource } from "octagonal-wheels/dataobject/reactive"; import { describe, expect, it, vi } from "vitest"; -import { VER, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { VER, type EntryDoc, type FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + isValidFilenameInAndroid, + isValidFilenameInWidows, +} from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { defaultLogger, LOG_LEVEL_DEBUG, @@ -28,6 +32,7 @@ function note(id: string): PouchDB.Core.ExistingDocument { type SetupOptions = { applicationReady?: boolean; + isValidPath?: (path: string) => boolean; processSynchroniseResult?: (entry: unknown) => Promise; setSnapshot?: (key: string, value: unknown) => Promise; }; @@ -38,6 +43,8 @@ function setup(options: SetupOptions = {}) { const runBoundedLocalApplicationActivity = vi.fn(async (task: () => Promise) => await task()); const onCloseActiveReplication = vi.fn(async () => true); const isReady = vi.fn(() => options.applicationReady ?? true); + const isValidPath = vi.fn(options.isValidPath ?? (() => true)); + const getDBEntryFromMeta = vi.fn(async (entry: object) => ({ ...entry, data: "x" })); const core = { services: { appLifecycle: { isReady, isSuspended: () => false }, @@ -54,13 +61,13 @@ function setup(options: SetupOptions = {}) { vault: { isTargetFile: vi.fn(async () => true), isFileSizeTooLarge: vi.fn(() => false), - isValidPath: vi.fn(() => true), + isValidPath, }, }, kvDB: { set: setSnapshot }, localDatabase: { getRaw: vi.fn(async (id: string) => ({ _id: id, _rev: "1-test" })), - getDBEntryFromMeta: vi.fn(async (entry: object) => ({ ...entry, data: "x" })), + getDBEntryFromMeta, }, }; const processor = new ReplicateResultProcessor({ @@ -74,7 +81,9 @@ function setup(options: SetupOptions = {}) { services: core.services, } as never); return { + getDBEntryFromMeta, isReady, + isValidPath, onCloseActiveReplication, processor, processSynchroniseResult, @@ -83,6 +92,24 @@ function setup(options: SetupOptions = {}) { } describe("ReplicateResultProcessor", () => { + it.each([ + ["Windows", isValidFilenameInWidows], + ["Android", isValidFilenameInAndroid], + ])("does not reflect a replicated colon path into the %s Vault", async (_platform, validatePath) => { + const path = "Folder/Poem: Example.md" as FilePathWithPrefix; + const document = { ...note("colon-path"), path }; + const { getDBEntryFromMeta, isValidPath, processor, processSynchroniseResult } = setup({ + isValidPath: validatePath, + }); + + processor.enqueueAll([document]); + + await vi.waitFor(() => expect(isValidPath).toHaveBeenCalledWith(path)); + await vi.waitFor(() => expect(processor["_processingChanges"]).toHaveLength(0)); + expect(getDBEntryFromMeta).toHaveBeenCalledWith(expect.objectContaining({ path }), false, true); + expect(processSynchroniseResult).not.toHaveBeenCalled(); + }); + it("resumes another document after in-flight updates to one document fill the application slots", async () => { const hotGate = promiseWithResolvers(); const { processor, processSynchroniseResult } = setup({ diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index 7aaf3dee..b0d908f5 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -82,7 +82,7 @@ npm run test:e2e:obsidian:focused -- security-seed-reconnect The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. -`folder-batch` needs no remote service. It creates 24 notes in nested folders, renames and deletes the parent through the Obsidian Vault API, and checks descendant events, content, Chunks, deletion markers, and provenance. A note outside the parent must remain writable. +`folder-batch` needs no remote service. It creates 24 notes and imports three notes with colons in their names into nested folders, reflects the database content into the existing files, then renames and deletes the parent through the Obsidian Vault API. It checks exact paths, descendant events, content, Chunks, deletion markers, provenance, and the absence of unexpected files. A note outside the parent must remain writable. The colon fixtures use the adapter to represent externally created files because Obsidian's Vault creation API rejects those names. The scenario also seeds a database-only colon-named note and verifies that Obsidian's refusal to create it preserves its Metadata without writing a differently named Vault file. It does not establish successful restoration of that absent note or behaviour on other operating systems. `stale-file-restart` needs no remote service. It advances the local database while old Vault bytes remain, persists pending storage events, and restarts the same isolated Vault and profile. It checks that an unchanged file with exact provenance receives the newer database content without creating a revision, that unknown-origin content is preserved on a fresh independent branch, and that losing provenance and processing the file again does not duplicate or automatically merge that branch. The database advance and pending snapshot are controlled fixtures; startup processing, persistence, file reflection, and conflict checking run in real Obsidian. The scenario does not simulate a mobile operating system suspending the application. diff --git a/test/e2e-obsidian/scripts/folder-batch.ts b/test/e2e-obsidian/scripts/folder-batch.ts index 54bc572a..94018137 100644 --- a/test/e2e-obsidian/scripts/folder-batch.ts +++ b/test/e2e-obsidian/scripts/folder-batch.ts @@ -8,11 +8,17 @@ process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; const originalRoot = "batch/original"; const renamedRoot = "batch/renamed"; const outsidePath = "batch/outside.md"; +const missingColonPath = "batch/incoming/Poem: Example.md"; const folders = ["alpha", "alpha/deep", "beta"]; const notes = Array.from({ length: 24 }, (_, index) => ({ relativePath: `${folders[index % folders.length]}/note-${index}.md`, body: `# Descendant ${index}\n\nThis body must survive a parent folder rename.\n`, })); +notes.push( + { relativePath: "alpha/Poem: Example.md", body: "First poem\n" }, + { relativePath: "beta/Poem: Example.md", body: "Second poem\n" }, + { relativePath: "alpha/deep/Poem: Part: Example.md", body: "Poem with multiple colons\n" } +); async function main(): Promise { const binary = requireObsidianBinary(); @@ -45,7 +51,7 @@ async function main(): Promise { localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), }); await waitForLiveSyncCoreReady(cliBinary, session.cliEnv); - const result = await evalObsidianJson<{ descendants: number; renamed: number; deleted: number }>( + const result = await evalObsidianJson<{ descendants: number; renamed: number; deleted: number; missingRejected: boolean }>( cliBinary, `(async()=>{ const core=app.plugins.plugins['obsidian-livesync'].core; @@ -54,6 +60,7 @@ async function main(): Promise { const originalRoot=${JSON.stringify(originalRoot)}; const renamedRoot=${JSON.stringify(renamedRoot)}; const outsidePath=${JSON.stringify(outsidePath)}; + const missingColonPath=${JSON.stringify(missingColonPath)}; const renamed=new Set(), deleted=new Set(); const refs=[ app.vault.on('rename',(file,oldPath)=>{ @@ -112,11 +119,23 @@ async function main(): Promise { await app.vault.createFolder(originalRoot); for(const folder of ${JSON.stringify(folders)}) await app.vault.createFolder(originalRoot+'/'+folder); - await Promise.all(notes.map(note=>app.vault.create(originalRoot+'/'+note.relativePath,note.body))); + // Obsidian indexes imported colon names but rejects them in Vault.create. + await Promise.all(notes.map(note=>note.relativePath.includes(':') + ? app.vault.adapter.write(originalRoot+'/'+note.relativePath,note.body) + : app.vault.create(originalRoot+'/'+note.relativePath,note.body))); await app.vault.create(outsidePath,'Outside note'); await waitFor('Initial batch',async()=>[ ...await liveBatch(originalRoot), ...await liveErrors(outsidePath,'Outside note'), ]); + for(const note of notes){ + const path=originalRoot+'/'+note.relativePath; + if(!await core.serviceModules.fileHandler.dbToStorage(await meta(path),null,true)) + throw new Error('Database reflection failed: '+path); + } + await waitFor('Reflected batch',()=>liveBatch(originalRoot)); + const expectedPaths=new Set([outsidePath,...notes.map(note=>originalRoot+'/'+note.relativePath)]); + const unexpected=app.vault.getFiles().map(file=>file.path).filter(path=>!expectedPaths.has(path)); + if(unexpected.length) throw new Error('Unexpected reflected files: '+unexpected.join(', ')); const originalIds=await Promise.all(notes.map(async note=>(await meta(originalRoot+'/'+note.relativePath))._id)); // Rename the parent once: Obsidian must emit every descendant event. @@ -144,7 +163,40 @@ async function main(): Promise { if(app.vault.getAbstractFileByPath(renamedRoot)) throw new Error('Deleted folder remains'); await app.vault.modify(app.vault.getAbstractFileByPath(outsidePath),'Outside note updated'); await waitFor('Outside update',()=>liveErrors(outsidePath,'Outside note updated')); - return JSON.stringify({descendants:notes.length,renamed:renamed.size,deleted:deleted.size}); + + // A received database entry must not create a different Vault file when Obsidian rejects its name. + const filesBefore=new Set(app.vault.getFiles().map(file=>file.path)); + const incomingBody='Received colon note\\n'; + const incomingData=new Blob([incomingBody],{type:'text/plain'}); + const incomingId=await core.services.path.path2id(missingColonPath); + const incomingTime=Date.now(); + const saved=await core.localDatabase.putDBEntry({ + _id:incomingId,path:missingColonPath,data:incomingData, + ctime:incomingTime,mtime:incomingTime,size:incomingData.size, + children:[],datatype:'plain',type:'plain',eden:{}, + }); + if(!saved?.ok) throw new Error('Could not seed received Metadata: '+missingColonPath); + const incomingMeta=await meta(missingColonPath); + if(!incomingMeta || incomingMeta._id!==incomingId || incomingMeta.path!==missingColonPath) + throw new Error('Received Metadata has the wrong path: '+missingColonPath); + const incomingEntry=await core.localDatabase.getDBEntry(missingColonPath,{rev:incomingMeta._rev},false,true,true); + if(!incomingEntry || getContent(incomingEntry)!==incomingBody) + throw new Error('Received content could not be read: '+missingColonPath); + let creationFailure=''; + try{ + const reflected=await core.serviceModules.fileHandler.dbToStorage(incomingMeta,null,true); + if(reflected) throw new Error('Obsidian unexpectedly created: '+missingColonPath); + }catch(error){ + creationFailure=String(error); + if(!creationFailure.includes('File name cannot contain')) throw error; + } + if(!creationFailure) throw new Error('Missing name rejection: '+missingColonPath); + const filesAfter=app.vault.getFiles().map(file=>file.path); + const newFiles=filesAfter.filter(path=>!filesBefore.has(path)); + if(newFiles.length) throw new Error('Received note was written under another name: '+newFiles.join(', ')); + if((await meta(missingColonPath))?.path!==missingColonPath) + throw new Error('Received Metadata changed after rejection: '+missingColonPath); + return JSON.stringify({descendants:notes.length,renamed:renamed.size,deleted:deleted.size,missingRejected:true}); }finally{ for(const ref of refs) app.vault.offref(ref); } @@ -153,7 +205,8 @@ async function main(): Promise { ); console.log( `Folder batch: ${result.descendants} descendants persisted, renamed, and deleted; ` + - `${result.renamed} rename and ${result.deleted} delete events observed; outside note remained writable.` + `${result.renamed} rename and ${result.deleted} delete events observed; outside note remained writable; ` + + `missing colon note rejected without an alternate file: ${result.missingRejected}.` ); } finally { if (session) await session.app.stop(); diff --git a/updates.md b/updates.md index 9ab6bf94..3047ad7c 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### Synchronisation and storage + +#### Fixed + +- Files with colons in their names now retain their full paths in synchronisation data instead of appearing as incorrectly named copies at the Vault root. (#1206) + - Obsidian may refuse to create a missing file with such a name. LiveSync also treats these names as invalid on Windows and Android, so the file may not appear in those devices' Vaults. Existing misplaced copies are left for you to review; this change does not remove them automatically. + ## 1.0.30 18th September, 2026