Merge pull request #1171 from vrtmrz/test/partial-startup-file-failure-e2e

test: cover partial start-up file failures in real Obsidian

package.json

@@ -73,6 +73,7 @@
         "pretest:e2e:obsidian:p2p-connection-check": "npm run build && npm run build --workspace webpeer",
         "test:e2e:obsidian:p2p-connection-check": "tsx test/e2e-obsidian/scripts/p2p-connection-check.ts",
         "test:e2e:obsidian:p2p-connection-check:services": "npm run test:e2e:obsidian:p2p-connection-check -- --manage-p2p",
+        "test:e2e:obsidian:partial-startup-file-failure": "tsx test/e2e-obsidian/scripts/partial-startup-file-failure.ts",
         "test:e2e:obsidian:startup-scan": "tsx test/e2e-obsidian/scripts/startup-scan.ts",
         "test:e2e:obsidian:setup-uri-workflow": "tsx test/e2e-obsidian/scripts/setup-uri-workflow.ts",
         "test:e2e:obsidian:two-vault-sync": "tsx test/e2e-obsidian/scripts/two-vault-sync.ts",

src/common/messages/combinedMessages.prod.ts

@@ -4213,7 +4213,7 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
         "zh-tw": "正在等待就緒⋯",
     },
     "moduleLog.pathComponentTooLong": {
-        def: "A file or folder name exceeds ${maxBytes} UTF-8 bytes and may not work on some Android and Linux file systems: ${components}",
+        def: "This path contains a file or folder name longer than ${maxBytes} UTF-8 bytes. It may not work on some Android and Linux file systems.",
     },
     "moduleLog.showLog": {
         def: "Show Log",

src/common/messagesJson/en.json

@@ -483,7 +483,7 @@
     "moduleLiveSyncMain.optionResumeAndRestart": "Resume and restart Obsidian",
     "moduleLiveSyncMain.titleScramEnabled": "Scram Enabled",
     "moduleLocalDatabase.logWaitingForReady": "Waiting for ready...",
-    "moduleLog.pathComponentTooLong": "A file or folder name exceeds ${maxBytes} UTF-8 bytes and may not work on some Android and Linux file systems: ${components}",
+    "moduleLog.pathComponentTooLong": "This path contains a file or folder name longer than ${maxBytes} UTF-8 bytes. It may not work on some Android and Linux file systems.",
     "moduleLog.showLog": "Show Log",
     "moduleMigration.fix0256.buttons.checkItLater": "Check it later",
     "moduleMigration.fix0256.buttons.DismissForever": "I have fixed it, and do not ask again",

src/common/messagesYAML/en.yaml

@@ -733,8 +733,8 @@ moduleLocalDatabase:
     logWaitingForReady: Waiting for ready...
 moduleLog:
     pathComponentTooLong: >-
-        A file or folder name exceeds ${maxBytes} UTF-8 bytes and may not work on
-        some Android and Linux file systems: ${components}
+        This path contains a file or folder name longer than ${maxBytes} UTF-8
+        bytes. It may not work on some Android and Linux file systems.
     showLog: Show Log
 moduleMigration:
     fix0256:

src/common/translation.unit.spec.ts

@@ -21,6 +21,23 @@ describe("LiveSync-owned translation catalogue", () => {
         expect($msg("moduleCheckRemoteSize.optionIncreaseLimit", { newMax: "800" }, "def")).toBe("increase to 800MB");
     });
 
+    it("keeps the active-file path compatibility warning concise", () => {
+        const oversizedComponent = `${"界".repeat(86)} (258 bytes)`;
+
+        expect(
+            $msg(
+                "moduleLog.pathComponentTooLong",
+                {
+                    maxBytes: "255",
+                    components: oversizedComponent,
+                },
+                "def"
+            )
+        ).toBe(
+            "This path contains a file or folder name longer than 255 UTF-8 bytes. It may not work on some Android and Linux file systems."
+        );
+    });
+
     it("uses Commonlib's canonical English when the application catalogue has no translation", () => {
         setLang("es");
 

src/modules/features/ModuleLog.ts

@@ -299,13 +299,9 @@ export class ModuleLog extends AbstractObsidianModule {
         }
         const oversizedPathComponents = findPathComponentsExceedingUtf8Limit(thisFile.path);
         if (oversizedPathComponents.length > 0) {
-            const components = oversizedPathComponents
-                .map(({ component, utf8Bytes }) => `${component} (${utf8Bytes} bytes)`)
-                .join(", ");
             reasonWarn.push(
                 $msg("moduleLog.pathComponentTooLong", {
                     maxBytes: `${ANDROID_LINUX_PATH_COMPONENT_UTF8_WARNING_BOUNDARY}`,
-                    components,
                 })
             );
         }

test/e2e-obsidian/README.md

@@ -166,6 +166,8 @@ LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) --
 
 `test:e2e:obsidian:startup-scan` starts from a CouchDB fixture using current settings with its device-local compatibility marker already acknowledged, stops Obsidian, writes a note directly into the Vault, restarts the same isolated Vault and profile without rewriting its plug-in data, and verifies from CouchDB that the start-up scan picked up the offline file. Onboarding remains covered by `onboarding-invitation`; this scenario owns the ordinary configured restart and start-up scan.
 
+`test:e2e:obsidian:partial-startup-file-failure` is a focused Linux release-acceptance scenario for an ordinary configured restart. It stores one valid database-only note and one database-only note whose path component is 258 UTF-8 bytes, then restarts the same isolated Vault and profile. On a Linux test Vault which enforces the conventional 255-byte component limit, the scenario requires the valid file to be reflected, the application to become ready, the partial-failure Notice to appear, and the failed path to remain readable and eligible for a later scan with its exact path in the verbose log. It remains outside `local-suite` because the failure fixture is deliberately platform-specific.
+
 `test:e2e:obsidian:setup-uri-workflow` runs the repository's public Commonlib-backed CouchDB provisioning and Setup URI tools against the local CouchDB fixture. It configures a new, empty Vault in the first real Obsidian session through the visible onboarding wizard and uses Rebuild. After that device is working, it generates a new Setup URI through the registered command; the second real Obsidian Vault uses that URI for Fetch instead of reusing the initial Setup URI produced by the provisioning tool. The workflow verifies ordinary notes from the first device to the second and back again, independently enables Hidden File Sync on each device, and verifies a snippet. The retained Setup URI screenshots show only encrypted URIs and visually masked Setup URI passphrases; plaintext credentials are not captured. Files prefixed with `guide-` capture the relevant dialogue, settings panel, or workspace leaf without transient Notices. Public documentation copies selected images only after visual inspection; the E2E run does not overwrite repository documentation assets.
 
 `test:e2e:obsidian:two-vault-sync` runs a two-vault note synchronisation workflow. It verifies note creation, update, ordinary rename, a case-only file name change within the same directory, deletion, and a separate encrypted round-trip with Path Obfuscation enabled. Its target-filter scenario confirms that one Vault receives and checkpoints a remote document without reflecting it, restarts with the same profile and filter, and then reflects the stored document after the filter is broadened through the settings service. Directory case changes deliberately remain outside this scenario because they require directory-aware rename handling. The optional Markdown conflict check can be enabled with `E2E_OBSIDIAN_INCLUDE_MARKDOWN_CONFLICT=true`. It creates divergent revisions in two separate Vaults, performs a conservative merge on one Vault, edits that result again, and requires the other Vault to replace its known deleted losing revision without recreating the conflict. The separate `E2E_OBSIDIAN_INCLUDE_CONFLICT_OPERATIONS=true` check keeps four conflicts active while one Vault edits, deletes, performs a case-only rename, and performs a cross-path rename. It asserts that each operation extends the revision displayed on that device, replicates the exact resulting revision tree, and preserves the other conflict branch. During focused development, `E2E_OBSIDIAN_ONLY_CONFLICT_OPERATIONS=true` runs that self-contained scope without the ordinary, target-filter, or encrypted scenarios. Both conflict checks remain outside the default local suite.

test/e2e-obsidian/scripts/partial-startup-file-failure.ts

@@ -0,0 +1,301 @@
+/**
+ * Proves that one file which cannot be reflected during an ordinary start-up
+ * does not keep the entire configured application unready.
+ *
+ * The fixture relies on the conventional Linux 255-byte path component
+ * limit. It stores one ordinary note and one note with a 258-byte component in
+ * the local database, then restarts the same real Obsidian Vault and profile.
+ */
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+    assertCouchDbReachable,
+    createCouchDbDatabase,
+    deleteCouchDbDatabase,
+    loadCouchDbConfig,
+    makeUniqueDatabaseName,
+} from "../runner/couchdb.ts";
+import { evalObsidianJson } from "../runner/cli.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertEqual,
+    createE2eCouchDbPluginData,
+    createE2eObsidianDeviceLocalState,
+    prepareRemote,
+    waitForLiveSyncCoreReady,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { withObsidianPage } from "../runner/ui.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
+
+const validPath = "E2E/partial-startup-valid.md";
+const oversizedComponent = `${"界".repeat(85)}.md`;
+const failedPath = `E2E/${oversizedComponent}`;
+const validContent = `# Partial start-up\n\n${"V".repeat(4096)}\n`;
+const failedContent = `# Retry this file\n\n${"R".repeat(4096)}\n`;
+const partialFailureNotice =
+    "Not all files could be synchronised. Check the affected files. Generate a report to review the detailed log.";
+const failedPathLog =
+    `Offline scan failed to synchronise ${failedPath} between storage and the local database; ` +
+    "this path remains eligible for a later scan.";
+const assertionTimeoutMs = Number(process.env.E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS ?? 20000);
+
+type SeededEntry = {
+    id: string;
+    path: string;
+    revision: string;
+    children: string[];
+};
+
+type FailedPathState = {
+    appReady: boolean;
+    databaseReady: boolean;
+    fileExists: boolean;
+    entryReadable: boolean;
+    metadataRevision?: string;
+    provenance: { revision: string; observedStorageMtime?: number } | null;
+    logText: string;
+};
+
+type RetryState = Omit<FailedPathState, "databaseReady" | "logText"> & {
+    scanResult: string | false;
+};
+
+async function seedDatabaseOnlyEntries(cliBinary: string, env: NodeJS.ProcessEnv): Promise<SeededEntry[]> {
+    return await evalObsidianJson<SeededEntry[]>(
+        cliBinary,
+        [
+            "(async()=>{",
+            `const fixtures=${JSON.stringify([
+                { path: validPath, content: validContent },
+                { path: failedPath, content: failedContent },
+            ])};`,
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            "const seeded=[];",
+            "for(const {path,content} of fixtures){",
+            "  if(app.vault.getAbstractFileByPath(path)!==null){",
+            "    throw new Error(`Database-only fixture already exists in the Vault: ${path}`);",
+            "  }",
+            "  const blob=new Blob([content],{type:'text/plain'});",
+            "  const id=await core.services.path.path2id(path);",
+            "  const now=Date.now();",
+            "  const result=await core.localDatabase.putDBEntry({",
+            "    _id:id,path,data:blob,ctime:now,mtime:now,",
+            "    size:(await blob.arrayBuffer()).byteLength,children:[],",
+            "    datatype:'plain',type:'plain',eden:{},",
+            "  });",
+            "  if(!result?.ok) throw new Error(`Could not seed database-only fixture: ${path}`);",
+            "  const metadata=await core.localDatabase.getDBEntryMeta(path,undefined,true);",
+            "  if(!metadata) throw new Error(`Could not reload seeded Metadata: ${path}`);",
+            "  seeded.push({id,path,revision:result.rev,children:metadata.children??[]});",
+            "}",
+            "return JSON.stringify(seeded);",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function observePartialFailureNotice(remoteDebuggingPort: number): Promise<void> {
+    await withObsidianPage(remoteDebuggingPort, async (page) => {
+        await page
+            .locator(".notice")
+            .filter({ hasText: partialFailureNotice })
+            .first()
+            .waitFor({ state: "visible", timeout: assertionTimeoutMs });
+    });
+}
+
+async function inspectFailedPathState(cliBinary: string, env: NodeJS.ProcessEnv): Promise<FailedPathState> {
+    return await evalObsidianJson<FailedPathState>(
+        cliBinary,
+        [
+            "(async()=>{",
+            `const path=${JSON.stringify(failedPath)};`,
+            `const expectedLog=${JSON.stringify(failedPathLog)};`,
+            `const timeoutMs=${JSON.stringify(assertionTimeoutMs)};`,
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            "const metadata=await core.localDatabase.getDBEntryMeta(path,undefined,true);",
+            "const entry=await core.localDatabase.getDBEntry(path,undefined,false,true,true);",
+            "const provenanceStore=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1');",
+            "const provenance=(await provenanceStore.get(path))??null;",
+            "await core.services.API.showWindow('log-log');",
+            "const deadline=Date.now()+timeoutMs;",
+            "const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));",
+            "let logText='';",
+            "while(Date.now()<deadline){",
+            "  logText=Array.from(document.querySelectorAll('.logpane .log pre'))",
+            "    .map((element)=>element.textContent??'').join('\\n');",
+            "  if(logText.includes(expectedLog)) break;",
+            "  await sleep(100);",
+            "}",
+            "for(const leaf of app.workspace.getLeavesOfType('log-log')) leaf.detach();",
+            "return JSON.stringify({",
+            "  appReady:core.services.appLifecycle.isReady(),",
+            "  databaseReady:core.services.database.isDatabaseReady(),",
+            "  fileExists:app.vault.getAbstractFileByPath(path)!==null,",
+            "  entryReadable:entry!==false,",
+            "  metadataRevision:metadata?._rev,",
+            "  provenance,",
+            "  logText,",
+            "});",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function retryFailedPath(cliBinary: string, env: NodeJS.ProcessEnv): Promise<RetryState> {
+    return await evalObsidianJson<RetryState>(
+        cliBinary,
+        [
+            "(async()=>{",
+            `const path=${JSON.stringify(failedPath)};`,
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            "const scanResult=await core.services.vault.scanVault(false,false,true);",
+            "const metadata=await core.localDatabase.getDBEntryMeta(path,undefined,true);",
+            "const entry=await core.localDatabase.getDBEntry(path,undefined,false,true,true);",
+            "const provenanceStore=core.services.keyValueDB.openSimpleStore('file-reflection-provenance-v1');",
+            "return JSON.stringify({",
+            "  scanResult,",
+            "  appReady:core.services.appLifecycle.isReady(),",
+            "  fileExists:app.vault.getAbstractFileByPath(path)!==null,",
+            "  entryReadable:entry!==false,",
+            "  metadataRevision:metadata?._rev,",
+            "  provenance:(await provenanceStore.get(path))??null,",
+            "});",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function main(): Promise<void> {
+    if (process.platform !== "linux") {
+        throw new Error("The partial start-up file-failure scenario currently requires a Linux test Vault.");
+    }
+    assertEqual(
+        Buffer.byteLength(oversizedComponent, "utf8"),
+        258,
+        "The failing path component no longer exercises the intended UTF-8 byte boundary."
+    );
+
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) {
+        throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`);
+    }
+
+    const couchDb = await loadCouchDbConfig();
+    const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "partial-startup-file-failure");
+    const couchDbSettings = {
+        uri: couchDb.uri,
+        username: couchDb.username,
+        password: couchDb.password,
+        dbName,
+    };
+    const vault = await createTemporaryVault("obsidian-livesync-partial-startup-");
+    let session: ObsidianLiveSyncSession | undefined;
+
+    try {
+        await assertCouchDbReachable(couchDb);
+        await createCouchDbDatabase(couchDb, dbName);
+
+        console.log(`Using Obsidian executable: ${binary}`);
+        console.log(`Temporary vault: ${vault.path}`);
+        console.log(`Temporary CouchDB database: ${dbName}`);
+
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: cli.binary,
+            vault,
+            startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
+            pluginData: createE2eCouchDbPluginData(couchDbSettings, {
+                showVerboseLog: true,
+                lessInformationInLog: false,
+            }),
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+        });
+        await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
+        await prepareRemote(cli.binary, session.cliEnv);
+
+        const seeded = await seedDatabaseOnlyEntries(cli.binary, session.cliEnv);
+        const validSeed = seeded.find((entry) => entry.path === validPath);
+        const failedSeed = seeded.find((entry) => entry.path === failedPath);
+        if (!validSeed || !failedSeed) throw new Error("The database-only start-up fixtures were incomplete.");
+        if (validSeed.children.length === 0 || failedSeed.children.length === 0) {
+            throw new Error("The database-only fixtures did not create independently stored chunks.");
+        }
+
+        await session.app.stop();
+        session = undefined;
+
+        let partialNoticeObserved = false;
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: cli.binary,
+            vault,
+            pluginStartup: "natural",
+            startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
+            lifecycle: {
+                afterPluginLoad: async ({ remoteDebuggingPort }) => {
+                    await observePartialFailureNotice(remoteDebuggingPort);
+                    partialNoticeObserved = true;
+                },
+            },
+        });
+        const readiness = await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
+        assertEqual(readiness.configured, true, "Self-hosted LiveSync lost its configuration on restart.");
+        assertEqual(partialNoticeObserved, true, "The partial start-up failure Notice was not observed.");
+
+        assertEqual(
+            await readFile(join(vault.path, validPath), "utf8"),
+            validContent,
+            "The valid database-only file was not reflected during the same start-up scan."
+        );
+
+        const state = await inspectFailedPathState(cli.binary, session.cliEnv);
+        assertEqual(state.databaseReady, true, "The local database did not remain ready after one file failed.");
+        assertEqual(state.appReady, true, "One file failure kept the application unready.");
+        assertEqual(state.fileExists, false, "The overlong path was unexpectedly reflected to the Linux Vault.");
+        assertEqual(state.entryReadable, true, "The failed database entry was no longer readable.");
+        assertEqual(state.metadataRevision, failedSeed.revision, "The failed database entry revision changed.");
+        assertEqual(state.provenance, null, "A failed reflection was recorded as successful provenance.");
+        assertEqual(
+            state.logText.includes(failedPathLog),
+            true,
+            "The verbose log did not identify the path which failed during the start-up scan."
+        );
+
+        const retry = await retryFailedPath(cli.binary, session.cliEnv);
+        assertEqual(
+            retry.scanResult,
+            "completed-with-file-failures",
+            "A later scan did not retry and report the same individual file failure."
+        );
+        assertEqual(retry.appReady, true, "Retrying the failed path cleared application readiness.");
+        assertEqual(retry.fileExists, false, "The overlong path was unexpectedly reflected during retry.");
+        assertEqual(retry.entryReadable, true, "Retrying removed the failed database entry.");
+        assertEqual(retry.metadataRevision, failedSeed.revision, "Retrying changed the failed database revision.");
+        assertEqual(retry.provenance, null, "Retrying recorded a failed reflection as successful provenance.");
+
+        console.log(`Ordinary start-up remained ready, reflected ${validPath}, and retained ${failedPath} for retry.`);
+    } finally {
+        if (session) {
+            await session.app.stop();
+        }
+        await vault.dispose();
+        if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
+            await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => {
+                console.warn(error instanceof Error ? error.message : error);
+            });
+        }
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exit(1);
+});

test/e2e-obsidian/scripts/run-focused.ts

@@ -22,6 +22,7 @@ const focusedScenarios = new Set([
     "minio-upload",
     "object-storage-setup-uri-workflow",
     "p2p-setup-uri-workflow",
+    "partial-startup-file-failure",
     "startup-scan",
     "setup-uri-workflow",
     "two-vault-sync",

updates.md

@@ -23,11 +23,11 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi
 #### Improved
 
 - Start-up now keeps unconfigured Vaults on the onboarding path without running configured-only checks or accepting Config Doctor and incomplete-document repair requests. Returning a configured Vault to an unconfigured state also retires those requests for the current plug-in process, so completing setup admits them only after the requested restart.
-- The active-file warning now identifies file or folder names longer than 255 UTF-8 bytes as an Android and Linux compatibility risk, without rejecting or changing the path.
+- The active-file warning now concisely identifies file or folder names longer than 255 UTF-8 bytes as an Android and Linux compatibility risk, without rejecting or changing the path.
 
 ### Testing
 
-- Start-up migrations, integrity checks, Config Doctor, basic commands, and the Obsidian replication ribbon now have focused regression tests for their service composition. Real Obsidian checks cover unconfigured onboarding, configured start-up scanning, Config Doctor detection and layout, command registration, and the established ribbon icon.
+- Start-up migrations, integrity checks, Config Doctor, basic commands, and the Obsidian replication ribbon now have focused regression tests for their service composition. Real Obsidian checks cover unconfigured onboarding, configured start-up scanning and individual file failures, Config Doctor detection and layout, command registration, and the established ribbon icon.
 
 ## 1.0.24